diff --git a/common_arm64.h b/common_arm64.h index d37a4a9cba..45e7bbcbf4 100644 --- a/common_arm64.h +++ b/common_arm64.h @@ -151,6 +151,12 @@ static inline int blas_quickdivide(blasint x, blasint y){ #if defined(ASSEMBLER) && !defined(NEEDPARAM) +#if defined(__APPLE__) +#define L(x) L##x +#else +#define L(x) .L##x +#endif + .macro PROLOGUE .text ; .p2align 2 ; @@ -162,7 +168,11 @@ static inline int blas_quickdivide(blasint x, blasint y){ .endm +#if defined(__APPLE__) +#define EPILOGUE .subsections_via_symbols +#else #define EPILOGUE +#endif #define PROFCODE diff --git a/common_x86_64.h b/common_x86_64.h index 425bef4e5f..3d38ee3ebb 100644 --- a/common_x86_64.h +++ b/common_x86_64.h @@ -405,6 +405,12 @@ static __inline unsigned int blas_quickdivide(unsigned int x, unsigned int y){ #define REALNAME ASMFNAME #endif +#ifdef OS_DARWIN +#define L(x) L##x +#else +#define L(x) .L##x +#endif + #ifdef OS_DARWIN #define PROLOGUE .text;.align 5; .globl REALNAME; REALNAME: #define PROLOGUE_EXPORT .text;.align 5; .globl REALNAME; REALNAME: diff --git a/kernel/arm64/amax.S b/kernel/arm64/amax.S index f535ddf273..08a69cef24 100644 --- a/kernel/arm64/amax.S +++ b/kernel/arm64/amax.S @@ -160,62 +160,62 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Lamax_kernel_zero + ble L(amax_kernel_zero) cmp INC_X, xzr - ble .Lamax_kernel_zero + ble L(amax_kernel_zero) cmp INC_X, #1 - bne .Lamax_kernel_S_BEGIN + bne L(amax_kernel_S_BEGIN) -.Lamax_kernel_F_BEGIN: +L(amax_kernel_F_BEGIN): asr I, N, #2 cmp I, xzr - beq .Lamax_kernel_F1_INIT + beq L(amax_kernel_F1_INIT) INIT_F4 subs I, I, #1 - beq .Lamax_kernel_F1 + beq L(amax_kernel_F1) -.Lamax_kernel_F4: +L(amax_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lamax_kernel_F4 + bne L(amax_kernel_F4) -.Lamax_kernel_F1: +L(amax_kernel_F1): ands I, N, #3 - ble .Lamax_kernel_L999 + ble L(amax_kernel_L999) -.Lamax_kernel_F10: +L(amax_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lamax_kernel_F10 + bne L(amax_kernel_F10) ret -.Lamax_kernel_F1_INIT: +L(amax_kernel_F1_INIT): INIT_F1 subs N, N, #1 - b .Lamax_kernel_F1 + b L(amax_kernel_F1) -.Lamax_kernel_S_BEGIN: +L(amax_kernel_S_BEGIN): INIT_S subs N, N, #1 - ble .Lamax_kernel_L999 + ble L(amax_kernel_L999) asr I, N, #2 cmp I, xzr - ble .Lamax_kernel_S1 + ble L(amax_kernel_S1) -.Lamax_kernel_S4: +L(amax_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -223,25 +223,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lamax_kernel_S4 + bne L(amax_kernel_S4) -.Lamax_kernel_S1: +L(amax_kernel_S1): ands I, N, #3 - ble .Lamax_kernel_L999 + ble L(amax_kernel_L999) -.Lamax_kernel_S10: +L(amax_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lamax_kernel_S10 + bne L(amax_kernel_S10) -.Lamax_kernel_L999: +L(amax_kernel_L999): ret -.Lamax_kernel_zero: +L(amax_kernel_zero): fmov MAXF, REG0 ret diff --git a/kernel/arm64/asum.S b/kernel/arm64/asum.S index e88eb07c2e..9ca92c392a 100644 --- a/kernel/arm64/asum.S +++ b/kernel/arm64/asum.S @@ -122,52 +122,52 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif cmp N, xzr - ble .Lasum_kernel_L999 + ble L(asum_kernel_L999) cmp INC_X, xzr - ble .Lasum_kernel_L999 + ble L(asum_kernel_L999) cmp INC_X, #1 - bne .Lasum_kernel_S_BEGIN + bne L(asum_kernel_S_BEGIN) -.Lasum_kernel_F_BEGIN: +L(asum_kernel_F_BEGIN): asr I, N, #3 cmp I, xzr - beq .Lasum_kernel_F1 + beq L(asum_kernel_F1) -.Lasum_kernel_F8: +L(asum_kernel_F8): KERNEL_F8 subs I, I, #1 - bne .Lasum_kernel_F8 + bne L(asum_kernel_F8) KERNEL_F8_FINALIZE -.Lasum_kernel_F1: +L(asum_kernel_F1): ands I, N, #7 - ble .Lasum_kernel_L999 + ble L(asum_kernel_L999) -.Lasum_kernel_F10: +L(asum_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lasum_kernel_F10 + bne L(asum_kernel_F10) -.Lasum_kernel_L999: +L(asum_kernel_L999): ret -.Lasum_kernel_S_BEGIN: +L(asum_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lasum_kernel_S1 + ble L(asum_kernel_S1) -.Lasum_kernel_S4: +L(asum_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -175,19 +175,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lasum_kernel_S4 + bne L(asum_kernel_S4) -.Lasum_kernel_S1: +L(asum_kernel_S1): ands I, N, #3 - ble .Lasum_kernel_L999 + ble L(asum_kernel_L999) -.Lasum_kernel_S10: +L(asum_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lasum_kernel_S10 + bne L(asum_kernel_S10) ret diff --git a/kernel/arm64/axpy.S b/kernel/arm64/axpy.S index 8094351105..22a9a8e4d9 100644 --- a/kernel/arm64/axpy.S +++ b/kernel/arm64/axpy.S @@ -135,53 +135,53 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Laxpy_kernel_L999 + ble L(axpy_kernel_L999) fcmp DA, #0.0 - beq .Laxpy_kernel_L999 + beq L(axpy_kernel_L999) cmp INC_X, #1 - bne .Laxpy_kernel_S_BEGIN + bne L(axpy_kernel_S_BEGIN) cmp INC_Y, #1 - bne .Laxpy_kernel_S_BEGIN + bne L(axpy_kernel_S_BEGIN) -.Laxpy_kernel_F_BEGIN: +L(axpy_kernel_F_BEGIN): asr I, N, #3 cmp I, xzr - beq .Laxpy_kernel_F1 + beq L(axpy_kernel_F1) -.Laxpy_kernel_F8: +L(axpy_kernel_F8): KERNEL_F8 subs I, I, #1 - bne .Laxpy_kernel_F8 + bne L(axpy_kernel_F8) -.Laxpy_kernel_F1: +L(axpy_kernel_F1): ands I, N, #7 - ble .Laxpy_kernel_L999 + ble L(axpy_kernel_L999) -.Laxpy_kernel_F10: +L(axpy_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Laxpy_kernel_F10 + bne L(axpy_kernel_F10) mov w0, wzr ret -.Laxpy_kernel_S_BEGIN: +L(axpy_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Laxpy_kernel_S1 + ble L(axpy_kernel_S1) -.Laxpy_kernel_S4: +L(axpy_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -189,21 +189,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Laxpy_kernel_S4 + bne L(axpy_kernel_S4) -.Laxpy_kernel_S1: +L(axpy_kernel_S1): ands I, N, #3 - ble .Laxpy_kernel_L999 + ble L(axpy_kernel_L999) -.Laxpy_kernel_S10: +L(axpy_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Laxpy_kernel_S10 + bne L(axpy_kernel_S10) -.Laxpy_kernel_L999: +L(axpy_kernel_L999): mov w0, wzr ret + + EPILOGUE diff --git a/kernel/arm64/casum.S b/kernel/arm64/casum.S index 7c82827a54..8311fed45f 100644 --- a/kernel/arm64/casum.S +++ b/kernel/arm64/casum.S @@ -98,52 +98,52 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. fmov s1, SUMF cmp N, xzr - ble .Lcasum_kernel_L999 + ble L(casum_kernel_L999) cmp INC_X, xzr - ble .Lcasum_kernel_L999 + ble L(casum_kernel_L999) cmp INC_X, #1 - bne .Lcasum_kernel_S_BEGIN + bne L(casum_kernel_S_BEGIN) -.Lcasum_kernel_F_BEGIN: +L(casum_kernel_F_BEGIN): asr I, N, #3 cmp I, xzr - beq .Lcasum_kernel_F1 + beq L(casum_kernel_F1) -.Lcasum_kernel_F8: +L(casum_kernel_F8): KERNEL_F8 subs I, I, #1 - bne .Lcasum_kernel_F8 + bne L(casum_kernel_F8) KERNEL_F8_FINALIZE -.Lcasum_kernel_F1: +L(casum_kernel_F1): ands I, N, #7 - ble .Lcasum_kernel_L999 + ble L(casum_kernel_L999) -.Lcasum_kernel_F10: +L(casum_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lcasum_kernel_F10 + bne L(casum_kernel_F10) -.Lcasum_kernel_L999: +L(casum_kernel_L999): ret -.Lcasum_kernel_S_BEGIN: +L(casum_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lcasum_kernel_S1 + ble L(casum_kernel_S1) -.Lcasum_kernel_S4: +L(casum_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -151,19 +151,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lcasum_kernel_S4 + bne L(casum_kernel_S4) -.Lcasum_kernel_S1: +L(casum_kernel_S1): ands I, N, #3 - ble .Lcasum_kernel_L999 + ble L(casum_kernel_L999) -.Lcasum_kernel_S10: +L(casum_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lcasum_kernel_S10 + bne L(casum_kernel_S10) ret diff --git a/kernel/arm64/cgemm_kernel_4x4.S b/kernel/arm64/cgemm_kernel_4x4.S index bbf0c7537c..252138e150 100644 --- a/kernel/arm64/cgemm_kernel_4x4.S +++ b/kernel/arm64/cgemm_kernel_4x4.S @@ -1072,11 +1072,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lcgemm_kernel_L2_BEGIN + ble L(cgemm_kernel_L2_BEGIN) /******************************************************************************/ -.Lcgemm_kernel_L4_BEGIN: +L(cgemm_kernel_L4_BEGIN): mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #2 @@ -1084,96 +1084,96 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array add ppA, temp, pA -.Lcgemm_kernel_L4_M8_BEGIN: +L(cgemm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lcgemm_kernel_L4_M4_BEGIN + ble L(cgemm_kernel_L4_M4_BEGIN) -.Lcgemm_kernel_L4_M8_20: +L(cgemm_kernel_L4_M8_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lcgemm_kernel_L4_M8_32 + blt L(cgemm_kernel_L4_M8_32) KERNEL8x4_I // do one in the K KERNEL8x4_M2 // do another in the K subs counterL, counterL, #2 // subtract 2 - ble .Lcgemm_kernel_L4_M8_22a + ble L(cgemm_kernel_L4_M8_22a) .align 5 -.Lcgemm_kernel_L4_M8_22: +L(cgemm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M8_22 + bgt L(cgemm_kernel_L4_M8_22) -.Lcgemm_kernel_L4_M8_22a: +L(cgemm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_E - b .Lcgemm_kernel_L4_M8_44 + b L(cgemm_kernel_L4_M8_44) -.Lcgemm_kernel_L4_M8_32: +L(cgemm_kernel_L4_M8_32): tst counterL, #1 - ble .Lcgemm_kernel_L4_M8_40 + ble L(cgemm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_E - b .Lcgemm_kernel_L4_M8_44 + b L(cgemm_kernel_L4_M8_44) -.Lcgemm_kernel_L4_M8_40: +L(cgemm_kernel_L4_M8_40): INIT8x4 -.Lcgemm_kernel_L4_M8_44: +L(cgemm_kernel_L4_M8_44): ands counterL , origK, #1 - ble .Lcgemm_kernel_L4_M8_100 + ble L(cgemm_kernel_L4_M8_100) -.Lcgemm_kernel_L4_M8_46: +L(cgemm_kernel_L4_M8_46): KERNEL8x4_SUB -.Lcgemm_kernel_L4_M8_100: +L(cgemm_kernel_L4_M8_100): SAVE8x4 -.Lcgemm_kernel_L4_M8_END: +L(cgemm_kernel_L4_M8_END): lsl temp, origK, #5 // k * 4 * 8 add pA, pA, temp add ppA, ppA, temp subs counterI, counterI, #1 - bne .Lcgemm_kernel_L4_M8_20 + bne L(cgemm_kernel_L4_M8_20) -.Lcgemm_kernel_L4_M4_BEGIN: +L(cgemm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lcgemm_kernel_L4_END + ble L(cgemm_kernel_L4_END) tst counterI, #4 - ble .Lcgemm_kernel_L4_M2_BEGIN + ble L(cgemm_kernel_L4_M2_BEGIN) -.Lcgemm_kernel_L4_M4_20: +L(cgemm_kernel_L4_M4_20): INIT4x4 mov pB, origPB asr counterL, origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lcgemm_kernel_L4_M4_40 + ble L(cgemm_kernel_L4_M4_40) -.Lcgemm_kernel_L4_M4_22: +L(cgemm_kernel_L4_M4_22): KERNEL4x4_SUB KERNEL4x4_SUB @@ -1186,47 +1186,47 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M4_22 + bgt L(cgemm_kernel_L4_M4_22) -.Lcgemm_kernel_L4_M4_40: +L(cgemm_kernel_L4_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L4_M4_100 + ble L(cgemm_kernel_L4_M4_100) -.Lcgemm_kernel_L4_M4_42: +L(cgemm_kernel_L4_M4_42): KERNEL4x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M4_42 + bgt L(cgemm_kernel_L4_M4_42) -.Lcgemm_kernel_L4_M4_100: +L(cgemm_kernel_L4_M4_100): SAVE4x4 -.Lcgemm_kernel_L4_M4_END: +L(cgemm_kernel_L4_M4_END): -.Lcgemm_kernel_L4_M2_BEGIN: +L(cgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lcgemm_kernel_L4_END + ble L(cgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lcgemm_kernel_L4_M1_BEGIN + ble L(cgemm_kernel_L4_M1_BEGIN) -.Lcgemm_kernel_L4_M2_20: +L(cgemm_kernel_L4_M2_20): INIT2x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L4_M2_40 + ble L(cgemm_kernel_L4_M2_40) -.Lcgemm_kernel_L4_M2_22: +L(cgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1239,43 +1239,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M2_22 + bgt L(cgemm_kernel_L4_M2_22) -.Lcgemm_kernel_L4_M2_40: +L(cgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L4_M2_100 + ble L(cgemm_kernel_L4_M2_100) -.Lcgemm_kernel_L4_M2_42: +L(cgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M2_42 + bgt L(cgemm_kernel_L4_M2_42) -.Lcgemm_kernel_L4_M2_100: +L(cgemm_kernel_L4_M2_100): SAVE2x4 -.Lcgemm_kernel_L4_M2_END: +L(cgemm_kernel_L4_M2_END): -.Lcgemm_kernel_L4_M1_BEGIN: +L(cgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lcgemm_kernel_L4_END + ble L(cgemm_kernel_L4_END) -.Lcgemm_kernel_L4_M1_20: +L(cgemm_kernel_L4_M1_20): INIT1x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L4_M1_40 + ble L(cgemm_kernel_L4_M1_40) -.Lcgemm_kernel_L4_M1_22: +L(cgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1287,45 +1287,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M1_22 + bgt L(cgemm_kernel_L4_M1_22) -.Lcgemm_kernel_L4_M1_40: +L(cgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L4_M1_100 + ble L(cgemm_kernel_L4_M1_100) -.Lcgemm_kernel_L4_M1_42: +L(cgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M1_42 + bgt L(cgemm_kernel_L4_M1_42) -.Lcgemm_kernel_L4_M1_100: +L(cgemm_kernel_L4_M1_100): SAVE1x4 -.Lcgemm_kernel_L4_END: +L(cgemm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 subs counterJ, counterJ , #1 // j-- - bgt .Lcgemm_kernel_L4_BEGIN + bgt L(cgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Lcgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(cgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lcgemm_kernel_L999 // error, N was less than 4? + ble L(cgemm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Lcgemm_kernel_L1_BEGIN + ble L(cgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1335,24 +1335,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lcgemm_kernel_L2_M4_BEGIN: +L(cgemm_kernel_L2_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI,#0 - ble .Lcgemm_kernel_L2_M2_BEGIN + ble L(cgemm_kernel_L2_M2_BEGIN) -.Lcgemm_kernel_L2_M4_20: +L(cgemm_kernel_L2_M4_20): INIT4x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lcgemm_kernel_L2_M4_40 + ble L(cgemm_kernel_L2_M4_40) .align 5 -.Lcgemm_kernel_L2_M4_22: +L(cgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1364,50 +1364,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M4_22 + bgt L(cgemm_kernel_L2_M4_22) -.Lcgemm_kernel_L2_M4_40: +L(cgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_M4_100 + ble L(cgemm_kernel_L2_M4_100) -.Lcgemm_kernel_L2_M4_42: +L(cgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M4_42 + bgt L(cgemm_kernel_L2_M4_42) -.Lcgemm_kernel_L2_M4_100: +L(cgemm_kernel_L2_M4_100): SAVE4x2 -.Lcgemm_kernel_L2_M4_END: +L(cgemm_kernel_L2_M4_END): subs counterI, counterI, #1 - bgt .Lcgemm_kernel_L2_M4_20 + bgt L(cgemm_kernel_L2_M4_20) -.Lcgemm_kernel_L2_M2_BEGIN: +L(cgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lcgemm_kernel_L2_END + ble L(cgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lcgemm_kernel_L2_M1_BEGIN + ble L(cgemm_kernel_L2_M1_BEGIN) -.Lcgemm_kernel_L2_M2_20: +L(cgemm_kernel_L2_M2_20): INIT2x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lcgemm_kernel_L2_M2_40 + ble L(cgemm_kernel_L2_M2_40) -.Lcgemm_kernel_L2_M2_22: +L(cgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1420,43 +1420,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M2_22 + bgt L(cgemm_kernel_L2_M2_22) -.Lcgemm_kernel_L2_M2_40: +L(cgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_M2_100 + ble L(cgemm_kernel_L2_M2_100) -.Lcgemm_kernel_L2_M2_42: +L(cgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M2_42 + bgt L(cgemm_kernel_L2_M2_42) -.Lcgemm_kernel_L2_M2_100: +L(cgemm_kernel_L2_M2_100): SAVE2x2 -.Lcgemm_kernel_L2_M2_END: +L(cgemm_kernel_L2_M2_END): -.Lcgemm_kernel_L2_M1_BEGIN: +L(cgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lcgemm_kernel_L2_END + ble L(cgemm_kernel_L2_END) -.Lcgemm_kernel_L2_M1_20: +L(cgemm_kernel_L2_M1_20): INIT1x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lcgemm_kernel_L2_M1_40 + ble L(cgemm_kernel_L2_M1_40) -.Lcgemm_kernel_L2_M1_22: +L(cgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1468,36 +1468,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M1_22 + bgt L(cgemm_kernel_L2_M1_22) -.Lcgemm_kernel_L2_M1_40: +L(cgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_M1_100 + ble L(cgemm_kernel_L2_M1_100) -.Lcgemm_kernel_L2_M1_42: +L(cgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M1_42 + bgt L(cgemm_kernel_L2_M1_42) -.Lcgemm_kernel_L2_M1_100: +L(cgemm_kernel_L2_M1_100): SAVE1x2 -.Lcgemm_kernel_L2_END: +L(cgemm_kernel_L2_END): add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 /******************************************************************************/ -.Lcgemm_kernel_L1_BEGIN: +L(cgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lcgemm_kernel_L999 // done + ble L(cgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1507,24 +1507,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lcgemm_kernel_L1_M4_BEGIN: +L(cgemm_kernel_L1_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lcgemm_kernel_L1_M2_BEGIN + ble L(cgemm_kernel_L1_M2_BEGIN) -.Lcgemm_kernel_L1_M4_20: +L(cgemm_kernel_L1_M4_20): INIT4x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_M4_40 + ble L(cgemm_kernel_L1_M4_40) .align 5 -.Lcgemm_kernel_L1_M4_22: +L(cgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1536,50 +1536,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M4_22 + bgt L(cgemm_kernel_L1_M4_22) -.Lcgemm_kernel_L1_M4_40: +L(cgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_M4_100 + ble L(cgemm_kernel_L1_M4_100) -.Lcgemm_kernel_L1_M4_42: +L(cgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M4_42 + bgt L(cgemm_kernel_L1_M4_42) -.Lcgemm_kernel_L1_M4_100: +L(cgemm_kernel_L1_M4_100): SAVE4x1 -.Lcgemm_kernel_L1_M4_END: +L(cgemm_kernel_L1_M4_END): subs counterI, counterI, #1 - bgt .Lcgemm_kernel_L1_M4_20 + bgt L(cgemm_kernel_L1_M4_20) -.Lcgemm_kernel_L1_M2_BEGIN: +L(cgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lcgemm_kernel_L1_END + ble L(cgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lcgemm_kernel_L1_M1_BEGIN + ble L(cgemm_kernel_L1_M1_BEGIN) -.Lcgemm_kernel_L1_M2_20: +L(cgemm_kernel_L1_M2_20): INIT2x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_M2_40 + ble L(cgemm_kernel_L1_M2_40) -.Lcgemm_kernel_L1_M2_22: +L(cgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1592,43 +1592,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M2_22 + bgt L(cgemm_kernel_L1_M2_22) -.Lcgemm_kernel_L1_M2_40: +L(cgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_M2_100 + ble L(cgemm_kernel_L1_M2_100) -.Lcgemm_kernel_L1_M2_42: +L(cgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M2_42 + bgt L(cgemm_kernel_L1_M2_42) -.Lcgemm_kernel_L1_M2_100: +L(cgemm_kernel_L1_M2_100): SAVE2x1 -.Lcgemm_kernel_L1_M2_END: +L(cgemm_kernel_L1_M2_END): -.Lcgemm_kernel_L1_M1_BEGIN: +L(cgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lcgemm_kernel_L1_END + ble L(cgemm_kernel_L1_END) -.Lcgemm_kernel_L1_M1_20: +L(cgemm_kernel_L1_M1_20): INIT1x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_M1_40 + ble L(cgemm_kernel_L1_M1_40) -.Lcgemm_kernel_L1_M1_22: +L(cgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1640,30 +1640,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M1_22 + bgt L(cgemm_kernel_L1_M1_22) -.Lcgemm_kernel_L1_M1_40: +L(cgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_M1_100 + ble L(cgemm_kernel_L1_M1_100) -.Lcgemm_kernel_L1_M1_42: +L(cgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M1_42 + bgt L(cgemm_kernel_L1_M1_42) -.Lcgemm_kernel_L1_M1_100: +L(cgemm_kernel_L1_M1_100): SAVE1x1 -.Lcgemm_kernel_L1_END: +L(cgemm_kernel_L1_END): -.Lcgemm_kernel_L999: +L(cgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/cgemm_kernel_8x4.S b/kernel/arm64/cgemm_kernel_8x4.S index f100adc7af..0107d83f5e 100644 --- a/kernel/arm64/cgemm_kernel_8x4.S +++ b/kernel/arm64/cgemm_kernel_8x4.S @@ -1407,11 +1407,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lcgemm_kernel_L2_BEGIN + ble L(cgemm_kernel_L2_BEGIN) /******************************************************************************/ -.Lcgemm_kernel_L4_BEGIN: +L(cgemm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -1421,21 +1421,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lcgemm_kernel_L4_M8_BEGIN: +L(cgemm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lcgemm_kernel_L4_M4_BEGIN + ble L(cgemm_kernel_L4_M4_BEGIN) .align 5 -.Lcgemm_kernel_L4_M8_20: +L(cgemm_kernel_L4_M8_20): mov pB, origPB asr counterL , origK, #3 cmp counterL , #2 - blt .Lcgemm_kernel_L4_M8_32 + blt L(cgemm_kernel_L4_M8_32) KERNEL8x4_I KERNEL8x4_M2 @@ -1447,10 +1447,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lcgemm_kernel_L4_M8_22a + ble L(cgemm_kernel_L4_M8_22a) .align 5 -.Lcgemm_kernel_L4_M8_22: +L(cgemm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 @@ -1462,10 +1462,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M8_22 + bgt L(cgemm_kernel_L4_M8_22) .align 5 -.Lcgemm_kernel_L4_M8_22a: +L(cgemm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_M2 @@ -1476,13 +1476,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Lcgemm_kernel_L4_M8_44 + b L(cgemm_kernel_L4_M8_44) .align 5 -.Lcgemm_kernel_L4_M8_32: +L(cgemm_kernel_L4_M8_32): tst counterL, #1 - ble .Lcgemm_kernel_L4_M8_40 + ble L(cgemm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_M2 @@ -1493,116 +1493,116 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Lcgemm_kernel_L4_M8_44 + b L(cgemm_kernel_L4_M8_44) -.Lcgemm_kernel_L4_M8_40: +L(cgemm_kernel_L4_M8_40): INIT8x4 -.Lcgemm_kernel_L4_M8_44: +L(cgemm_kernel_L4_M8_44): ands counterL , origK, #7 - ble .Lcgemm_kernel_L4_M8_100 + ble L(cgemm_kernel_L4_M8_100) .align 5 -.Lcgemm_kernel_L4_M8_46: +L(cgemm_kernel_L4_M8_46): KERNEL8x4_SUB subs counterL, counterL, #1 - bne .Lcgemm_kernel_L4_M8_46 + bne L(cgemm_kernel_L4_M8_46) -.Lcgemm_kernel_L4_M8_100: +L(cgemm_kernel_L4_M8_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVE8x4 -.Lcgemm_kernel_L4_M8_END: +L(cgemm_kernel_L4_M8_END): subs counterI, counterI, #1 - bne .Lcgemm_kernel_L4_M8_20 + bne L(cgemm_kernel_L4_M8_20) -.Lcgemm_kernel_L4_M4_BEGIN: +L(cgemm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lcgemm_kernel_L4_END + ble L(cgemm_kernel_L4_END) tst counterI, #4 - ble .Lcgemm_kernel_L4_M2_BEGIN + ble L(cgemm_kernel_L4_M2_BEGIN) -.Lcgemm_kernel_L4_M4_20: +L(cgemm_kernel_L4_M4_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lcgemm_kernel_L4_M4_32 + blt L(cgemm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lcgemm_kernel_L4_M4_22a + ble L(cgemm_kernel_L4_M4_22a) .align 5 -.Lcgemm_kernel_L4_M4_22: +L(cgemm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M4_22 + bgt L(cgemm_kernel_L4_M4_22) -.Lcgemm_kernel_L4_M4_22a: +L(cgemm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lcgemm_kernel_L4_M4_44 -.Lcgemm_kernel_L4_M4_32: + b L(cgemm_kernel_L4_M4_44) +L(cgemm_kernel_L4_M4_32): tst counterL, #1 - ble .Lcgemm_kernel_L4_M4_40 + ble L(cgemm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lcgemm_kernel_L4_M4_44 -.Lcgemm_kernel_L4_M4_40: + b L(cgemm_kernel_L4_M4_44) +L(cgemm_kernel_L4_M4_40): INIT4x4 -.Lcgemm_kernel_L4_M4_44: +L(cgemm_kernel_L4_M4_44): ands counterL , origK, #1 - ble .Lcgemm_kernel_L4_M4_100 + ble L(cgemm_kernel_L4_M4_100) -.Lcgemm_kernel_L4_M4_46: +L(cgemm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lcgemm_kernel_L4_M4_100: +L(cgemm_kernel_L4_M4_100): SAVE4x4 -.Lcgemm_kernel_L4_M4_END: +L(cgemm_kernel_L4_M4_END): -.Lcgemm_kernel_L4_M2_BEGIN: +L(cgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lcgemm_kernel_L4_END + ble L(cgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lcgemm_kernel_L4_M1_BEGIN + ble L(cgemm_kernel_L4_M1_BEGIN) -.Lcgemm_kernel_L4_M2_20: +L(cgemm_kernel_L4_M2_20): INIT2x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L4_M2_40 + ble L(cgemm_kernel_L4_M2_40) -.Lcgemm_kernel_L4_M2_22: +L(cgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1615,43 +1615,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M2_22 + bgt L(cgemm_kernel_L4_M2_22) -.Lcgemm_kernel_L4_M2_40: +L(cgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L4_M2_100 + ble L(cgemm_kernel_L4_M2_100) -.Lcgemm_kernel_L4_M2_42: +L(cgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M2_42 + bgt L(cgemm_kernel_L4_M2_42) -.Lcgemm_kernel_L4_M2_100: +L(cgemm_kernel_L4_M2_100): SAVE2x4 -.Lcgemm_kernel_L4_M2_END: +L(cgemm_kernel_L4_M2_END): -.Lcgemm_kernel_L4_M1_BEGIN: +L(cgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lcgemm_kernel_L4_END + ble L(cgemm_kernel_L4_END) -.Lcgemm_kernel_L4_M1_20: +L(cgemm_kernel_L4_M1_20): INIT1x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L4_M1_40 + ble L(cgemm_kernel_L4_M1_40) -.Lcgemm_kernel_L4_M1_22: +L(cgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1663,45 +1663,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M1_22 + bgt L(cgemm_kernel_L4_M1_22) -.Lcgemm_kernel_L4_M1_40: +L(cgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L4_M1_100 + ble L(cgemm_kernel_L4_M1_100) -.Lcgemm_kernel_L4_M1_42: +L(cgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M1_42 + bgt L(cgemm_kernel_L4_M1_42) -.Lcgemm_kernel_L4_M1_100: +L(cgemm_kernel_L4_M1_100): SAVE1x4 -.Lcgemm_kernel_L4_END: +L(cgemm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 subs counterJ, counterJ , #1 // j-- - bgt .Lcgemm_kernel_L4_BEGIN + bgt L(cgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Lcgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(cgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lcgemm_kernel_L999 // error, N was less than 4? + ble L(cgemm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Lcgemm_kernel_L1_BEGIN + ble L(cgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1710,14 +1710,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lcgemm_kernel_L2_M8_BEGIN: +L(cgemm_kernel_L2_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lcgemm_kernel_L2_M4_BEGIN + ble L(cgemm_kernel_L2_M4_BEGIN) -.Lcgemm_kernel_L2_M8_20: +L(cgemm_kernel_L2_M8_20): INIT8x2 @@ -1725,10 +1725,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lcgemm_kernel_L2_M8_40 + ble L(cgemm_kernel_L2_M8_40) .align 5 -.Lcgemm_kernel_L2_M8_22: +L(cgemm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB KERNEL8x2_SUB @@ -1740,50 +1740,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M8_22 + bgt L(cgemm_kernel_L2_M8_22) -.Lcgemm_kernel_L2_M8_40: +L(cgemm_kernel_L2_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_M8_100 + ble L(cgemm_kernel_L2_M8_100) -.Lcgemm_kernel_L2_M8_42: +L(cgemm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M8_42 + bgt L(cgemm_kernel_L2_M8_42) -.Lcgemm_kernel_L2_M8_100: +L(cgemm_kernel_L2_M8_100): SAVE8x2 -.Lcgemm_kernel_L2_M8_END: +L(cgemm_kernel_L2_M8_END): subs counterI, counterI, #1 - bgt .Lcgemm_kernel_L2_M8_20 + bgt L(cgemm_kernel_L2_M8_20) -.Lcgemm_kernel_L2_M4_BEGIN: +L(cgemm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lcgemm_kernel_L2_END + ble L(cgemm_kernel_L2_END) tst counterI, #4 // counterI = counterI / 2 - ble .Lcgemm_kernel_L2_M2_BEGIN + ble L(cgemm_kernel_L2_M2_BEGIN) -.Lcgemm_kernel_L2_M4_20: +L(cgemm_kernel_L2_M4_20): INIT4x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lcgemm_kernel_L2_M4_40 + ble L(cgemm_kernel_L2_M4_40) .align 5 -.Lcgemm_kernel_L2_M4_22: +L(cgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1795,46 +1795,46 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M4_22 + bgt L(cgemm_kernel_L2_M4_22) -.Lcgemm_kernel_L2_M4_40: +L(cgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_M4_100 + ble L(cgemm_kernel_L2_M4_100) -.Lcgemm_kernel_L2_M4_42: +L(cgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M4_42 + bgt L(cgemm_kernel_L2_M4_42) -.Lcgemm_kernel_L2_M4_100: +L(cgemm_kernel_L2_M4_100): SAVE4x2 -.Lcgemm_kernel_L2_M4_END: +L(cgemm_kernel_L2_M4_END): -.Lcgemm_kernel_L2_M2_BEGIN: +L(cgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lcgemm_kernel_L2_END + ble L(cgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lcgemm_kernel_L2_M1_BEGIN + ble L(cgemm_kernel_L2_M1_BEGIN) -.Lcgemm_kernel_L2_M2_20: +L(cgemm_kernel_L2_M2_20): INIT2x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lcgemm_kernel_L2_M2_40 + ble L(cgemm_kernel_L2_M2_40) -.Lcgemm_kernel_L2_M2_22: +L(cgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1847,43 +1847,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M2_22 + bgt L(cgemm_kernel_L2_M2_22) -.Lcgemm_kernel_L2_M2_40: +L(cgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_M2_100 + ble L(cgemm_kernel_L2_M2_100) -.Lcgemm_kernel_L2_M2_42: +L(cgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M2_42 + bgt L(cgemm_kernel_L2_M2_42) -.Lcgemm_kernel_L2_M2_100: +L(cgemm_kernel_L2_M2_100): SAVE2x2 -.Lcgemm_kernel_L2_M2_END: +L(cgemm_kernel_L2_M2_END): -.Lcgemm_kernel_L2_M1_BEGIN: +L(cgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lcgemm_kernel_L2_END + ble L(cgemm_kernel_L2_END) -.Lcgemm_kernel_L2_M1_20: +L(cgemm_kernel_L2_M1_20): INIT1x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lcgemm_kernel_L2_M1_40 + ble L(cgemm_kernel_L2_M1_40) -.Lcgemm_kernel_L2_M1_22: +L(cgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1895,36 +1895,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M1_22 + bgt L(cgemm_kernel_L2_M1_22) -.Lcgemm_kernel_L2_M1_40: +L(cgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_M1_100 + ble L(cgemm_kernel_L2_M1_100) -.Lcgemm_kernel_L2_M1_42: +L(cgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M1_42 + bgt L(cgemm_kernel_L2_M1_42) -.Lcgemm_kernel_L2_M1_100: +L(cgemm_kernel_L2_M1_100): SAVE1x2 -.Lcgemm_kernel_L2_END: +L(cgemm_kernel_L2_END): add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 /******************************************************************************/ -.Lcgemm_kernel_L1_BEGIN: +L(cgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lcgemm_kernel_L999 // done + ble L(cgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1933,24 +1933,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lcgemm_kernel_L1_M8_BEGIN: +L(cgemm_kernel_L1_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lcgemm_kernel_L1_M4_BEGIN + ble L(cgemm_kernel_L1_M4_BEGIN) -.Lcgemm_kernel_L1_M8_20: +L(cgemm_kernel_L1_M8_20): INIT8x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_M8_40 + ble L(cgemm_kernel_L1_M8_40) .align 5 -.Lcgemm_kernel_L1_M8_22: +L(cgemm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -1962,51 +1962,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M8_22 + bgt L(cgemm_kernel_L1_M8_22) -.Lcgemm_kernel_L1_M8_40: +L(cgemm_kernel_L1_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_M8_100 + ble L(cgemm_kernel_L1_M8_100) -.Lcgemm_kernel_L1_M8_42: +L(cgemm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M8_42 + bgt L(cgemm_kernel_L1_M8_42) -.Lcgemm_kernel_L1_M8_100: +L(cgemm_kernel_L1_M8_100): SAVE8x1 -.Lcgemm_kernel_L1_M8_END: +L(cgemm_kernel_L1_M8_END): subs counterI, counterI, #1 - bgt .Lcgemm_kernel_L1_M8_20 + bgt L(cgemm_kernel_L1_M8_20) -.Lcgemm_kernel_L1_M4_BEGIN: +L(cgemm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lcgemm_kernel_L1_END + ble L(cgemm_kernel_L1_END) tst counterI, #4 // counterI = counterI / 2 - ble .Lcgemm_kernel_L1_M2_BEGIN + ble L(cgemm_kernel_L1_M2_BEGIN) -.Lcgemm_kernel_L1_M4_20: +L(cgemm_kernel_L1_M4_20): INIT4x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_M4_40 + ble L(cgemm_kernel_L1_M4_40) .align 5 -.Lcgemm_kernel_L1_M4_22: +L(cgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -2018,47 +2018,47 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M4_22 + bgt L(cgemm_kernel_L1_M4_22) -.Lcgemm_kernel_L1_M4_40: +L(cgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_M4_100 + ble L(cgemm_kernel_L1_M4_100) -.Lcgemm_kernel_L1_M4_42: +L(cgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M4_42 + bgt L(cgemm_kernel_L1_M4_42) -.Lcgemm_kernel_L1_M4_100: +L(cgemm_kernel_L1_M4_100): SAVE4x1 -.Lcgemm_kernel_L1_M4_END: +L(cgemm_kernel_L1_M4_END): -.Lcgemm_kernel_L1_M2_BEGIN: +L(cgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lcgemm_kernel_L1_END + ble L(cgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lcgemm_kernel_L1_M1_BEGIN + ble L(cgemm_kernel_L1_M1_BEGIN) -.Lcgemm_kernel_L1_M2_20: +L(cgemm_kernel_L1_M2_20): INIT2x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_M2_40 + ble L(cgemm_kernel_L1_M2_40) -.Lcgemm_kernel_L1_M2_22: +L(cgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -2071,43 +2071,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M2_22 + bgt L(cgemm_kernel_L1_M2_22) -.Lcgemm_kernel_L1_M2_40: +L(cgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_M2_100 + ble L(cgemm_kernel_L1_M2_100) -.Lcgemm_kernel_L1_M2_42: +L(cgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M2_42 + bgt L(cgemm_kernel_L1_M2_42) -.Lcgemm_kernel_L1_M2_100: +L(cgemm_kernel_L1_M2_100): SAVE2x1 -.Lcgemm_kernel_L1_M2_END: +L(cgemm_kernel_L1_M2_END): -.Lcgemm_kernel_L1_M1_BEGIN: +L(cgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lcgemm_kernel_L1_END + ble L(cgemm_kernel_L1_END) -.Lcgemm_kernel_L1_M1_20: +L(cgemm_kernel_L1_M1_20): INIT1x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_M1_40 + ble L(cgemm_kernel_L1_M1_40) -.Lcgemm_kernel_L1_M1_22: +L(cgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -2119,30 +2119,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M1_22 + bgt L(cgemm_kernel_L1_M1_22) -.Lcgemm_kernel_L1_M1_40: +L(cgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_M1_100 + ble L(cgemm_kernel_L1_M1_100) -.Lcgemm_kernel_L1_M1_42: +L(cgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M1_42 + bgt L(cgemm_kernel_L1_M1_42) -.Lcgemm_kernel_L1_M1_100: +L(cgemm_kernel_L1_M1_100): SAVE1x1 -.Lcgemm_kernel_L1_END: +L(cgemm_kernel_L1_END): -.Lcgemm_kernel_L999: +L(cgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/cgemm_kernel_8x4_thunderx2t99.S b/kernel/arm64/cgemm_kernel_8x4_thunderx2t99.S index 2c63925be2..a6fb58e2fc 100644 --- a/kernel/arm64/cgemm_kernel_8x4_thunderx2t99.S +++ b/kernel/arm64/cgemm_kernel_8x4_thunderx2t99.S @@ -1432,11 +1432,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lcgemm_kernel_L2_BEGIN + ble L(cgemm_kernel_L2_BEGIN) /******************************************************************************/ -.Lcgemm_kernel_L4_BEGIN: +L(cgemm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -1446,21 +1446,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lcgemm_kernel_L4_M8_BEGIN: +L(cgemm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lcgemm_kernel_L4_M4_BEGIN + ble L(cgemm_kernel_L4_M4_BEGIN) .align 5 -.Lcgemm_kernel_L4_M8_20: +L(cgemm_kernel_L4_M8_20): mov pB, origPB asr counterL , origK, #5 // origK / 32 cmp counterL , #2 - blt .Lcgemm_kernel_L4_M8_32 + blt L(cgemm_kernel_L4_M8_32) KERNEL8x4_I KERNEL8x4_M2 @@ -1470,18 +1470,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1_M2_x8 subs counterL, counterL, #2 // subtract 2 - ble .Lcgemm_kernel_L4_M8_22a + ble L(cgemm_kernel_L4_M8_22a) .align 5 -.Lcgemm_kernel_L4_M8_22: +L(cgemm_kernel_L4_M8_22): KERNEL8x4_M1_M2_x16 subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M8_22 + bgt L(cgemm_kernel_L4_M8_22) .align 5 -.Lcgemm_kernel_L4_M8_22a: +L(cgemm_kernel_L4_M8_22a): KERNEL8x4_M1_M2_x8 KERNEL8x4_M1_M2_x4 @@ -1490,13 +1490,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Lcgemm_kernel_L4_M8_44 + b L(cgemm_kernel_L4_M8_44) .align 5 -.Lcgemm_kernel_L4_M8_32: +L(cgemm_kernel_L4_M8_32): tst counterL, #1 - ble .Lcgemm_kernel_L4_M8_40 + ble L(cgemm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_M2 @@ -1506,116 +1506,116 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Lcgemm_kernel_L4_M8_44 + b L(cgemm_kernel_L4_M8_44) -.Lcgemm_kernel_L4_M8_40: +L(cgemm_kernel_L4_M8_40): INIT8x4 -.Lcgemm_kernel_L4_M8_44: +L(cgemm_kernel_L4_M8_44): ands counterL , origK, #31 - ble .Lcgemm_kernel_L4_M8_100 + ble L(cgemm_kernel_L4_M8_100) .align 5 -.Lcgemm_kernel_L4_M8_46: +L(cgemm_kernel_L4_M8_46): KERNEL8x4_SUB subs counterL, counterL, #1 - bne .Lcgemm_kernel_L4_M8_46 + bne L(cgemm_kernel_L4_M8_46) -.Lcgemm_kernel_L4_M8_100: +L(cgemm_kernel_L4_M8_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVE8x4 -.Lcgemm_kernel_L4_M8_END: +L(cgemm_kernel_L4_M8_END): subs counterI, counterI, #1 - bne .Lcgemm_kernel_L4_M8_20 + bne L(cgemm_kernel_L4_M8_20) -.Lcgemm_kernel_L4_M4_BEGIN: +L(cgemm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lcgemm_kernel_L4_END + ble L(cgemm_kernel_L4_END) tst counterI, #4 - ble .Lcgemm_kernel_L4_M2_BEGIN + ble L(cgemm_kernel_L4_M2_BEGIN) -.Lcgemm_kernel_L4_M4_20: +L(cgemm_kernel_L4_M4_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lcgemm_kernel_L4_M4_32 + blt L(cgemm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lcgemm_kernel_L4_M4_22a + ble L(cgemm_kernel_L4_M4_22a) .align 5 -.Lcgemm_kernel_L4_M4_22: +L(cgemm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M4_22 + bgt L(cgemm_kernel_L4_M4_22) -.Lcgemm_kernel_L4_M4_22a: +L(cgemm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lcgemm_kernel_L4_M4_44 -.Lcgemm_kernel_L4_M4_32: + b L(cgemm_kernel_L4_M4_44) +L(cgemm_kernel_L4_M4_32): tst counterL, #1 - ble .Lcgemm_kernel_L4_M4_40 + ble L(cgemm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lcgemm_kernel_L4_M4_44 -.Lcgemm_kernel_L4_M4_40: + b L(cgemm_kernel_L4_M4_44) +L(cgemm_kernel_L4_M4_40): INIT4x4 -.Lcgemm_kernel_L4_M4_44: +L(cgemm_kernel_L4_M4_44): ands counterL , origK, #1 - ble .Lcgemm_kernel_L4_M4_100 + ble L(cgemm_kernel_L4_M4_100) -.Lcgemm_kernel_L4_M4_46: +L(cgemm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lcgemm_kernel_L4_M4_100: +L(cgemm_kernel_L4_M4_100): SAVE4x4 -.Lcgemm_kernel_L4_M4_END: +L(cgemm_kernel_L4_M4_END): -.Lcgemm_kernel_L4_M2_BEGIN: +L(cgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lcgemm_kernel_L4_END + ble L(cgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lcgemm_kernel_L4_M1_BEGIN + ble L(cgemm_kernel_L4_M1_BEGIN) -.Lcgemm_kernel_L4_M2_20: +L(cgemm_kernel_L4_M2_20): INIT2x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L4_M2_40 + ble L(cgemm_kernel_L4_M2_40) -.Lcgemm_kernel_L4_M2_22: +L(cgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1628,43 +1628,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M2_22 + bgt L(cgemm_kernel_L4_M2_22) -.Lcgemm_kernel_L4_M2_40: +L(cgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L4_M2_100 + ble L(cgemm_kernel_L4_M2_100) -.Lcgemm_kernel_L4_M2_42: +L(cgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M2_42 + bgt L(cgemm_kernel_L4_M2_42) -.Lcgemm_kernel_L4_M2_100: +L(cgemm_kernel_L4_M2_100): SAVE2x4 -.Lcgemm_kernel_L4_M2_END: +L(cgemm_kernel_L4_M2_END): -.Lcgemm_kernel_L4_M1_BEGIN: +L(cgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lcgemm_kernel_L4_END + ble L(cgemm_kernel_L4_END) -.Lcgemm_kernel_L4_M1_20: +L(cgemm_kernel_L4_M1_20): INIT1x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L4_M1_40 + ble L(cgemm_kernel_L4_M1_40) -.Lcgemm_kernel_L4_M1_22: +L(cgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1676,45 +1676,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M1_22 + bgt L(cgemm_kernel_L4_M1_22) -.Lcgemm_kernel_L4_M1_40: +L(cgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L4_M1_100 + ble L(cgemm_kernel_L4_M1_100) -.Lcgemm_kernel_L4_M1_42: +L(cgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M1_42 + bgt L(cgemm_kernel_L4_M1_42) -.Lcgemm_kernel_L4_M1_100: +L(cgemm_kernel_L4_M1_100): SAVE1x4 -.Lcgemm_kernel_L4_END: +L(cgemm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 subs counterJ, counterJ , #1 // j-- - bgt .Lcgemm_kernel_L4_BEGIN + bgt L(cgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Lcgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(cgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lcgemm_kernel_L999 // error, N was less than 4? + ble L(cgemm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Lcgemm_kernel_L1_BEGIN + ble L(cgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1723,14 +1723,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lcgemm_kernel_L2_M8_BEGIN: +L(cgemm_kernel_L2_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lcgemm_kernel_L2_M4_BEGIN + ble L(cgemm_kernel_L2_M4_BEGIN) -.Lcgemm_kernel_L2_M8_20: +L(cgemm_kernel_L2_M8_20): INIT8x2 @@ -1738,10 +1738,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lcgemm_kernel_L2_M8_40 + ble L(cgemm_kernel_L2_M8_40) .align 5 -.Lcgemm_kernel_L2_M8_22: +L(cgemm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB KERNEL8x2_SUB @@ -1753,50 +1753,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M8_22 + bgt L(cgemm_kernel_L2_M8_22) -.Lcgemm_kernel_L2_M8_40: +L(cgemm_kernel_L2_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_M8_100 + ble L(cgemm_kernel_L2_M8_100) -.Lcgemm_kernel_L2_M8_42: +L(cgemm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M8_42 + bgt L(cgemm_kernel_L2_M8_42) -.Lcgemm_kernel_L2_M8_100: +L(cgemm_kernel_L2_M8_100): SAVE8x2 -.Lcgemm_kernel_L2_M8_END: +L(cgemm_kernel_L2_M8_END): subs counterI, counterI, #1 - bgt .Lcgemm_kernel_L2_M8_20 + bgt L(cgemm_kernel_L2_M8_20) -.Lcgemm_kernel_L2_M4_BEGIN: +L(cgemm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lcgemm_kernel_L2_END + ble L(cgemm_kernel_L2_END) tst counterI, #4 // counterI = counterI / 2 - ble .Lcgemm_kernel_L2_M2_BEGIN + ble L(cgemm_kernel_L2_M2_BEGIN) -.Lcgemm_kernel_L2_M4_20: +L(cgemm_kernel_L2_M4_20): INIT4x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lcgemm_kernel_L2_M4_40 + ble L(cgemm_kernel_L2_M4_40) .align 5 -.Lcgemm_kernel_L2_M4_22: +L(cgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1808,46 +1808,46 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M4_22 + bgt L(cgemm_kernel_L2_M4_22) -.Lcgemm_kernel_L2_M4_40: +L(cgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_M4_100 + ble L(cgemm_kernel_L2_M4_100) -.Lcgemm_kernel_L2_M4_42: +L(cgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M4_42 + bgt L(cgemm_kernel_L2_M4_42) -.Lcgemm_kernel_L2_M4_100: +L(cgemm_kernel_L2_M4_100): SAVE4x2 -.Lcgemm_kernel_L2_M4_END: +L(cgemm_kernel_L2_M4_END): -.Lcgemm_kernel_L2_M2_BEGIN: +L(cgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lcgemm_kernel_L2_END + ble L(cgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lcgemm_kernel_L2_M1_BEGIN + ble L(cgemm_kernel_L2_M1_BEGIN) -.Lcgemm_kernel_L2_M2_20: +L(cgemm_kernel_L2_M2_20): INIT2x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lcgemm_kernel_L2_M2_40 + ble L(cgemm_kernel_L2_M2_40) -.Lcgemm_kernel_L2_M2_22: +L(cgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1860,43 +1860,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M2_22 + bgt L(cgemm_kernel_L2_M2_22) -.Lcgemm_kernel_L2_M2_40: +L(cgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_M2_100 + ble L(cgemm_kernel_L2_M2_100) -.Lcgemm_kernel_L2_M2_42: +L(cgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M2_42 + bgt L(cgemm_kernel_L2_M2_42) -.Lcgemm_kernel_L2_M2_100: +L(cgemm_kernel_L2_M2_100): SAVE2x2 -.Lcgemm_kernel_L2_M2_END: +L(cgemm_kernel_L2_M2_END): -.Lcgemm_kernel_L2_M1_BEGIN: +L(cgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lcgemm_kernel_L2_END + ble L(cgemm_kernel_L2_END) -.Lcgemm_kernel_L2_M1_20: +L(cgemm_kernel_L2_M1_20): INIT1x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lcgemm_kernel_L2_M1_40 + ble L(cgemm_kernel_L2_M1_40) -.Lcgemm_kernel_L2_M1_22: +L(cgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1908,36 +1908,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M1_22 + bgt L(cgemm_kernel_L2_M1_22) -.Lcgemm_kernel_L2_M1_40: +L(cgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_M1_100 + ble L(cgemm_kernel_L2_M1_100) -.Lcgemm_kernel_L2_M1_42: +L(cgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M1_42 + bgt L(cgemm_kernel_L2_M1_42) -.Lcgemm_kernel_L2_M1_100: +L(cgemm_kernel_L2_M1_100): SAVE1x2 -.Lcgemm_kernel_L2_END: +L(cgemm_kernel_L2_END): add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 /******************************************************************************/ -.Lcgemm_kernel_L1_BEGIN: +L(cgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lcgemm_kernel_L999 // done + ble L(cgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1946,24 +1946,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lcgemm_kernel_L1_M8_BEGIN: +L(cgemm_kernel_L1_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lcgemm_kernel_L1_M4_BEGIN + ble L(cgemm_kernel_L1_M4_BEGIN) -.Lcgemm_kernel_L1_M8_20: +L(cgemm_kernel_L1_M8_20): INIT8x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_M8_40 + ble L(cgemm_kernel_L1_M8_40) .align 5 -.Lcgemm_kernel_L1_M8_22: +L(cgemm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -1975,51 +1975,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M8_22 + bgt L(cgemm_kernel_L1_M8_22) -.Lcgemm_kernel_L1_M8_40: +L(cgemm_kernel_L1_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_M8_100 + ble L(cgemm_kernel_L1_M8_100) -.Lcgemm_kernel_L1_M8_42: +L(cgemm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M8_42 + bgt L(cgemm_kernel_L1_M8_42) -.Lcgemm_kernel_L1_M8_100: +L(cgemm_kernel_L1_M8_100): SAVE8x1 -.Lcgemm_kernel_L1_M8_END: +L(cgemm_kernel_L1_M8_END): subs counterI, counterI, #1 - bgt .Lcgemm_kernel_L1_M8_20 + bgt L(cgemm_kernel_L1_M8_20) -.Lcgemm_kernel_L1_M4_BEGIN: +L(cgemm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lcgemm_kernel_L1_END + ble L(cgemm_kernel_L1_END) tst counterI, #4 // counterI = counterI / 2 - ble .Lcgemm_kernel_L1_M2_BEGIN + ble L(cgemm_kernel_L1_M2_BEGIN) -.Lcgemm_kernel_L1_M4_20: +L(cgemm_kernel_L1_M4_20): INIT4x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_M4_40 + ble L(cgemm_kernel_L1_M4_40) .align 5 -.Lcgemm_kernel_L1_M4_22: +L(cgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -2031,47 +2031,47 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M4_22 + bgt L(cgemm_kernel_L1_M4_22) -.Lcgemm_kernel_L1_M4_40: +L(cgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_M4_100 + ble L(cgemm_kernel_L1_M4_100) -.Lcgemm_kernel_L1_M4_42: +L(cgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M4_42 + bgt L(cgemm_kernel_L1_M4_42) -.Lcgemm_kernel_L1_M4_100: +L(cgemm_kernel_L1_M4_100): SAVE4x1 -.Lcgemm_kernel_L1_M4_END: +L(cgemm_kernel_L1_M4_END): -.Lcgemm_kernel_L1_M2_BEGIN: +L(cgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lcgemm_kernel_L1_END + ble L(cgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lcgemm_kernel_L1_M1_BEGIN + ble L(cgemm_kernel_L1_M1_BEGIN) -.Lcgemm_kernel_L1_M2_20: +L(cgemm_kernel_L1_M2_20): INIT2x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_M2_40 + ble L(cgemm_kernel_L1_M2_40) -.Lcgemm_kernel_L1_M2_22: +L(cgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -2084,43 +2084,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M2_22 + bgt L(cgemm_kernel_L1_M2_22) -.Lcgemm_kernel_L1_M2_40: +L(cgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_M2_100 + ble L(cgemm_kernel_L1_M2_100) -.Lcgemm_kernel_L1_M2_42: +L(cgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M2_42 + bgt L(cgemm_kernel_L1_M2_42) -.Lcgemm_kernel_L1_M2_100: +L(cgemm_kernel_L1_M2_100): SAVE2x1 -.Lcgemm_kernel_L1_M2_END: +L(cgemm_kernel_L1_M2_END): -.Lcgemm_kernel_L1_M1_BEGIN: +L(cgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lcgemm_kernel_L1_END + ble L(cgemm_kernel_L1_END) -.Lcgemm_kernel_L1_M1_20: +L(cgemm_kernel_L1_M1_20): INIT1x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_M1_40 + ble L(cgemm_kernel_L1_M1_40) -.Lcgemm_kernel_L1_M1_22: +L(cgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -2132,30 +2132,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M1_22 + bgt L(cgemm_kernel_L1_M1_22) -.Lcgemm_kernel_L1_M1_40: +L(cgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_M1_100 + ble L(cgemm_kernel_L1_M1_100) -.Lcgemm_kernel_L1_M1_42: +L(cgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M1_42 + bgt L(cgemm_kernel_L1_M1_42) -.Lcgemm_kernel_L1_M1_100: +L(cgemm_kernel_L1_M1_100): SAVE1x1 -.Lcgemm_kernel_L1_END: +L(cgemm_kernel_L1_END): -.Lcgemm_kernel_L999: +L(cgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/cgemm_kernel_sve_v1x4.S b/kernel/arm64/cgemm_kernel_sve_v1x4.S index 2136ebbeee..f8fbf398f6 100644 --- a/kernel/arm64/cgemm_kernel_sve_v1x4.S +++ b/kernel/arm64/cgemm_kernel_sve_v1x4.S @@ -535,10 +535,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lcgemm_kernel_L2_BEGIN + ble L(cgemm_kernel_L2_BEGIN) /******************************************************************************/ -.Lcgemm_kernel_L4_BEGIN: +L(cgemm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -548,7 +548,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lcgemm_kernel_L4_Mv1_BEGIN: +L(cgemm_kernel_L4_Mv1_BEGIN): /* Loop over M is done in an SVE fashion. This has the benefit of the last M%SVE_LEN iterations being done in a single sweep */ mov counterI, #0 @@ -556,14 +556,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension .align 5 -.Lcgemm_kernel_L4_Mv1_20: +L(cgemm_kernel_L4_Mv1_20): mov pB, origPB INITv1x4 // fill with zeros asr counterL , origK, #3 cmp counterL , #2 - blt .Lcgemm_kernel_L4_Mv1_32 + blt L(cgemm_kernel_L4_Mv1_32) KERNELv1x4_I KERNELv1x4_M2 @@ -575,10 +575,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lcgemm_kernel_L4_Mv1_22a + ble L(cgemm_kernel_L4_Mv1_22a) .align 5 -.Lcgemm_kernel_L4_Mv1_22: +L(cgemm_kernel_L4_Mv1_22): KERNELv1x4_M1 KERNELv1x4_M2 @@ -590,10 +590,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M2 subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_Mv1_22 + bgt L(cgemm_kernel_L4_Mv1_22) .align 5 -.Lcgemm_kernel_L4_Mv1_22a: +L(cgemm_kernel_L4_Mv1_22a): KERNELv1x4_M1 KERNELv1x4_M2 @@ -604,13 +604,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M1 KERNELv1x4_E - b .Lcgemm_kernel_L4_Mv1_44 + b L(cgemm_kernel_L4_Mv1_44) .align 5 -.Lcgemm_kernel_L4_Mv1_32: +L(cgemm_kernel_L4_Mv1_32): tst counterL, #1 - ble .Lcgemm_kernel_L4_Mv1_40 + ble L(cgemm_kernel_L4_Mv1_40) KERNELv1x4_I KERNELv1x4_M2 @@ -621,56 +621,56 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M1 KERNELv1x4_E - b .Lcgemm_kernel_L4_Mv1_44 + b L(cgemm_kernel_L4_Mv1_44) -.Lcgemm_kernel_L4_Mv1_40: +L(cgemm_kernel_L4_Mv1_40): INITv1x4 -.Lcgemm_kernel_L4_Mv1_44: +L(cgemm_kernel_L4_Mv1_44): ands counterL , origK, #7 - ble .Lcgemm_kernel_L4_Mv1_100 + ble L(cgemm_kernel_L4_Mv1_100) .align 5 -.Lcgemm_kernel_L4_Mv1_46: +L(cgemm_kernel_L4_Mv1_46): KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Lcgemm_kernel_L4_Mv1_46 + bne L(cgemm_kernel_L4_Mv1_46) -.Lcgemm_kernel_L4_Mv1_100: +L(cgemm_kernel_L4_Mv1_100): SAVEv1x4 -.Lcgemm_kernel_L4_Mv1_END: +L(cgemm_kernel_L4_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension - b.any .Lcgemm_kernel_L4_Mv1_20 + b.any L(cgemm_kernel_L4_Mv1_20) -.Lcgemm_kernel_L4_END: +L(cgemm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 4 * 2 subs counterJ, counterJ , #1 // j-- - bgt .Lcgemm_kernel_L4_BEGIN + bgt L(cgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Lcgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(cgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lcgemm_kernel_L999 + ble L(cgemm_kernel_L999) tst counterJ , #2 - ble .Lcgemm_kernel_L1_BEGIN + ble L(cgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC add pCRow1, pCRow0, LDC @@ -681,24 +681,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lcgemm_kernel_L2_Mv1_BEGIN: +L(cgemm_kernel_L2_Mv1_BEGIN): mov counterI, #0 whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s -.Lcgemm_kernel_L2_Mv1_20: +L(cgemm_kernel_L2_Mv1_20): INITv1x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lcgemm_kernel_L2_Mv1_40 + ble L(cgemm_kernel_L2_Mv1_40) .align 5 -.Lcgemm_kernel_L2_Mv1_22: +L(cgemm_kernel_L2_Mv1_22): KERNELv1x2_SUB KERNELv1x2_SUB KERNELv1x2_SUB @@ -710,45 +710,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_Mv1_22 + bgt L(cgemm_kernel_L2_Mv1_22) -.Lcgemm_kernel_L2_Mv1_40: +L(cgemm_kernel_L2_Mv1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_Mv1_100 + ble L(cgemm_kernel_L2_Mv1_100) -.Lcgemm_kernel_L2_Mv1_42: +L(cgemm_kernel_L2_Mv1_42): KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_Mv1_42 + bgt L(cgemm_kernel_L2_Mv1_42) -.Lcgemm_kernel_L2_Mv1_100: +L(cgemm_kernel_L2_Mv1_100): SAVEv1x2 -.Lcgemm_kernel_L2_Mv1_END: +L(cgemm_kernel_L2_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lcgemm_kernel_L2_Mv1_20 + b.any L(cgemm_kernel_L2_Mv1_20) -.Lcgemm_kernel_L2_END: +L(cgemm_kernel_L2_END): lsl temp, origK, #4 add origPB, origPB, temp // B = B + K * 2 * 4 * 2 /******************************************************************************/ -.Lcgemm_kernel_L1_BEGIN: +L(cgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lcgemm_kernel_L999 // done + ble L(cgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -756,24 +756,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lcgemm_kernel_L1_Mv1_BEGIN: +L(cgemm_kernel_L1_Mv1_BEGIN): mov counterI, #0 whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s -.Lcgemm_kernel_L1_Mv1_20: +L(cgemm_kernel_L1_Mv1_20): INITv1x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_Mv1_40 + ble L(cgemm_kernel_L1_Mv1_40) .align 5 -.Lcgemm_kernel_L1_Mv1_22: +L(cgemm_kernel_L1_Mv1_22): KERNELv1x1_SUB KERNELv1x1_SUB KERNELv1x1_SUB @@ -785,37 +785,37 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_Mv1_22 + bgt L(cgemm_kernel_L1_Mv1_22) -.Lcgemm_kernel_L1_Mv1_40: +L(cgemm_kernel_L1_Mv1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_Mv1_100 + ble L(cgemm_kernel_L1_Mv1_100) -.Lcgemm_kernel_L1_Mv1_42: +L(cgemm_kernel_L1_Mv1_42): KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_Mv1_42 + bgt L(cgemm_kernel_L1_Mv1_42) -.Lcgemm_kernel_L1_Mv1_100: +L(cgemm_kernel_L1_Mv1_100): SAVEv1x1 -.Lcgemm_kernel_L1_Mv1_END: +L(cgemm_kernel_L1_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lcgemm_kernel_L1_Mv1_20 + b.any L(cgemm_kernel_L1_Mv1_20) -.Lcgemm_kernel_L1_END: +L(cgemm_kernel_L1_END): /******************************************************************************/ -.Lcgemm_kernel_L999: +L(cgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/copy.S b/kernel/arm64/copy.S index b8c6bfcd42..c04a95448d 100644 --- a/kernel/arm64/copy.S +++ b/kernel/arm64/copy.S @@ -159,50 +159,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Lcopy_kernel_L999 + ble L(copy_kernel_L999) cmp INC_X, #1 - bne .Lcopy_kernel_S_BEGIN + bne L(copy_kernel_S_BEGIN) cmp INC_Y, #1 - bne .Lcopy_kernel_S_BEGIN + bne L(copy_kernel_S_BEGIN) -.Lcopy_kernel_F_BEGIN: +L(copy_kernel_F_BEGIN): asr I, N, #2 cmp I, xzr - beq .Lcopy_kernel_F1 + beq L(copy_kernel_F1) -.Lcopy_kernel_F4: +L(copy_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lcopy_kernel_F4 + bne L(copy_kernel_F4) -.Lcopy_kernel_F1: +L(copy_kernel_F1): ands I, N, #3 - ble .Lcopy_kernel_L999 + ble L(copy_kernel_L999) -.Lcopy_kernel_F10: +L(copy_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lcopy_kernel_F10 + bne L(copy_kernel_F10) mov w0, wzr ret -.Lcopy_kernel_S_BEGIN: +L(copy_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lcopy_kernel_S1 + ble L(copy_kernel_S1) -.Lcopy_kernel_S4: +L(copy_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -210,21 +210,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lcopy_kernel_S4 + bne L(copy_kernel_S4) -.Lcopy_kernel_S1: +L(copy_kernel_S1): ands I, N, #3 - ble .Lcopy_kernel_L999 + ble L(copy_kernel_L999) -.Lcopy_kernel_S10: +L(copy_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lcopy_kernel_S10 + bne L(copy_kernel_S10) -.Lcopy_kernel_L999: +L(copy_kernel_L999): mov w0, wzr ret diff --git a/kernel/arm64/csum.S b/kernel/arm64/csum.S index 90746bc392..a5aba37fb3 100644 --- a/kernel/arm64/csum.S +++ b/kernel/arm64/csum.S @@ -92,52 +92,52 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. fmov s1, SUMF cmp N, xzr - ble .Lcsum_kernel_L999 + ble L(csum_kernel_L999) cmp INC_X, xzr - ble .Lcsum_kernel_L999 + ble L(csum_kernel_L999) cmp INC_X, #1 - bne .Lcsum_kernel_S_BEGIN + bne L(csum_kernel_S_BEGIN) -.Lcsum_kernel_F_BEGIN: +L(csum_kernel_F_BEGIN): asr I, N, #3 cmp I, xzr - beq .Lcsum_kernel_F1 + beq L(csum_kernel_F1) -.Lcsum_kernel_F8: +L(csum_kernel_F8): KERNEL_F8 subs I, I, #1 - bne .Lcsum_kernel_F8 + bne L(csum_kernel_F8) KERNEL_F8_FINALIZE -.Lcsum_kernel_F1: +L(csum_kernel_F1): ands I, N, #7 - ble .Lcsum_kernel_L999 + ble L(csum_kernel_L999) -.Lcsum_kernel_F10: +L(csum_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lcsum_kernel_F10 + bne L(csum_kernel_F10) -.Lcsum_kernel_L999: +L(csum_kernel_L999): ret -.Lcsum_kernel_S_BEGIN: +L(csum_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lcsum_kernel_S1 + ble L(csum_kernel_S1) -.Lcsum_kernel_S4: +L(csum_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -145,19 +145,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lcsum_kernel_S4 + bne L(csum_kernel_S4) -.Lcsum_kernel_S1: +L(csum_kernel_S1): ands I, N, #3 - ble .Lcsum_kernel_L999 + ble L(csum_kernel_L999) -.Lcsum_kernel_S10: +L(csum_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lcsum_kernel_S10 + bne L(csum_kernel_S10) ret diff --git a/kernel/arm64/ctrmm_kernel_4x4.S b/kernel/arm64/ctrmm_kernel_4x4.S index 79d33e93c0..edca492dc9 100644 --- a/kernel/arm64/ctrmm_kernel_4x4.S +++ b/kernel/arm64/ctrmm_kernel_4x4.S @@ -785,11 +785,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lctrmm_kernel_L2_BEGIN + ble L(ctrmm_kernel_L2_BEGIN) /******************************************************************************/ -.Lctrmm_kernel_L4_BEGIN: +L(ctrmm_kernel_L4_BEGIN): mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #2 @@ -798,14 +798,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = start of A array -.Lctrmm_kernel_L4_M4_BEGIN: +L(ctrmm_kernel_L4_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lctrmm_kernel_L4_M2_BEGIN + ble L(ctrmm_kernel_L4_M2_BEGIN) -.Lctrmm_kernel_L4_M4_20: +L(ctrmm_kernel_L4_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -826,55 +826,55 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lctrmm_kernel_L4_M4_32 + blt L(ctrmm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lctrmm_kernel_L4_M4_22a + ble L(ctrmm_kernel_L4_M4_22a) .align 5 -.Lctrmm_kernel_L4_M4_22: +L(ctrmm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_M4_22 + bgt L(ctrmm_kernel_L4_M4_22) -.Lctrmm_kernel_L4_M4_22a: +L(ctrmm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lctrmm_kernel_L4_M4_44 + b L(ctrmm_kernel_L4_M4_44) -.Lctrmm_kernel_L4_M4_32: +L(ctrmm_kernel_L4_M4_32): tst counterL, #1 - ble .Lctrmm_kernel_L4_M4_40 + ble L(ctrmm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lctrmm_kernel_L4_M4_44 + b L(ctrmm_kernel_L4_M4_44) -.Lctrmm_kernel_L4_M4_40: +L(ctrmm_kernel_L4_M4_40): INIT4x4 -.Lctrmm_kernel_L4_M4_44: +L(ctrmm_kernel_L4_M4_44): ands counterL , tempK, #1 - ble .Lctrmm_kernel_L4_M4_100 + ble L(ctrmm_kernel_L4_M4_100) -.Lctrmm_kernel_L4_M4_46: +L(ctrmm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lctrmm_kernel_L4_M4_100: +L(ctrmm_kernel_L4_M4_100): SAVE4x4 @@ -893,20 +893,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lctrmm_kernel_L4_M4_END: +L(ctrmm_kernel_L4_M4_END): subs counterI, counterI, #1 - bne .Lctrmm_kernel_L4_M4_20 + bne L(ctrmm_kernel_L4_M4_20) -.Lctrmm_kernel_L4_M2_BEGIN: +L(ctrmm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lctrmm_kernel_L4_END + ble L(ctrmm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lctrmm_kernel_L4_M1_BEGIN + ble L(ctrmm_kernel_L4_M1_BEGIN) -.Lctrmm_kernel_L4_M2_20: +L(ctrmm_kernel_L4_M2_20): INIT2x4 @@ -930,9 +930,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L4_M2_40 + ble L(ctrmm_kernel_L4_M2_40) -.Lctrmm_kernel_L4_M2_22: +L(ctrmm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -945,22 +945,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_M2_22 + bgt L(ctrmm_kernel_L4_M2_22) -.Lctrmm_kernel_L4_M2_40: +L(ctrmm_kernel_L4_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L4_M2_100 + ble L(ctrmm_kernel_L4_M2_100) -.Lctrmm_kernel_L4_M2_42: +L(ctrmm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_M2_42 + bgt L(ctrmm_kernel_L4_M2_42) -.Lctrmm_kernel_L4_M2_100: +L(ctrmm_kernel_L4_M2_100): SAVE2x4 @@ -980,15 +980,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lctrmm_kernel_L4_M2_END: +L(ctrmm_kernel_L4_M2_END): -.Lctrmm_kernel_L4_M1_BEGIN: +L(ctrmm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lctrmm_kernel_L4_END + ble L(ctrmm_kernel_L4_END) -.Lctrmm_kernel_L4_M1_20: +L(ctrmm_kernel_L4_M1_20): INIT1x4 @@ -1012,9 +1012,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L4_M1_40 + ble L(ctrmm_kernel_L4_M1_40) -.Lctrmm_kernel_L4_M1_22: +L(ctrmm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1026,22 +1026,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_M1_22 + bgt L(ctrmm_kernel_L4_M1_22) -.Lctrmm_kernel_L4_M1_40: +L(ctrmm_kernel_L4_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L4_M1_100 + ble L(ctrmm_kernel_L4_M1_100) -.Lctrmm_kernel_L4_M1_42: +L(ctrmm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_M1_42 + bgt L(ctrmm_kernel_L4_M1_42) -.Lctrmm_kernel_L4_M1_100: +L(ctrmm_kernel_L4_M1_100): SAVE1x4 @@ -1061,7 +1061,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Lctrmm_kernel_L4_END: +L(ctrmm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 @@ -1071,19 +1071,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Lctrmm_kernel_L4_BEGIN + bgt L(ctrmm_kernel_L4_BEGIN) /******************************************************************************/ -.Lctrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(ctrmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lctrmm_kernel_L999 // error, N was less than 4? + ble L(ctrmm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Lctrmm_kernel_L1_BEGIN + ble L(ctrmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1095,14 +1095,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lctrmm_kernel_L2_M4_BEGIN: +L(ctrmm_kernel_L2_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI,#0 - ble .Lctrmm_kernel_L2_M2_BEGIN + ble L(ctrmm_kernel_L2_M2_BEGIN) -.Lctrmm_kernel_L2_M4_20: +L(ctrmm_kernel_L2_M4_20): INIT4x2 @@ -1126,10 +1126,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lctrmm_kernel_L2_M4_40 + ble L(ctrmm_kernel_L2_M4_40) .align 5 -.Lctrmm_kernel_L2_M4_22: +L(ctrmm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1141,22 +1141,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M4_22 + bgt L(ctrmm_kernel_L2_M4_22) -.Lctrmm_kernel_L2_M4_40: +L(ctrmm_kernel_L2_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L2_M4_100 + ble L(ctrmm_kernel_L2_M4_100) -.Lctrmm_kernel_L2_M4_42: +L(ctrmm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M4_42 + bgt L(ctrmm_kernel_L2_M4_42) -.Lctrmm_kernel_L2_M4_100: +L(ctrmm_kernel_L2_M4_100): SAVE4x2 @@ -1176,22 +1176,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lctrmm_kernel_L2_M4_END: +L(ctrmm_kernel_L2_M4_END): subs counterI, counterI, #1 - bgt .Lctrmm_kernel_L2_M4_20 + bgt L(ctrmm_kernel_L2_M4_20) -.Lctrmm_kernel_L2_M2_BEGIN: +L(ctrmm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lctrmm_kernel_L2_END + ble L(ctrmm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lctrmm_kernel_L2_M1_BEGIN + ble L(ctrmm_kernel_L2_M1_BEGIN) -.Lctrmm_kernel_L2_M2_20: +L(ctrmm_kernel_L2_M2_20): INIT2x2 @@ -1215,9 +1215,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lctrmm_kernel_L2_M2_40 + ble L(ctrmm_kernel_L2_M2_40) -.Lctrmm_kernel_L2_M2_22: +L(ctrmm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1230,22 +1230,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M2_22 + bgt L(ctrmm_kernel_L2_M2_22) -.Lctrmm_kernel_L2_M2_40: +L(ctrmm_kernel_L2_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L2_M2_100 + ble L(ctrmm_kernel_L2_M2_100) -.Lctrmm_kernel_L2_M2_42: +L(ctrmm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M2_42 + bgt L(ctrmm_kernel_L2_M2_42) -.Lctrmm_kernel_L2_M2_100: +L(ctrmm_kernel_L2_M2_100): SAVE2x2 @@ -1265,15 +1265,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lctrmm_kernel_L2_M2_END: +L(ctrmm_kernel_L2_M2_END): -.Lctrmm_kernel_L2_M1_BEGIN: +L(ctrmm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lctrmm_kernel_L2_END + ble L(ctrmm_kernel_L2_END) -.Lctrmm_kernel_L2_M1_20: +L(ctrmm_kernel_L2_M1_20): INIT1x2 @@ -1297,9 +1297,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lctrmm_kernel_L2_M1_40 + ble L(ctrmm_kernel_L2_M1_40) -.Lctrmm_kernel_L2_M1_22: +L(ctrmm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1311,22 +1311,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M1_22 + bgt L(ctrmm_kernel_L2_M1_22) -.Lctrmm_kernel_L2_M1_40: +L(ctrmm_kernel_L2_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L2_M1_100 + ble L(ctrmm_kernel_L2_M1_100) -.Lctrmm_kernel_L2_M1_42: +L(ctrmm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M1_42 + bgt L(ctrmm_kernel_L2_M1_42) -.Lctrmm_kernel_L2_M1_100: +L(ctrmm_kernel_L2_M1_100): SAVE1x2 @@ -1346,7 +1346,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Lctrmm_kernel_L2_END: +L(ctrmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -1354,11 +1354,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lctrmm_kernel_L1_BEGIN: +L(ctrmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lctrmm_kernel_L999 // done + ble L(ctrmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1370,14 +1370,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lctrmm_kernel_L1_M4_BEGIN: +L(ctrmm_kernel_L1_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lctrmm_kernel_L1_M2_BEGIN + ble L(ctrmm_kernel_L1_M2_BEGIN) -.Lctrmm_kernel_L1_M4_20: +L(ctrmm_kernel_L1_M4_20): INIT4x1 @@ -1401,10 +1401,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L1_M4_40 + ble L(ctrmm_kernel_L1_M4_40) .align 5 -.Lctrmm_kernel_L1_M4_22: +L(ctrmm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1416,22 +1416,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M4_22 + bgt L(ctrmm_kernel_L1_M4_22) -.Lctrmm_kernel_L1_M4_40: +L(ctrmm_kernel_L1_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L1_M4_100 + ble L(ctrmm_kernel_L1_M4_100) -.Lctrmm_kernel_L1_M4_42: +L(ctrmm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M4_42 + bgt L(ctrmm_kernel_L1_M4_42) -.Lctrmm_kernel_L1_M4_100: +L(ctrmm_kernel_L1_M4_100): SAVE4x1 @@ -1451,22 +1451,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lctrmm_kernel_L1_M4_END: +L(ctrmm_kernel_L1_M4_END): subs counterI, counterI, #1 - bgt .Lctrmm_kernel_L1_M4_20 + bgt L(ctrmm_kernel_L1_M4_20) -.Lctrmm_kernel_L1_M2_BEGIN: +L(ctrmm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lctrmm_kernel_L1_END + ble L(ctrmm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lctrmm_kernel_L1_M1_BEGIN + ble L(ctrmm_kernel_L1_M1_BEGIN) -.Lctrmm_kernel_L1_M2_20: +L(ctrmm_kernel_L1_M2_20): INIT2x1 @@ -1490,9 +1490,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L1_M2_40 + ble L(ctrmm_kernel_L1_M2_40) -.Lctrmm_kernel_L1_M2_22: +L(ctrmm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1505,22 +1505,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M2_22 + bgt L(ctrmm_kernel_L1_M2_22) -.Lctrmm_kernel_L1_M2_40: +L(ctrmm_kernel_L1_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L1_M2_100 + ble L(ctrmm_kernel_L1_M2_100) -.Lctrmm_kernel_L1_M2_42: +L(ctrmm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M2_42 + bgt L(ctrmm_kernel_L1_M2_42) -.Lctrmm_kernel_L1_M2_100: +L(ctrmm_kernel_L1_M2_100): SAVE2x1 @@ -1540,15 +1540,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lctrmm_kernel_L1_M2_END: +L(ctrmm_kernel_L1_M2_END): -.Lctrmm_kernel_L1_M1_BEGIN: +L(ctrmm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lctrmm_kernel_L1_END + ble L(ctrmm_kernel_L1_END) -.Lctrmm_kernel_L1_M1_20: +L(ctrmm_kernel_L1_M1_20): INIT1x1 @@ -1572,9 +1572,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L1_M1_40 + ble L(ctrmm_kernel_L1_M1_40) -.Lctrmm_kernel_L1_M1_22: +L(ctrmm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1586,30 +1586,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M1_22 + bgt L(ctrmm_kernel_L1_M1_22) -.Lctrmm_kernel_L1_M1_40: +L(ctrmm_kernel_L1_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L1_M1_100 + ble L(ctrmm_kernel_L1_M1_100) -.Lctrmm_kernel_L1_M1_42: +L(ctrmm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M1_42 + bgt L(ctrmm_kernel_L1_M1_42) -.Lctrmm_kernel_L1_M1_100: +L(ctrmm_kernel_L1_M1_100): SAVE1x1 -.Lctrmm_kernel_L1_END: +L(ctrmm_kernel_L1_END): -.Lctrmm_kernel_L999: +L(ctrmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/ctrmm_kernel_8x4.S b/kernel/arm64/ctrmm_kernel_8x4.S index e8f1d8cf30..417273a6e3 100644 --- a/kernel/arm64/ctrmm_kernel_8x4.S +++ b/kernel/arm64/ctrmm_kernel_8x4.S @@ -1405,11 +1405,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lctrmm_kernel_L2_BEGIN + ble L(ctrmm_kernel_L2_BEGIN) /******************************************************************************/ -.Lctrmm_kernel_L4_BEGIN: +L(ctrmm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -1423,14 +1423,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = start of A array -.Lctrmm_kernel_L4_M8_BEGIN: +L(ctrmm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lctrmm_kernel_L4_M4_BEGIN + ble L(ctrmm_kernel_L4_M4_BEGIN) -.Lctrmm_kernel_L4_M8_20: +L(ctrmm_kernel_L4_M8_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1452,7 +1452,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 cmp counterL , #2 - blt .Lctrmm_kernel_L4_M8_32 + blt L(ctrmm_kernel_L4_M8_32) KERNEL8x4_I KERNEL8x4_M2 @@ -1464,10 +1464,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lctrmm_kernel_L4_M8_22a + ble L(ctrmm_kernel_L4_M8_22a) .align 5 -.Lctrmm_kernel_L4_M8_22: +L(ctrmm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 @@ -1479,10 +1479,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_M8_22 + bgt L(ctrmm_kernel_L4_M8_22) .align 5 -.Lctrmm_kernel_L4_M8_22a: +L(ctrmm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_M2 @@ -1493,13 +1493,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Lctrmm_kernel_L4_M8_44 + b L(ctrmm_kernel_L4_M8_44) .align 5 -.Lctrmm_kernel_L4_M8_32: +L(ctrmm_kernel_L4_M8_32): tst counterL, #1 - ble .Lctrmm_kernel_L4_M8_40 + ble L(ctrmm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_M2 @@ -1510,26 +1510,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Lctrmm_kernel_L4_M8_44 + b L(ctrmm_kernel_L4_M8_44) -.Lctrmm_kernel_L4_M8_40: +L(ctrmm_kernel_L4_M8_40): INIT8x4 -.Lctrmm_kernel_L4_M8_44: +L(ctrmm_kernel_L4_M8_44): ands counterL , tempK, #7 - ble .Lctrmm_kernel_L4_M8_100 + ble L(ctrmm_kernel_L4_M8_100) .align 5 -.Lctrmm_kernel_L4_M8_46: +L(ctrmm_kernel_L4_M8_46): KERNEL8x4_SUB subs counterL, counterL, #1 - bne .Lctrmm_kernel_L4_M8_46 + bne L(ctrmm_kernel_L4_M8_46) -.Lctrmm_kernel_L4_M8_100: +L(ctrmm_kernel_L4_M8_100): SAVE8x4 @@ -1552,21 +1552,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] -.Lctrmm_kernel_L4_M8_END: +L(ctrmm_kernel_L4_M8_END): subs counterI, counterI, #1 - bne .Lctrmm_kernel_L4_M8_20 + bne L(ctrmm_kernel_L4_M8_20) -.Lctrmm_kernel_L4_M4_BEGIN: +L(ctrmm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lctrmm_kernel_L4_END + ble L(ctrmm_kernel_L4_END) tst counterI, #4 - ble .Lctrmm_kernel_L4_M2_BEGIN + ble L(ctrmm_kernel_L4_M2_BEGIN) -.Lctrmm_kernel_L4_M4_20: +L(ctrmm_kernel_L4_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1587,46 +1587,46 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lctrmm_kernel_L4_M4_32 + blt L(ctrmm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lctrmm_kernel_L4_M4_22a + ble L(ctrmm_kernel_L4_M4_22a) .align 5 -.Lctrmm_kernel_L4_M4_22: +L(ctrmm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_M4_22 + bgt L(ctrmm_kernel_L4_M4_22) -.Lctrmm_kernel_L4_M4_22a: +L(ctrmm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lctrmm_kernel_L4_M4_44 -.Lctrmm_kernel_L4_M4_32: + b L(ctrmm_kernel_L4_M4_44) +L(ctrmm_kernel_L4_M4_32): tst counterL, #1 - ble .Lctrmm_kernel_L4_M4_40 + ble L(ctrmm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lctrmm_kernel_L4_M4_44 -.Lctrmm_kernel_L4_M4_40: + b L(ctrmm_kernel_L4_M4_44) +L(ctrmm_kernel_L4_M4_40): INIT4x4 -.Lctrmm_kernel_L4_M4_44: +L(ctrmm_kernel_L4_M4_44): ands counterL , tempK, #1 - ble .Lctrmm_kernel_L4_M4_100 + ble L(ctrmm_kernel_L4_M4_100) -.Lctrmm_kernel_L4_M4_46: +L(ctrmm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lctrmm_kernel_L4_M4_100: +L(ctrmm_kernel_L4_M4_100): SAVE4x4 @@ -1645,18 +1645,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lctrmm_kernel_L4_M4_END: +L(ctrmm_kernel_L4_M4_END): -.Lctrmm_kernel_L4_M2_BEGIN: +L(ctrmm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lctrmm_kernel_L4_END + ble L(ctrmm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lctrmm_kernel_L4_M1_BEGIN + ble L(ctrmm_kernel_L4_M1_BEGIN) -.Lctrmm_kernel_L4_M2_20: +L(ctrmm_kernel_L4_M2_20): INIT2x4 @@ -1679,9 +1679,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L4_M2_40 + ble L(ctrmm_kernel_L4_M2_40) -.Lctrmm_kernel_L4_M2_22: +L(ctrmm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1694,22 +1694,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_M2_22 + bgt L(ctrmm_kernel_L4_M2_22) -.Lctrmm_kernel_L4_M2_40: +L(ctrmm_kernel_L4_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L4_M2_100 + ble L(ctrmm_kernel_L4_M2_100) -.Lctrmm_kernel_L4_M2_42: +L(ctrmm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_M2_42 + bgt L(ctrmm_kernel_L4_M2_42) -.Lctrmm_kernel_L4_M2_100: +L(ctrmm_kernel_L4_M2_100): SAVE2x4 @@ -1729,15 +1729,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lctrmm_kernel_L4_M2_END: +L(ctrmm_kernel_L4_M2_END): -.Lctrmm_kernel_L4_M1_BEGIN: +L(ctrmm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lctrmm_kernel_L4_END + ble L(ctrmm_kernel_L4_END) -.Lctrmm_kernel_L4_M1_20: +L(ctrmm_kernel_L4_M1_20): INIT1x4 @@ -1761,9 +1761,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L4_M1_40 + ble L(ctrmm_kernel_L4_M1_40) -.Lctrmm_kernel_L4_M1_22: +L(ctrmm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1775,22 +1775,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_M1_22 + bgt L(ctrmm_kernel_L4_M1_22) -.Lctrmm_kernel_L4_M1_40: +L(ctrmm_kernel_L4_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L4_M1_100 + ble L(ctrmm_kernel_L4_M1_100) -.Lctrmm_kernel_L4_M1_42: +L(ctrmm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_M1_42 + bgt L(ctrmm_kernel_L4_M1_42) -.Lctrmm_kernel_L4_M1_100: +L(ctrmm_kernel_L4_M1_100): SAVE1x4 @@ -1810,7 +1810,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Lctrmm_kernel_L4_END: +L(ctrmm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 @@ -1820,19 +1820,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Lctrmm_kernel_L4_BEGIN + bgt L(ctrmm_kernel_L4_BEGIN) /******************************************************************************/ -.Lctrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(ctrmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lctrmm_kernel_L999 // error, N was less than 4? + ble L(ctrmm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Lctrmm_kernel_L1_BEGIN + ble L(ctrmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1843,14 +1843,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = A -.Lctrmm_kernel_L2_M8_BEGIN: +L(ctrmm_kernel_L2_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lctrmm_kernel_L2_M4_BEGIN + ble L(ctrmm_kernel_L2_M4_BEGIN) -.Lctrmm_kernel_L2_M8_20: +L(ctrmm_kernel_L2_M8_20): INIT8x2 @@ -1874,10 +1874,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lctrmm_kernel_L2_M8_40 + ble L(ctrmm_kernel_L2_M8_40) .align 5 -.Lctrmm_kernel_L2_M8_22: +L(ctrmm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB KERNEL8x2_SUB @@ -1889,22 +1889,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M8_22 + bgt L(ctrmm_kernel_L2_M8_22) -.Lctrmm_kernel_L2_M8_40: +L(ctrmm_kernel_L2_M8_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L2_M8_100 + ble L(ctrmm_kernel_L2_M8_100) -.Lctrmm_kernel_L2_M8_42: +L(ctrmm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M8_42 + bgt L(ctrmm_kernel_L2_M8_42) -.Lctrmm_kernel_L2_M8_100: +L(ctrmm_kernel_L2_M8_100): SAVE8x2 @@ -1924,21 +1924,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #8 #endif -.Lctrmm_kernel_L2_M8_END: +L(ctrmm_kernel_L2_M8_END): subs counterI, counterI, #1 - bgt .Lctrmm_kernel_L2_M8_20 + bgt L(ctrmm_kernel_L2_M8_20) -.Lctrmm_kernel_L2_M4_BEGIN: +L(ctrmm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lctrmm_kernel_L2_END + ble L(ctrmm_kernel_L2_END) tst counterI, #4 // counterI = counterI / 2 - ble .Lctrmm_kernel_L2_M2_BEGIN + ble L(ctrmm_kernel_L2_M2_BEGIN) -.Lctrmm_kernel_L2_M4_20: +L(ctrmm_kernel_L2_M4_20): INIT4x2 @@ -1962,10 +1962,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lctrmm_kernel_L2_M4_40 + ble L(ctrmm_kernel_L2_M4_40) .align 5 -.Lctrmm_kernel_L2_M4_22: +L(ctrmm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1977,22 +1977,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M4_22 + bgt L(ctrmm_kernel_L2_M4_22) -.Lctrmm_kernel_L2_M4_40: +L(ctrmm_kernel_L2_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L2_M4_100 + ble L(ctrmm_kernel_L2_M4_100) -.Lctrmm_kernel_L2_M4_42: +L(ctrmm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M4_42 + bgt L(ctrmm_kernel_L2_M4_42) -.Lctrmm_kernel_L2_M4_100: +L(ctrmm_kernel_L2_M4_100): SAVE4x2 @@ -2012,19 +2012,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lctrmm_kernel_L2_M4_END: +L(ctrmm_kernel_L2_M4_END): -.Lctrmm_kernel_L2_M2_BEGIN: +L(ctrmm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lctrmm_kernel_L2_END + ble L(ctrmm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lctrmm_kernel_L2_M1_BEGIN + ble L(ctrmm_kernel_L2_M1_BEGIN) -.Lctrmm_kernel_L2_M2_20: +L(ctrmm_kernel_L2_M2_20): INIT2x2 @@ -2048,9 +2048,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lctrmm_kernel_L2_M2_40 + ble L(ctrmm_kernel_L2_M2_40) -.Lctrmm_kernel_L2_M2_22: +L(ctrmm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -2063,22 +2063,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M2_22 + bgt L(ctrmm_kernel_L2_M2_22) -.Lctrmm_kernel_L2_M2_40: +L(ctrmm_kernel_L2_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L2_M2_100 + ble L(ctrmm_kernel_L2_M2_100) -.Lctrmm_kernel_L2_M2_42: +L(ctrmm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M2_42 + bgt L(ctrmm_kernel_L2_M2_42) -.Lctrmm_kernel_L2_M2_100: +L(ctrmm_kernel_L2_M2_100): SAVE2x2 @@ -2098,15 +2098,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lctrmm_kernel_L2_M2_END: +L(ctrmm_kernel_L2_M2_END): -.Lctrmm_kernel_L2_M1_BEGIN: +L(ctrmm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lctrmm_kernel_L2_END + ble L(ctrmm_kernel_L2_END) -.Lctrmm_kernel_L2_M1_20: +L(ctrmm_kernel_L2_M1_20): INIT1x2 @@ -2130,9 +2130,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lctrmm_kernel_L2_M1_40 + ble L(ctrmm_kernel_L2_M1_40) -.Lctrmm_kernel_L2_M1_22: +L(ctrmm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -2144,22 +2144,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M1_22 + bgt L(ctrmm_kernel_L2_M1_22) -.Lctrmm_kernel_L2_M1_40: +L(ctrmm_kernel_L2_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L2_M1_100 + ble L(ctrmm_kernel_L2_M1_100) -.Lctrmm_kernel_L2_M1_42: +L(ctrmm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M1_42 + bgt L(ctrmm_kernel_L2_M1_42) -.Lctrmm_kernel_L2_M1_100: +L(ctrmm_kernel_L2_M1_100): SAVE1x2 @@ -2179,7 +2179,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Lctrmm_kernel_L2_END: +L(ctrmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -2187,11 +2187,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lctrmm_kernel_L1_BEGIN: +L(ctrmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lctrmm_kernel_L999 // done + ble L(ctrmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C add pC , pC , LDC // Update pC to point to next @@ -2201,14 +2201,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = A -.Lctrmm_kernel_L1_M8_BEGIN: +L(ctrmm_kernel_L1_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lctrmm_kernel_L1_M4_BEGIN + ble L(ctrmm_kernel_L1_M4_BEGIN) -.Lctrmm_kernel_L1_M8_20: +L(ctrmm_kernel_L1_M8_20): INIT8x1 @@ -2232,10 +2232,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L1_M8_40 + ble L(ctrmm_kernel_L1_M8_40) .align 5 -.Lctrmm_kernel_L1_M8_22: +L(ctrmm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -2247,22 +2247,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M8_22 + bgt L(ctrmm_kernel_L1_M8_22) -.Lctrmm_kernel_L1_M8_40: +L(ctrmm_kernel_L1_M8_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L1_M8_100 + ble L(ctrmm_kernel_L1_M8_100) -.Lctrmm_kernel_L1_M8_42: +L(ctrmm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M8_42 + bgt L(ctrmm_kernel_L1_M8_42) -.Lctrmm_kernel_L1_M8_100: +L(ctrmm_kernel_L1_M8_100): SAVE8x1 @@ -2282,21 +2282,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #8 #endif -.Lctrmm_kernel_L1_M8_END: +L(ctrmm_kernel_L1_M8_END): subs counterI, counterI, #1 - bgt .Lctrmm_kernel_L1_M8_20 + bgt L(ctrmm_kernel_L1_M8_20) -.Lctrmm_kernel_L1_M4_BEGIN: +L(ctrmm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lctrmm_kernel_L1_END + ble L(ctrmm_kernel_L1_END) tst counterI, #4 // counterI = counterI / 2 - ble .Lctrmm_kernel_L1_M2_BEGIN + ble L(ctrmm_kernel_L1_M2_BEGIN) -.Lctrmm_kernel_L1_M4_20: +L(ctrmm_kernel_L1_M4_20): INIT4x1 @@ -2319,10 +2319,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L1_M4_40 + ble L(ctrmm_kernel_L1_M4_40) .align 5 -.Lctrmm_kernel_L1_M4_22: +L(ctrmm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -2334,22 +2334,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M4_22 + bgt L(ctrmm_kernel_L1_M4_22) -.Lctrmm_kernel_L1_M4_40: +L(ctrmm_kernel_L1_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L1_M4_100 + ble L(ctrmm_kernel_L1_M4_100) -.Lctrmm_kernel_L1_M4_42: +L(ctrmm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M4_42 + bgt L(ctrmm_kernel_L1_M4_42) -.Lctrmm_kernel_L1_M4_100: +L(ctrmm_kernel_L1_M4_100): SAVE4x1 @@ -2369,18 +2369,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lctrmm_kernel_L1_M4_END: +L(ctrmm_kernel_L1_M4_END): -.Lctrmm_kernel_L1_M2_BEGIN: +L(ctrmm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lctrmm_kernel_L1_END + ble L(ctrmm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lctrmm_kernel_L1_M1_BEGIN + ble L(ctrmm_kernel_L1_M1_BEGIN) -.Lctrmm_kernel_L1_M2_20: +L(ctrmm_kernel_L1_M2_20): INIT2x1 @@ -2404,9 +2404,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L1_M2_40 + ble L(ctrmm_kernel_L1_M2_40) -.Lctrmm_kernel_L1_M2_22: +L(ctrmm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -2419,22 +2419,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M2_22 + bgt L(ctrmm_kernel_L1_M2_22) -.Lctrmm_kernel_L1_M2_40: +L(ctrmm_kernel_L1_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L1_M2_100 + ble L(ctrmm_kernel_L1_M2_100) -.Lctrmm_kernel_L1_M2_42: +L(ctrmm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M2_42 + bgt L(ctrmm_kernel_L1_M2_42) -.Lctrmm_kernel_L1_M2_100: +L(ctrmm_kernel_L1_M2_100): SAVE2x1 @@ -2454,15 +2454,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lctrmm_kernel_L1_M2_END: +L(ctrmm_kernel_L1_M2_END): -.Lctrmm_kernel_L1_M1_BEGIN: +L(ctrmm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lctrmm_kernel_L1_END + ble L(ctrmm_kernel_L1_END) -.Lctrmm_kernel_L1_M1_20: +L(ctrmm_kernel_L1_M1_20): INIT1x1 @@ -2486,9 +2486,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L1_M1_40 + ble L(ctrmm_kernel_L1_M1_40) -.Lctrmm_kernel_L1_M1_22: +L(ctrmm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -2500,30 +2500,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M1_22 + bgt L(ctrmm_kernel_L1_M1_22) -.Lctrmm_kernel_L1_M1_40: +L(ctrmm_kernel_L1_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L1_M1_100 + ble L(ctrmm_kernel_L1_M1_100) -.Lctrmm_kernel_L1_M1_42: +L(ctrmm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M1_42 + bgt L(ctrmm_kernel_L1_M1_42) -.Lctrmm_kernel_L1_M1_100: +L(ctrmm_kernel_L1_M1_100): SAVE1x1 -.Lctrmm_kernel_L1_END: +L(ctrmm_kernel_L1_END): -.Lctrmm_kernel_L999: +L(ctrmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/ctrmm_kernel_sve_v1x4.S b/kernel/arm64/ctrmm_kernel_sve_v1x4.S index 242968f636..1b7f23a96e 100644 --- a/kernel/arm64/ctrmm_kernel_sve_v1x4.S +++ b/kernel/arm64/ctrmm_kernel_sve_v1x4.S @@ -584,10 +584,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lctrmm_kernel_L2_BEGIN + ble L(ctrmm_kernel_L2_BEGIN) /******************************************************************************/ -.Lctrmm_kernel_L4_BEGIN: +L(ctrmm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -600,7 +600,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = start of A array -.Lctrmm_kernel_L4_Mv1_BEGIN: +L(ctrmm_kernel_L4_Mv1_BEGIN): /* Loop over M is done in an SVE fashion. This has the benefit of the last M%SVE_LEN iterations being done in a single sweep */ mov counterI, #0 @@ -608,7 +608,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension .align 5 -.Lctrmm_kernel_L4_Mv1_20: +L(ctrmm_kernel_L4_Mv1_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -631,7 +631,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 cmp counterL , #2 - blt .Lctrmm_kernel_L4_Mv1_32 + blt L(ctrmm_kernel_L4_Mv1_32) KERNELv1x4_I KERNELv1x4_M2 @@ -643,10 +643,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lctrmm_kernel_L4_Mv1_22a + ble L(ctrmm_kernel_L4_Mv1_22a) .align 5 -.Lctrmm_kernel_L4_Mv1_22: +L(ctrmm_kernel_L4_Mv1_22): KERNELv1x4_M1 KERNELv1x4_M2 @@ -658,10 +658,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M2 subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_Mv1_22 + bgt L(ctrmm_kernel_L4_Mv1_22) .align 5 -.Lctrmm_kernel_L4_Mv1_22a: +L(ctrmm_kernel_L4_Mv1_22a): KERNELv1x4_M1 KERNELv1x4_M2 @@ -672,13 +672,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M1 KERNELv1x4_E - b .Lctrmm_kernel_L4_Mv1_44 + b L(ctrmm_kernel_L4_Mv1_44) .align 5 -.Lctrmm_kernel_L4_Mv1_32: +L(ctrmm_kernel_L4_Mv1_32): tst counterL, #1 - ble .Lctrmm_kernel_L4_Mv1_40 + ble L(ctrmm_kernel_L4_Mv1_40) KERNELv1x4_I KERNELv1x4_M2 @@ -689,26 +689,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M1 KERNELv1x4_E - b .Lctrmm_kernel_L4_Mv1_44 + b L(ctrmm_kernel_L4_Mv1_44) -.Lctrmm_kernel_L4_Mv1_40: +L(ctrmm_kernel_L4_Mv1_40): INITv1x4 -.Lctrmm_kernel_L4_Mv1_44: +L(ctrmm_kernel_L4_Mv1_44): ands counterL , tempK, #7 - ble .Lctrmm_kernel_L4_Mv1_100 + ble L(ctrmm_kernel_L4_Mv1_100) .align 5 -.Lctrmm_kernel_L4_Mv1_46: +L(ctrmm_kernel_L4_Mv1_46): KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Lctrmm_kernel_L4_Mv1_46 + bne L(ctrmm_kernel_L4_Mv1_46) -.Lctrmm_kernel_L4_Mv1_100: +L(ctrmm_kernel_L4_Mv1_100): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -732,16 +732,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SAVEv1x4 -.Lctrmm_kernel_L4_Mv1_END: +L(ctrmm_kernel_L4_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension - b.any .Lctrmm_kernel_L4_Mv1_20 + b.any L(ctrmm_kernel_L4_Mv1_20) -.Lctrmm_kernel_L4_END: +L(ctrmm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 * 2 @@ -751,19 +751,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Lctrmm_kernel_L4_BEGIN + bgt L(ctrmm_kernel_L4_BEGIN) /******************************************************************************/ -.Lctrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(ctrmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lctrmm_kernel_L999 + ble L(ctrmm_kernel_L999) tst counterJ , #2 - ble .Lctrmm_kernel_L1_BEGIN + ble L(ctrmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC add pCRow1, pCRow0, LDC @@ -778,14 +778,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lctrmm_kernel_L2_Mv1_BEGIN: +L(ctrmm_kernel_L2_Mv1_BEGIN): mov counterI, #0 whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s -.Lctrmm_kernel_L2_Mv1_20: +L(ctrmm_kernel_L2_Mv1_20): INITv1x2 @@ -809,10 +809,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lctrmm_kernel_L2_Mv1_40 + ble L(ctrmm_kernel_L2_Mv1_40) .align 5 -.Lctrmm_kernel_L2_Mv1_22: +L(ctrmm_kernel_L2_Mv1_22): KERNELv1x2_SUB KERNELv1x2_SUB KERNELv1x2_SUB @@ -824,22 +824,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_Mv1_22 + bgt L(ctrmm_kernel_L2_Mv1_22) -.Lctrmm_kernel_L2_Mv1_40: +L(ctrmm_kernel_L2_Mv1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L2_Mv1_100 + ble L(ctrmm_kernel_L2_Mv1_100) -.Lctrmm_kernel_L2_Mv1_42: +L(ctrmm_kernel_L2_Mv1_42): KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_Mv1_42 + bgt L(ctrmm_kernel_L2_Mv1_42) -.Lctrmm_kernel_L2_Mv1_100: +L(ctrmm_kernel_L2_Mv1_100): SAVEv1x2 @@ -859,16 +859,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, lanes #endif -.Lctrmm_kernel_L2_Mv1_END: +L(ctrmm_kernel_L2_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lctrmm_kernel_L2_Mv1_20 + b.any L(ctrmm_kernel_L2_Mv1_20) -.Lctrmm_kernel_L2_END: +L(ctrmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -878,11 +878,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lctrmm_kernel_L1_BEGIN: +L(ctrmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lctrmm_kernel_L999 // done + ble L(ctrmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -894,14 +894,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lctrmm_kernel_L1_Mv1_BEGIN: +L(ctrmm_kernel_L1_Mv1_BEGIN): mov counterI, #0 whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s -.Lctrmm_kernel_L1_Mv1_20: +L(ctrmm_kernel_L1_Mv1_20): INITv1x1 @@ -925,10 +925,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L1_Mv1_40 + ble L(ctrmm_kernel_L1_Mv1_40) .align 5 -.Lctrmm_kernel_L1_Mv1_22: +L(ctrmm_kernel_L1_Mv1_22): KERNELv1x1_SUB KERNELv1x1_SUB KERNELv1x1_SUB @@ -940,22 +940,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_Mv1_22 + bgt L(ctrmm_kernel_L1_Mv1_22) -.Lctrmm_kernel_L1_Mv1_40: +L(ctrmm_kernel_L1_Mv1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L1_Mv1_100 + ble L(ctrmm_kernel_L1_Mv1_100) -.Lctrmm_kernel_L1_Mv1_42: +L(ctrmm_kernel_L1_Mv1_42): KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_Mv1_42 + bgt L(ctrmm_kernel_L1_Mv1_42) -.Lctrmm_kernel_L1_Mv1_100: +L(ctrmm_kernel_L1_Mv1_100): SAVEv1x1 @@ -975,18 +975,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, lanes #endif -.Lctrmm_kernel_L1_Mv1_END: +L(ctrmm_kernel_L1_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lctrmm_kernel_L1_Mv1_20 + b.any L(ctrmm_kernel_L1_Mv1_20) -.Lctrmm_kernel_L1_END: +L(ctrmm_kernel_L1_END): /******************************************************************************/ -.Lctrmm_kernel_L999: +L(ctrmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/daxpy_thunderx2t99.S b/kernel/arm64/daxpy_thunderx2t99.S index baf39150fe..cf2e9fd13f 100644 --- a/kernel/arm64/daxpy_thunderx2t99.S +++ b/kernel/arm64/daxpy_thunderx2t99.S @@ -169,65 +169,65 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Ldaxpy_kernel_L999 + ble L(daxpy_kernel_L999) fcmp DA, #0.0 - beq .Ldaxpy_kernel_L999 + beq L(daxpy_kernel_L999) cmp INC_X, #1 - bne .Ldaxpy_kernel_S_BEGIN + bne L(daxpy_kernel_S_BEGIN) cmp INC_Y, #1 - bne .Ldaxpy_kernel_S_BEGIN + bne L(daxpy_kernel_S_BEGIN) -.Ldaxpy_kernel_F_BEGIN: +L(daxpy_kernel_F_BEGIN): asr I, N, #5 cmp I, xzr - beq .Ldaxpy_kernel_F1 + beq L(daxpy_kernel_F1) cmp N, #2048 - ble .Ldaxpy_kernel_F32_L1CACHE + ble L(daxpy_kernel_F32_L1CACHE) .align 5 -.Ldaxpy_kernel_F32: +L(daxpy_kernel_F32): KERNEL_F32 subs I, I, #1 - bne .Ldaxpy_kernel_F32 - b .Ldaxpy_kernel_F1 + bne L(daxpy_kernel_F32) + b L(daxpy_kernel_F1) .align 5 -.Ldaxpy_kernel_F32_L1CACHE: +L(daxpy_kernel_F32_L1CACHE): KERNEL_F32_L1CACHE subs I, I, #1 - bne .Ldaxpy_kernel_F32_L1CACHE + bne L(daxpy_kernel_F32_L1CACHE) -.Ldaxpy_kernel_F1: +L(daxpy_kernel_F1): ands I, N, #31 - ble .Ldaxpy_kernel_L999 + ble L(daxpy_kernel_L999) -.Ldaxpy_kernel_F10: +L(daxpy_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Ldaxpy_kernel_F10 + bne L(daxpy_kernel_F10) - b .Ldaxpy_kernel_L999 + b L(daxpy_kernel_L999) -.Ldaxpy_kernel_S_BEGIN: +L(daxpy_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Ldaxpy_kernel_S1 + ble L(daxpy_kernel_S1) -.Ldaxpy_kernel_S4: +L(daxpy_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -235,21 +235,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Ldaxpy_kernel_S4 + bne L(daxpy_kernel_S4) -.Ldaxpy_kernel_S1: +L(daxpy_kernel_S1): ands I, N, #3 - ble .Ldaxpy_kernel_L999 + ble L(daxpy_kernel_L999) -.Ldaxpy_kernel_S10: +L(daxpy_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Ldaxpy_kernel_S10 + bne L(daxpy_kernel_S10) -.Ldaxpy_kernel_L999: +L(daxpy_kernel_L999): mov w0, wzr ret + + EPILOGUE diff --git a/kernel/arm64/dgemm_beta.S b/kernel/arm64/dgemm_beta.S index 7d21525c2e..26fa5d7770 100644 --- a/kernel/arm64/dgemm_beta.S +++ b/kernel/arm64/dgemm_beta.S @@ -102,33 +102,33 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ldr LDC, [sp] SAVE_REGS -.Lgemm_beta_BEGIN: +L(gemm_beta_BEGIN): fmov beta0, BETA cmp N, #0 - ble .Lgemm_beta_L999 + ble L(gemm_beta_L999) fcmp BETA, #0.0 - beq .Lgemm_beta_zero_01 + beq L(gemm_beta_zero_01) -.Lgemm_beta_01: +L(gemm_beta_01): lsl LDC, LDC, #3 .align 5 -.Lgemm_beta_02: +L(gemm_beta_02): mov A01, C00 add C00, C00, LDC asr I, M, #4 cmp I, #0 - ble .Lgemm_beta_04 + ble L(gemm_beta_04) add A02, A01, #32 add A03, A02, #32 add A04, A03, #32 .align 5 -.Lgemm_beta_03: +L(gemm_beta_03): ldp q0, q1, [A01] ldp q2, q3, [A02] @@ -161,17 +161,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add A04, A04, calc_size subs I , I , #1 - bne .Lgemm_beta_03 + bne L(gemm_beta_03) .align 5 -.Lgemm_beta_04: +L(gemm_beta_04): and I, M , #15 // M%16 cmp I, #0 - ble .Lgemm_beta_06 + ble L(gemm_beta_06) .align 5 -.Lgemm_beta_05: +L(gemm_beta_05): ldr d12, [A01] fmul d12, d12, beta0 @@ -179,38 +179,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add A01, A01, #8 subs I , I , #1 - bne .Lgemm_beta_05 + bne L(gemm_beta_05) .align 5 -.Lgemm_beta_06: +L(gemm_beta_06): subs N , N, #1 // N-- - bne .Lgemm_beta_02 + bne L(gemm_beta_02) .align 5 -.Lgemm_beta_L999: +L(gemm_beta_L999): mov x0, #0 RESTORE_REGS ret -.Lgemm_beta_zero_01: +L(gemm_beta_zero_01): INIT_ZERO lsl LDC, LDC, #3 .align 5 -.Lgemm_beta_zero_02: +L(gemm_beta_zero_02): mov A01, C00 add C00, C00, LDC asr I, M, #4 cmp I, #0 - ble .Lgemm_beta_zero_04 + ble L(gemm_beta_zero_04) add A02, A01, #64 .align 5 -.Lgemm_beta_zero_03: +L(gemm_beta_zero_03): st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [A01] add A01, A01, calc_size @@ -218,32 +218,32 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add A02, A02, calc_size subs I, I, #1 - bne .Lgemm_beta_zero_03 + bne L(gemm_beta_zero_03) .align 5 -.Lgemm_beta_zero_04: +L(gemm_beta_zero_04): and I, M, #15 cmp I, #0 - ble .Lgemm_beta_zero_06 + ble L(gemm_beta_zero_06) .align 5 -.Lgemm_beta_zero_05: +L(gemm_beta_zero_05): str beta0, [A01] add A01, A01, #8 subs I, I, #1 - bne .Lgemm_beta_zero_05 + bne L(gemm_beta_zero_05) .align 5 -.Lgemm_beta_zero_06: +L(gemm_beta_zero_06): subs N, N, #1 - bne .Lgemm_beta_zero_02 + bne L(gemm_beta_zero_02) .align 5 -.Lgemm_beta_zero_L999: +L(gemm_beta_zero_L999): mov x0, #0 RESTORE_REGS diff --git a/kernel/arm64/dgemm_kernel_4x4.S b/kernel/arm64/dgemm_kernel_4x4.S index 3491670628..5c2ec3a0ed 100644 --- a/kernel/arm64/dgemm_kernel_4x4.S +++ b/kernel/arm64/dgemm_kernel_4x4.S @@ -775,9 +775,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Ldgemm_kernel_L2_BEGIN + ble L(dgemm_kernel_L2_BEGIN) -.Ldgemm_kernel_L4_BEGIN: +L(dgemm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -791,20 +791,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //------------------------------------------------------------------------------ -.Ldgemm_kernel_L4_M8_BEGIN: +L(dgemm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Ldgemm_kernel_L4_M4_BEGIN + ble L(dgemm_kernel_L4_M4_BEGIN) .align 5 -.Ldgemm_kernel_L4_M8_20: +L(dgemm_kernel_L4_M8_20): mov pB, origPB asr counterL , origK, #2 // L = K / 4 cmp counterL , #2 - blt .Ldgemm_kernel_L4_M8_32 + blt L(dgemm_kernel_L4_M8_32) KERNEL8x4_I KERNEL8x4_M2 @@ -812,60 +812,60 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M2 subs counterL, counterL, #2 // subtract 2 - ble .Ldgemm_kernel_L4_M8_22a + ble L(dgemm_kernel_L4_M8_22a) .align 5 -.Ldgemm_kernel_L4_M8_22: +L(dgemm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 KERNEL8x4_M1 KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M8_22 + bgt L(dgemm_kernel_L4_M8_22) .align 5 -.Ldgemm_kernel_L4_M8_22a: +L(dgemm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_M2 KERNEL8x4_M1 KERNEL8x4_E - b .Ldgemm_kernel_L4_M8_44 + b L(dgemm_kernel_L4_M8_44) .align 5 -.Ldgemm_kernel_L4_M8_32: +L(dgemm_kernel_L4_M8_32): tst counterL, #1 - ble .Ldgemm_kernel_L4_M8_40 + ble L(dgemm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_M2 KERNEL8x4_M1 KERNEL8x4_E - b .Ldgemm_kernel_L4_M8_44 + b L(dgemm_kernel_L4_M8_44) -.Ldgemm_kernel_L4_M8_40: +L(dgemm_kernel_L4_M8_40): INIT8x4 -.Ldgemm_kernel_L4_M8_44: +L(dgemm_kernel_L4_M8_44): ands counterL , origK, #3 - ble .Ldgemm_kernel_L4_M8_100 + ble L(dgemm_kernel_L4_M8_100) .align 5 -.Ldgemm_kernel_L4_M8_46: +L(dgemm_kernel_L4_M8_46): KERNEL8x4_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L4_M8_46 + bne L(dgemm_kernel_L4_M8_46) -.Ldgemm_kernel_L4_M8_100: +L(dgemm_kernel_L4_M8_100): lsl temp, origK, #5 prfm PLDL1KEEP, [pA, temp] prfm PLDL1KEEP, [ppA, temp] @@ -873,31 +873,31 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SAVE8x4 -.Ldgemm_kernel_L4_M8_END: +L(dgemm_kernel_L4_M8_END): lsl temp, origK, #5 // k * 4 * 8 add pA, pA, temp add ppA, ppA, temp subs counterI, counterI, #1 - bne .Ldgemm_kernel_L4_M8_20 + bne L(dgemm_kernel_L4_M8_20) -.Ldgemm_kernel_L4_M4_BEGIN: +L(dgemm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Ldgemm_kernel_L4_END + ble L(dgemm_kernel_L4_END) tst counterI, #4 - ble .Ldgemm_kernel_L4_M2_BEGIN + ble L(dgemm_kernel_L4_M2_BEGIN) -.Ldgemm_kernel_L4_M4_20: +L(dgemm_kernel_L4_M4_20): INIT4x4 mov pB, origPB asr counterL, origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Ldgemm_kernel_L4_M4_40 + ble L(dgemm_kernel_L4_M4_40) -.Ldgemm_kernel_L4_M4_22: +L(dgemm_kernel_L4_M4_22): KERNEL4x4_SUB KERNEL4x4_SUB @@ -910,47 +910,47 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M4_22 + bgt L(dgemm_kernel_L4_M4_22) -.Ldgemm_kernel_L4_M4_40: +L(dgemm_kernel_L4_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L4_M4_100 + ble L(dgemm_kernel_L4_M4_100) -.Ldgemm_kernel_L4_M4_42: +L(dgemm_kernel_L4_M4_42): KERNEL4x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M4_42 + bgt L(dgemm_kernel_L4_M4_42) -.Ldgemm_kernel_L4_M4_100: +L(dgemm_kernel_L4_M4_100): SAVE4x4 -.Ldgemm_kernel_L4_M4_END: +L(dgemm_kernel_L4_M4_END): -.Ldgemm_kernel_L4_M2_BEGIN: +L(dgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L4_END + ble L(dgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L4_M1_BEGIN + ble L(dgemm_kernel_L4_M1_BEGIN) -.Ldgemm_kernel_L4_M2_20: +L(dgemm_kernel_L4_M2_20): INIT2x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L4_M2_40 + ble L(dgemm_kernel_L4_M2_40) -.Ldgemm_kernel_L4_M2_22: +L(dgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -963,43 +963,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M2_22 + bgt L(dgemm_kernel_L4_M2_22) -.Ldgemm_kernel_L4_M2_40: +L(dgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L4_M2_100 + ble L(dgemm_kernel_L4_M2_100) -.Ldgemm_kernel_L4_M2_42: +L(dgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M2_42 + bgt L(dgemm_kernel_L4_M2_42) -.Ldgemm_kernel_L4_M2_100: +L(dgemm_kernel_L4_M2_100): SAVE2x4 -.Ldgemm_kernel_L4_M2_END: +L(dgemm_kernel_L4_M2_END): -.Ldgemm_kernel_L4_M1_BEGIN: +L(dgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L4_END + ble L(dgemm_kernel_L4_END) -.Ldgemm_kernel_L4_M1_20: +L(dgemm_kernel_L4_M1_20): INIT1x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L4_M1_40 + ble L(dgemm_kernel_L4_M1_40) -.Ldgemm_kernel_L4_M1_22: +L(dgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1011,45 +1011,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M1_22 + bgt L(dgemm_kernel_L4_M1_22) -.Ldgemm_kernel_L4_M1_40: +L(dgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L4_M1_100 + ble L(dgemm_kernel_L4_M1_100) -.Ldgemm_kernel_L4_M1_42: +L(dgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M1_42 + bgt L(dgemm_kernel_L4_M1_42) -.Ldgemm_kernel_L4_M1_100: +L(dgemm_kernel_L4_M1_100): SAVE1x4 -.Ldgemm_kernel_L4_END: +L(dgemm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 subs counterJ, counterJ , #1 // j-- - bgt .Ldgemm_kernel_L4_BEGIN + bgt L(dgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Ldgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(dgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Ldgemm_kernel_L999 // error, N was less than 4? + ble L(dgemm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Ldgemm_kernel_L1_BEGIN + ble L(dgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1059,24 +1059,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Ldgemm_kernel_L2_M4_BEGIN: +L(dgemm_kernel_L2_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI,#0 - ble .Ldgemm_kernel_L2_M2_BEGIN + ble L(dgemm_kernel_L2_M2_BEGIN) -.Ldgemm_kernel_L2_M4_20: +L(dgemm_kernel_L2_M4_20): INIT4x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldgemm_kernel_L2_M4_40 + ble L(dgemm_kernel_L2_M4_40) .align 5 -.Ldgemm_kernel_L2_M4_22: +L(dgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1088,50 +1088,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M4_22 + bgt L(dgemm_kernel_L2_M4_22) -.Ldgemm_kernel_L2_M4_40: +L(dgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M4_100 + ble L(dgemm_kernel_L2_M4_100) -.Ldgemm_kernel_L2_M4_42: +L(dgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M4_42 + bgt L(dgemm_kernel_L2_M4_42) -.Ldgemm_kernel_L2_M4_100: +L(dgemm_kernel_L2_M4_100): SAVE4x2 -.Ldgemm_kernel_L2_M4_END: +L(dgemm_kernel_L2_M4_END): subs counterI, counterI, #1 - bgt .Ldgemm_kernel_L2_M4_20 + bgt L(dgemm_kernel_L2_M4_20) -.Ldgemm_kernel_L2_M2_BEGIN: +L(dgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L2_END + ble L(dgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L2_M1_BEGIN + ble L(dgemm_kernel_L2_M1_BEGIN) -.Ldgemm_kernel_L2_M2_20: +L(dgemm_kernel_L2_M2_20): INIT2x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldgemm_kernel_L2_M2_40 + ble L(dgemm_kernel_L2_M2_40) -.Ldgemm_kernel_L2_M2_22: +L(dgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1144,43 +1144,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M2_22 + bgt L(dgemm_kernel_L2_M2_22) -.Ldgemm_kernel_L2_M2_40: +L(dgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M2_100 + ble L(dgemm_kernel_L2_M2_100) -.Ldgemm_kernel_L2_M2_42: +L(dgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M2_42 + bgt L(dgemm_kernel_L2_M2_42) -.Ldgemm_kernel_L2_M2_100: +L(dgemm_kernel_L2_M2_100): SAVE2x2 -.Ldgemm_kernel_L2_M2_END: +L(dgemm_kernel_L2_M2_END): -.Ldgemm_kernel_L2_M1_BEGIN: +L(dgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L2_END + ble L(dgemm_kernel_L2_END) -.Ldgemm_kernel_L2_M1_20: +L(dgemm_kernel_L2_M1_20): INIT1x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Ldgemm_kernel_L2_M1_40 + ble L(dgemm_kernel_L2_M1_40) -.Ldgemm_kernel_L2_M1_22: +L(dgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1192,36 +1192,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M1_22 + bgt L(dgemm_kernel_L2_M1_22) -.Ldgemm_kernel_L2_M1_40: +L(dgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M1_100 + ble L(dgemm_kernel_L2_M1_100) -.Ldgemm_kernel_L2_M1_42: +L(dgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M1_42 + bgt L(dgemm_kernel_L2_M1_42) -.Ldgemm_kernel_L2_M1_100: +L(dgemm_kernel_L2_M1_100): SAVE1x2 -.Ldgemm_kernel_L2_END: +L(dgemm_kernel_L2_END): add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 /******************************************************************************/ -.Ldgemm_kernel_L1_BEGIN: +L(dgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldgemm_kernel_L999 // done + ble L(dgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1231,24 +1231,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Ldgemm_kernel_L1_M4_BEGIN: +L(dgemm_kernel_L1_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Ldgemm_kernel_L1_M2_BEGIN + ble L(dgemm_kernel_L1_M2_BEGIN) -.Ldgemm_kernel_L1_M4_20: +L(dgemm_kernel_L1_M4_20): INIT4x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M4_40 + ble L(dgemm_kernel_L1_M4_40) .align 5 -.Ldgemm_kernel_L1_M4_22: +L(dgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1260,50 +1260,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M4_22 + bgt L(dgemm_kernel_L1_M4_22) -.Ldgemm_kernel_L1_M4_40: +L(dgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M4_100 + ble L(dgemm_kernel_L1_M4_100) -.Ldgemm_kernel_L1_M4_42: +L(dgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M4_42 + bgt L(dgemm_kernel_L1_M4_42) -.Ldgemm_kernel_L1_M4_100: +L(dgemm_kernel_L1_M4_100): SAVE4x1 -.Ldgemm_kernel_L1_M4_END: +L(dgemm_kernel_L1_M4_END): subs counterI, counterI, #1 - bgt .Ldgemm_kernel_L1_M4_20 + bgt L(dgemm_kernel_L1_M4_20) -.Ldgemm_kernel_L1_M2_BEGIN: +L(dgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L1_END + ble L(dgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L1_M1_BEGIN + ble L(dgemm_kernel_L1_M1_BEGIN) -.Ldgemm_kernel_L1_M2_20: +L(dgemm_kernel_L1_M2_20): INIT2x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M2_40 + ble L(dgemm_kernel_L1_M2_40) -.Ldgemm_kernel_L1_M2_22: +L(dgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1316,43 +1316,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M2_22 + bgt L(dgemm_kernel_L1_M2_22) -.Ldgemm_kernel_L1_M2_40: +L(dgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M2_100 + ble L(dgemm_kernel_L1_M2_100) -.Ldgemm_kernel_L1_M2_42: +L(dgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M2_42 + bgt L(dgemm_kernel_L1_M2_42) -.Ldgemm_kernel_L1_M2_100: +L(dgemm_kernel_L1_M2_100): SAVE2x1 -.Ldgemm_kernel_L1_M2_END: +L(dgemm_kernel_L1_M2_END): -.Ldgemm_kernel_L1_M1_BEGIN: +L(dgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L1_END + ble L(dgemm_kernel_L1_END) -.Ldgemm_kernel_L1_M1_20: +L(dgemm_kernel_L1_M1_20): INIT1x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M1_40 + ble L(dgemm_kernel_L1_M1_40) -.Ldgemm_kernel_L1_M1_22: +L(dgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1364,30 +1364,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M1_22 + bgt L(dgemm_kernel_L1_M1_22) -.Ldgemm_kernel_L1_M1_40: +L(dgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M1_100 + ble L(dgemm_kernel_L1_M1_100) -.Ldgemm_kernel_L1_M1_42: +L(dgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M1_42 + bgt L(dgemm_kernel_L1_M1_42) -.Ldgemm_kernel_L1_M1_100: +L(dgemm_kernel_L1_M1_100): SAVE1x1 -.Ldgemm_kernel_L1_END: +L(dgemm_kernel_L1_END): -.Ldgemm_kernel_L999: +L(dgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dgemm_kernel_4x8.S b/kernel/arm64/dgemm_kernel_4x8.S index ced26b49ce..f4af8cc25d 100644 --- a/kernel/arm64/dgemm_kernel_4x8.S +++ b/kernel/arm64/dgemm_kernel_4x8.S @@ -938,98 +938,98 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Ldgemm_kernel_L4_BEGIN + ble L(dgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Ldgemm_kernel_L8_BEGIN: +L(dgemm_kernel_L8_BEGIN): mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #3 mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L8_M4_BEGIN: +L(dgemm_kernel_L8_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Ldgemm_kernel_L8_M2_BEGIN + ble L(dgemm_kernel_L8_M2_BEGIN) -.Ldgemm_kernel_L8_M4_20: +L(dgemm_kernel_L8_M4_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Ldgemm_kernel_L8_M4_32 + blt L(dgemm_kernel_L8_M4_32) KERNEL4x8_I // do one in the K KERNEL4x8_M2 // do another in the K subs counterL, counterL, #2 - ble .Ldgemm_kernel_L8_M4_22a + ble L(dgemm_kernel_L8_M4_22a) .align 5 -.Ldgemm_kernel_L8_M4_22: +L(dgemm_kernel_L8_M4_22): KERNEL4x8_M1 KERNEL4x8_M2 subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L8_M4_22 + bgt L(dgemm_kernel_L8_M4_22) -.Ldgemm_kernel_L8_M4_22a: +L(dgemm_kernel_L8_M4_22a): KERNEL4x8_M1 KERNEL4x8_E - b .Ldgemm_kernel_L8_M4_44 + b L(dgemm_kernel_L8_M4_44) -.Ldgemm_kernel_L8_M4_32: +L(dgemm_kernel_L8_M4_32): tst counterL, #1 - ble .Ldgemm_kernel_L8_M4_40 + ble L(dgemm_kernel_L8_M4_40) KERNEL4x8_I KERNEL4x8_E - b .Ldgemm_kernel_L8_M4_44 + b L(dgemm_kernel_L8_M4_44) -.Ldgemm_kernel_L8_M4_40: +L(dgemm_kernel_L8_M4_40): INIT4x8 -.Ldgemm_kernel_L8_M4_44: +L(dgemm_kernel_L8_M4_44): ands counterL , origK, #1 - ble .Ldgemm_kernel_L8_M4_100 + ble L(dgemm_kernel_L8_M4_100) -.Ldgemm_kernel_L8_M4_46: +L(dgemm_kernel_L8_M4_46): KERNEL4x8_SUB -.Ldgemm_kernel_L8_M4_100: +L(dgemm_kernel_L8_M4_100): SAVE4x8 -.Ldgemm_kernel_L8_M4_END: +L(dgemm_kernel_L8_M4_END): subs counterI, counterI, #1 - bne .Ldgemm_kernel_L8_M4_20 + bne L(dgemm_kernel_L8_M4_20) -.Ldgemm_kernel_L8_M2_BEGIN: +L(dgemm_kernel_L8_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L8_END + ble L(dgemm_kernel_L8_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L8_M1_BEGIN + ble L(dgemm_kernel_L8_M1_BEGIN) -.Ldgemm_kernel_L8_M2_20: +L(dgemm_kernel_L8_M2_20): INIT2x8 @@ -1037,9 +1037,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L8_M2_40 + ble L(dgemm_kernel_L8_M2_40) -.Ldgemm_kernel_L8_M2_22: +L(dgemm_kernel_L8_M2_22): KERNEL2x8_SUB KERNEL2x8_SUB @@ -1052,34 +1052,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L8_M2_22 + bgt L(dgemm_kernel_L8_M2_22) -.Ldgemm_kernel_L8_M2_40: +L(dgemm_kernel_L8_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L8_M2_100 + ble L(dgemm_kernel_L8_M2_100) -.Ldgemm_kernel_L8_M2_42: +L(dgemm_kernel_L8_M2_42): KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L8_M2_42 + bgt L(dgemm_kernel_L8_M2_42) -.Ldgemm_kernel_L8_M2_100: +L(dgemm_kernel_L8_M2_100): SAVE2x8 -.Ldgemm_kernel_L8_M2_END: +L(dgemm_kernel_L8_M2_END): -.Ldgemm_kernel_L8_M1_BEGIN: +L(dgemm_kernel_L8_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L8_END + ble L(dgemm_kernel_L8_END) -.Ldgemm_kernel_L8_M1_20: +L(dgemm_kernel_L8_M1_20): INIT1x8 @@ -1087,9 +1087,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L8_M1_40 + ble L(dgemm_kernel_L8_M1_40) -.Ldgemm_kernel_L8_M1_22: +L(dgemm_kernel_L8_M1_22): KERNEL1x8_SUB KERNEL1x8_SUB KERNEL1x8_SUB @@ -1101,131 +1101,131 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L8_M1_22 + bgt L(dgemm_kernel_L8_M1_22) -.Ldgemm_kernel_L8_M1_40: +L(dgemm_kernel_L8_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L8_M1_100 + ble L(dgemm_kernel_L8_M1_100) -.Ldgemm_kernel_L8_M1_42: +L(dgemm_kernel_L8_M1_42): KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L8_M1_42 + bgt L(dgemm_kernel_L8_M1_42) -.Ldgemm_kernel_L8_M1_100: +L(dgemm_kernel_L8_M1_100): SAVE1x8 -.Ldgemm_kernel_L8_END: +L(dgemm_kernel_L8_END): lsl temp, origK, #6 add origPB, origPB, temp // B = B + K * 8 * 8 subs counterJ, counterJ , #1 // j-- - bgt .Ldgemm_kernel_L8_BEGIN + bgt L(dgemm_kernel_L8_BEGIN) /******************************************************************************/ -.Ldgemm_kernel_L4_BEGIN: +L(dgemm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #7 - ble .Ldgemm_kernel_L999 + ble L(dgemm_kernel_L999) tst counterJ , #4 - ble .Ldgemm_kernel_L2_BEGIN + ble L(dgemm_kernel_L2_BEGIN) mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #2 mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L4_M4_BEGIN: +L(dgemm_kernel_L4_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Ldgemm_kernel_L4_M2_BEGIN + ble L(dgemm_kernel_L4_M2_BEGIN) -.Ldgemm_kernel_L4_M4_20: +L(dgemm_kernel_L4_M4_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Ldgemm_kernel_L4_M4_32 + blt L(dgemm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Ldgemm_kernel_L4_M4_22a + ble L(dgemm_kernel_L4_M4_22a) .align 5 -.Ldgemm_kernel_L4_M4_22: +L(dgemm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M4_22 + bgt L(dgemm_kernel_L4_M4_22) -.Ldgemm_kernel_L4_M4_22a: +L(dgemm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Ldgemm_kernel_L4_M4_44 + b L(dgemm_kernel_L4_M4_44) -.Ldgemm_kernel_L4_M4_32: +L(dgemm_kernel_L4_M4_32): tst counterL, #1 - ble .Ldgemm_kernel_L4_M4_40 + ble L(dgemm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Ldgemm_kernel_L4_M4_44 + b L(dgemm_kernel_L4_M4_44) -.Ldgemm_kernel_L4_M4_40: +L(dgemm_kernel_L4_M4_40): INIT4x4 -.Ldgemm_kernel_L4_M4_44: +L(dgemm_kernel_L4_M4_44): ands counterL , origK, #1 - ble .Ldgemm_kernel_L4_M4_100 + ble L(dgemm_kernel_L4_M4_100) -.Ldgemm_kernel_L4_M4_46: +L(dgemm_kernel_L4_M4_46): KERNEL4x4_SUB -.Ldgemm_kernel_L4_M4_100: +L(dgemm_kernel_L4_M4_100): SAVE4x4 -.Ldgemm_kernel_L4_M4_END: +L(dgemm_kernel_L4_M4_END): subs counterI, counterI, #1 - bne .Ldgemm_kernel_L4_M4_20 + bne L(dgemm_kernel_L4_M4_20) -.Ldgemm_kernel_L4_M2_BEGIN: +L(dgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L4_END + ble L(dgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L4_M1_BEGIN + ble L(dgemm_kernel_L4_M1_BEGIN) -.Ldgemm_kernel_L4_M2_20: +L(dgemm_kernel_L4_M2_20): INIT2x4 @@ -1233,9 +1233,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L4_M2_40 + ble L(dgemm_kernel_L4_M2_40) -.Ldgemm_kernel_L4_M2_22: +L(dgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1248,34 +1248,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M2_22 + bgt L(dgemm_kernel_L4_M2_22) -.Ldgemm_kernel_L4_M2_40: +L(dgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L4_M2_100 + ble L(dgemm_kernel_L4_M2_100) -.Ldgemm_kernel_L4_M2_42: +L(dgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M2_42 + bgt L(dgemm_kernel_L4_M2_42) -.Ldgemm_kernel_L4_M2_100: +L(dgemm_kernel_L4_M2_100): SAVE2x4 -.Ldgemm_kernel_L4_M2_END: +L(dgemm_kernel_L4_M2_END): -.Ldgemm_kernel_L4_M1_BEGIN: +L(dgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L4_END + ble L(dgemm_kernel_L4_END) -.Ldgemm_kernel_L4_M1_20: +L(dgemm_kernel_L4_M1_20): INIT1x4 @@ -1283,9 +1283,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L4_M1_40 + ble L(dgemm_kernel_L4_M1_40) -.Ldgemm_kernel_L4_M1_22: +L(dgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1297,40 +1297,40 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M1_22 + bgt L(dgemm_kernel_L4_M1_22) -.Ldgemm_kernel_L4_M1_40: +L(dgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L4_M1_100 + ble L(dgemm_kernel_L4_M1_100) -.Ldgemm_kernel_L4_M1_42: +L(dgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M1_42 + bgt L(dgemm_kernel_L4_M1_42) -.Ldgemm_kernel_L4_M1_100: +L(dgemm_kernel_L4_M1_100): SAVE1x4 -.Ldgemm_kernel_L4_END: +L(dgemm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 /******************************************************************************/ -.Ldgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(dgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Ldgemm_kernel_L999 // error, N was less than 4? + ble L(dgemm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Ldgemm_kernel_L1_BEGIN + ble L(dgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1339,14 +1339,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Ldgemm_kernel_L2_M4_BEGIN: +L(dgemm_kernel_L2_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI,#0 - ble .Ldgemm_kernel_L2_M2_BEGIN + ble L(dgemm_kernel_L2_M2_BEGIN) -.Ldgemm_kernel_L2_M4_20: +L(dgemm_kernel_L2_M4_20): INIT4x2 @@ -1354,10 +1354,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldgemm_kernel_L2_M4_40 + ble L(dgemm_kernel_L2_M4_40) .align 5 -.Ldgemm_kernel_L2_M4_22: +L(dgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1369,41 +1369,41 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M4_22 + bgt L(dgemm_kernel_L2_M4_22) -.Ldgemm_kernel_L2_M4_40: +L(dgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M4_100 + ble L(dgemm_kernel_L2_M4_100) -.Ldgemm_kernel_L2_M4_42: +L(dgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M4_42 + bgt L(dgemm_kernel_L2_M4_42) -.Ldgemm_kernel_L2_M4_100: +L(dgemm_kernel_L2_M4_100): SAVE4x2 -.Ldgemm_kernel_L2_M4_END: +L(dgemm_kernel_L2_M4_END): subs counterI, counterI, #1 - bgt .Ldgemm_kernel_L2_M4_20 + bgt L(dgemm_kernel_L2_M4_20) -.Ldgemm_kernel_L2_M2_BEGIN: +L(dgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L2_END + ble L(dgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L2_M1_BEGIN + ble L(dgemm_kernel_L2_M1_BEGIN) -.Ldgemm_kernel_L2_M2_20: +L(dgemm_kernel_L2_M2_20): INIT2x2 @@ -1411,9 +1411,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldgemm_kernel_L2_M2_40 + ble L(dgemm_kernel_L2_M2_40) -.Ldgemm_kernel_L2_M2_22: +L(dgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1426,34 +1426,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M2_22 + bgt L(dgemm_kernel_L2_M2_22) -.Ldgemm_kernel_L2_M2_40: +L(dgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M2_100 + ble L(dgemm_kernel_L2_M2_100) -.Ldgemm_kernel_L2_M2_42: +L(dgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M2_42 + bgt L(dgemm_kernel_L2_M2_42) -.Ldgemm_kernel_L2_M2_100: +L(dgemm_kernel_L2_M2_100): SAVE2x2 -.Ldgemm_kernel_L2_M2_END: +L(dgemm_kernel_L2_M2_END): -.Ldgemm_kernel_L2_M1_BEGIN: +L(dgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L2_END + ble L(dgemm_kernel_L2_END) -.Ldgemm_kernel_L2_M1_20: +L(dgemm_kernel_L2_M1_20): INIT1x2 @@ -1461,9 +1461,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Ldgemm_kernel_L2_M1_40 + ble L(dgemm_kernel_L2_M1_40) -.Ldgemm_kernel_L2_M1_22: +L(dgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1475,35 +1475,35 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M1_22 + bgt L(dgemm_kernel_L2_M1_22) -.Ldgemm_kernel_L2_M1_40: +L(dgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M1_100 + ble L(dgemm_kernel_L2_M1_100) -.Ldgemm_kernel_L2_M1_42: +L(dgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M1_42 + bgt L(dgemm_kernel_L2_M1_42) -.Ldgemm_kernel_L2_M1_100: +L(dgemm_kernel_L2_M1_100): SAVE1x2 -.Ldgemm_kernel_L2_END: +L(dgemm_kernel_L2_END): add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 /******************************************************************************/ -.Ldgemm_kernel_L1_BEGIN: +L(dgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldgemm_kernel_L999 // done + ble L(dgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1511,24 +1511,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Ldgemm_kernel_L1_M4_BEGIN: +L(dgemm_kernel_L1_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Ldgemm_kernel_L1_M2_BEGIN + ble L(dgemm_kernel_L1_M2_BEGIN) -.Ldgemm_kernel_L1_M4_20: +L(dgemm_kernel_L1_M4_20): INIT4x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M4_40 + ble L(dgemm_kernel_L1_M4_40) .align 5 -.Ldgemm_kernel_L1_M4_22: +L(dgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1540,41 +1540,41 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M4_22 + bgt L(dgemm_kernel_L1_M4_22) -.Ldgemm_kernel_L1_M4_40: +L(dgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M4_100 + ble L(dgemm_kernel_L1_M4_100) -.Ldgemm_kernel_L1_M4_42: +L(dgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M4_42 + bgt L(dgemm_kernel_L1_M4_42) -.Ldgemm_kernel_L1_M4_100: +L(dgemm_kernel_L1_M4_100): SAVE4x1 -.Ldgemm_kernel_L1_M4_END: +L(dgemm_kernel_L1_M4_END): subs counterI, counterI, #1 - bgt .Ldgemm_kernel_L1_M4_20 + bgt L(dgemm_kernel_L1_M4_20) -.Ldgemm_kernel_L1_M2_BEGIN: +L(dgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L1_END + ble L(dgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L1_M1_BEGIN + ble L(dgemm_kernel_L1_M1_BEGIN) -.Ldgemm_kernel_L1_M2_20: +L(dgemm_kernel_L1_M2_20): INIT2x1 @@ -1582,9 +1582,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M2_40 + ble L(dgemm_kernel_L1_M2_40) -.Ldgemm_kernel_L1_M2_22: +L(dgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1597,34 +1597,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M2_22 + bgt L(dgemm_kernel_L1_M2_22) -.Ldgemm_kernel_L1_M2_40: +L(dgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M2_100 + ble L(dgemm_kernel_L1_M2_100) -.Ldgemm_kernel_L1_M2_42: +L(dgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M2_42 + bgt L(dgemm_kernel_L1_M2_42) -.Ldgemm_kernel_L1_M2_100: +L(dgemm_kernel_L1_M2_100): SAVE2x1 -.Ldgemm_kernel_L1_M2_END: +L(dgemm_kernel_L1_M2_END): -.Ldgemm_kernel_L1_M1_BEGIN: +L(dgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L1_END + ble L(dgemm_kernel_L1_END) -.Ldgemm_kernel_L1_M1_20: +L(dgemm_kernel_L1_M1_20): INIT1x1 @@ -1632,9 +1632,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M1_40 + ble L(dgemm_kernel_L1_M1_40) -.Ldgemm_kernel_L1_M1_22: +L(dgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1646,30 +1646,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M1_22 + bgt L(dgemm_kernel_L1_M1_22) -.Ldgemm_kernel_L1_M1_40: +L(dgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M1_100 + ble L(dgemm_kernel_L1_M1_100) -.Ldgemm_kernel_L1_M1_42: +L(dgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M1_42 + bgt L(dgemm_kernel_L1_M1_42) -.Ldgemm_kernel_L1_M1_100: +L(dgemm_kernel_L1_M1_100): SAVE1x1 -.Ldgemm_kernel_L1_END: +L(dgemm_kernel_L1_END): -.Ldgemm_kernel_L999: +L(dgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dgemm_kernel_6x8_cortexa72.S b/kernel/arm64/dgemm_kernel_6x8_cortexa72.S index 54478af0b3..5a2a7f0999 100644 --- a/kernel/arm64/dgemm_kernel_6x8_cortexa72.S +++ b/kernel/arm64/dgemm_kernel_6x8_cortexa72.S @@ -774,86 +774,86 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 - cbz counterJ, .L6x8_L4_BEGIN + cbz counterJ, L(6x8_L4_BEGIN) -.L6x8_L8_BEGIN: +L(6x8_L8_BEGIN): mov pCRow0, pC add pC, pC, LDC, lsl #3 mov pA, origPA mov temp, #6 udiv counterI, origM, temp - cbz counterI, .L6x8_L8_Mrem + cbz counterI, L(6x8_L8_Mrem) -.L6x8_L8_M6: +L(6x8_L8_M6): INIT6x8 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L8_M6s -.L6x8_L8_M6l: + cbz counterL, L(6x8_L8_M6s) +L(6x8_L8_M6l): KERNEL6x8_SUB subs counterL, counterL, #1 - bgt .L6x8_L8_M6l -.L6x8_L8_M6s: + bgt L(6x8_L8_M6l) +L(6x8_L8_M6s): SAVE6x8 subs counterI, counterI, #1 - bgt .L6x8_L8_M6 + bgt L(6x8_L8_M6) -.L6x8_L8_Mrem: +L(6x8_L8_Mrem): mov temp, #6 udiv counterI, origM, temp msub counterI, counterI, temp, origM cmp counterI, #4 - blt .L6x8_L8_M2c + blt L(6x8_L8_M2c) INIT4x8 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L8_M4s -.L6x8_L8_M4l: + cbz counterL, L(6x8_L8_M4s) +L(6x8_L8_M4l): KERNEL4x8_SUB subs counterL, counterL, #1 - bgt .L6x8_L8_M4l -.L6x8_L8_M4s: + bgt L(6x8_L8_M4l) +L(6x8_L8_M4s): SAVE4x8 sub counterI, counterI, #4 -.L6x8_L8_M2c: +L(6x8_L8_M2c): cmp counterI, #2 - blt .L6x8_L8_M1c + blt L(6x8_L8_M1c) INIT2x8 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L8_M2s -.L6x8_L8_M2l: + cbz counterL, L(6x8_L8_M2s) +L(6x8_L8_M2l): KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .L6x8_L8_M2l -.L6x8_L8_M2s: + bgt L(6x8_L8_M2l) +L(6x8_L8_M2s): SAVE2x8 sub counterI, counterI, #2 -.L6x8_L8_M1c: - cbz counterI, .L6x8_L8_END +L(6x8_L8_M1c): + cbz counterI, L(6x8_L8_END) INIT1x8 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L8_M1s -.L6x8_L8_M1l: + cbz counterL, L(6x8_L8_M1s) +L(6x8_L8_M1l): KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .L6x8_L8_M1l -.L6x8_L8_M1s: + bgt L(6x8_L8_M1l) +L(6x8_L8_M1s): SAVE1x8 -.L6x8_L8_END: +L(6x8_L8_END): lsl temp, origK, #6 /* + K*8*8 bytes */ add origPB, origPB, temp subs counterJ, counterJ, #1 - bgt .L6x8_L8_BEGIN + bgt L(6x8_L8_BEGIN) -.L6x8_L4_BEGIN: +L(6x8_L4_BEGIN): tst origN, #7 - beq .L6x8_L999 + beq L(6x8_L999) tst origN, #4 - beq .L6x8_L2_BEGIN + beq L(6x8_L2_BEGIN) mov pCRow0, pC add pC, pC, LDC, lsl #2 @@ -861,200 +861,200 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov temp, #6 udiv counterI, origM, temp - cbz counterI, .L6x8_L4_Mrem -.L6x8_L4_M6: + cbz counterI, L(6x8_L4_Mrem) +L(6x8_L4_M6): INIT6x4 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L4_M6s -.L6x8_L4_M6l: + cbz counterL, L(6x8_L4_M6s) +L(6x8_L4_M6l): KERNEL6x4_SUB subs counterL, counterL, #1 - bgt .L6x8_L4_M6l -.L6x8_L4_M6s: + bgt L(6x8_L4_M6l) +L(6x8_L4_M6s): SAVE6x4 subs counterI, counterI, #1 - bgt .L6x8_L4_M6 -.L6x8_L4_Mrem: + bgt L(6x8_L4_M6) +L(6x8_L4_Mrem): mov temp, #6 udiv counterI, origM, temp msub counterI, counterI, temp, origM cmp counterI, #4 - blt .L6x8_L4_M2c + blt L(6x8_L4_M2c) INIT4x4 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L4_M4s -.L6x8_L4_M4l: + cbz counterL, L(6x8_L4_M4s) +L(6x8_L4_M4l): KERNEL4x4_SUB subs counterL, counterL, #1 - bgt .L6x8_L4_M4l -.L6x8_L4_M4s: + bgt L(6x8_L4_M4l) +L(6x8_L4_M4s): SAVE4x4 sub counterI, counterI, #4 -.L6x8_L4_M2c: +L(6x8_L4_M2c): cmp counterI, #2 - blt .L6x8_L4_M1c + blt L(6x8_L4_M1c) INIT2x4 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L4_M2s -.L6x8_L4_M2l: + cbz counterL, L(6x8_L4_M2s) +L(6x8_L4_M2l): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .L6x8_L4_M2l -.L6x8_L4_M2s: + bgt L(6x8_L4_M2l) +L(6x8_L4_M2s): SAVE2x4 sub counterI, counterI, #2 -.L6x8_L4_M1c: - cbz counterI, .L6x8_L4_END +L(6x8_L4_M1c): + cbz counterI, L(6x8_L4_END) INIT1x4 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L4_M1s -.L6x8_L4_M1l: + cbz counterL, L(6x8_L4_M1s) +L(6x8_L4_M1l): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .L6x8_L4_M1l -.L6x8_L4_M1s: + bgt L(6x8_L4_M1l) +L(6x8_L4_M1s): SAVE1x4 -.L6x8_L4_END: +L(6x8_L4_END): add origPB, origPB, origK, lsl #5 /* +K*4*8 */ -.L6x8_L2_BEGIN: +L(6x8_L2_BEGIN): tst origN, #2 - beq .L6x8_L1_BEGIN + beq L(6x8_L1_BEGIN) mov pCRow0, pC add pC, pC, LDC, lsl #1 mov pA, origPA mov temp, #6 udiv counterI, origM, temp - cbz counterI, .L6x8_L2_Mrem -.L6x8_L2_M6: + cbz counterI, L(6x8_L2_Mrem) +L(6x8_L2_M6): INIT6x2 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L2_M6s -.L6x8_L2_M6l: + cbz counterL, L(6x8_L2_M6s) +L(6x8_L2_M6l): KERNEL6x2_SUB subs counterL, counterL, #1 - bgt .L6x8_L2_M6l -.L6x8_L2_M6s: + bgt L(6x8_L2_M6l) +L(6x8_L2_M6s): SAVE6x2 subs counterI, counterI, #1 - bgt .L6x8_L2_M6 -.L6x8_L2_Mrem: + bgt L(6x8_L2_M6) +L(6x8_L2_Mrem): mov temp, #6 udiv counterI, origM, temp msub counterI, counterI, temp, origM cmp counterI, #4 - blt .L6x8_L2_M2c + blt L(6x8_L2_M2c) INIT4x2 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L2_M4s -.L6x8_L2_M4l: + cbz counterL, L(6x8_L2_M4s) +L(6x8_L2_M4l): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .L6x8_L2_M4l -.L6x8_L2_M4s: + bgt L(6x8_L2_M4l) +L(6x8_L2_M4s): SAVE4x2 sub counterI, counterI, #4 -.L6x8_L2_M2c: +L(6x8_L2_M2c): cmp counterI, #2 - blt .L6x8_L2_M1c + blt L(6x8_L2_M1c) INIT2x2 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L2_M2s -.L6x8_L2_M2l: + cbz counterL, L(6x8_L2_M2s) +L(6x8_L2_M2l): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .L6x8_L2_M2l -.L6x8_L2_M2s: + bgt L(6x8_L2_M2l) +L(6x8_L2_M2s): SAVE2x2 sub counterI, counterI, #2 -.L6x8_L2_M1c: - cbz counterI, .L6x8_L2_END +L(6x8_L2_M1c): + cbz counterI, L(6x8_L2_END) INIT1x2 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L2_M1s -.L6x8_L2_M1l: + cbz counterL, L(6x8_L2_M1s) +L(6x8_L2_M1l): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .L6x8_L2_M1l -.L6x8_L2_M1s: + bgt L(6x8_L2_M1l) +L(6x8_L2_M1s): SAVE1x2 -.L6x8_L2_END: +L(6x8_L2_END): add origPB, origPB, origK, lsl #4 -.L6x8_L1_BEGIN: +L(6x8_L1_BEGIN): tst origN, #1 - beq .L6x8_L999 + beq L(6x8_L999) mov pCRow0, pC mov pA, origPA mov temp, #6 udiv counterI, origM, temp - cbz counterI, .L6x8_L1_Mrem -.L6x8_L1_M6: + cbz counterI, L(6x8_L1_Mrem) +L(6x8_L1_M6): INIT6x1 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L1_M6s -.L6x8_L1_M6l: + cbz counterL, L(6x8_L1_M6s) +L(6x8_L1_M6l): KERNEL6x1_SUB subs counterL, counterL, #1 - bgt .L6x8_L1_M6l -.L6x8_L1_M6s: + bgt L(6x8_L1_M6l) +L(6x8_L1_M6s): SAVE6x1 subs counterI, counterI, #1 - bgt .L6x8_L1_M6 -.L6x8_L1_Mrem: + bgt L(6x8_L1_M6) +L(6x8_L1_Mrem): mov temp, #6 udiv counterI, origM, temp msub counterI, counterI, temp, origM cmp counterI, #4 - blt .L6x8_L1_M2c + blt L(6x8_L1_M2c) INIT4x1 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L1_M4s -.L6x8_L1_M4l: + cbz counterL, L(6x8_L1_M4s) +L(6x8_L1_M4l): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .L6x8_L1_M4l -.L6x8_L1_M4s: + bgt L(6x8_L1_M4l) +L(6x8_L1_M4s): SAVE4x1 sub counterI, counterI, #4 -.L6x8_L1_M2c: +L(6x8_L1_M2c): cmp counterI, #2 - blt .L6x8_L1_M1c + blt L(6x8_L1_M1c) INIT2x1 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L1_M2s -.L6x8_L1_M2l: + cbz counterL, L(6x8_L1_M2s) +L(6x8_L1_M2l): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .L6x8_L1_M2l -.L6x8_L1_M2s: + bgt L(6x8_L1_M2l) +L(6x8_L1_M2s): SAVE2x1 sub counterI, counterI, #2 -.L6x8_L1_M1c: - cbz counterI, .L6x8_L999 +L(6x8_L1_M1c): + cbz counterI, L(6x8_L999) INIT1x1 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L1_M1s -.L6x8_L1_M1l: + cbz counterL, L(6x8_L1_M1s) +L(6x8_L1_M1l): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .L6x8_L1_M1l -.L6x8_L1_M1s: + bgt L(6x8_L1_M1l) +L(6x8_L1_M1s): SAVE1x1 -.L6x8_L999: +L(6x8_L999): mov x0, #0 ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dgemm_kernel_8x4.S b/kernel/arm64/dgemm_kernel_8x4.S index af3aa0217d..02fd16570b 100644 --- a/kernel/arm64/dgemm_kernel_8x4.S +++ b/kernel/arm64/dgemm_kernel_8x4.S @@ -885,12 +885,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Ldgemm_kernel_L2_BEGIN + ble L(dgemm_kernel_L2_BEGIN) /******************************************************************************/ .align 5 -.Ldgemm_kernel_L4_BEGIN: +L(dgemm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -900,21 +900,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L4_M8_BEGIN: +L(dgemm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Ldgemm_kernel_L4_M4_BEGIN + ble L(dgemm_kernel_L4_M4_BEGIN) .align 5 -.Ldgemm_kernel_L4_M8_20: +L(dgemm_kernel_L4_M8_20): mov pB, origPB asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Ldgemm_kernel_L4_M8_32 + blt L(dgemm_kernel_L4_M8_32) KERNEL8x4_I KERNEL8x4_M2 @@ -926,10 +926,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M2 subs counterL, counterL, #2 // subtract 2 - ble .Ldgemm_kernel_L4_M8_22a + ble L(dgemm_kernel_L4_M8_22a) .align 5 -.Ldgemm_kernel_L4_M8_22: +L(dgemm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 @@ -941,10 +941,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M8_22 + bgt L(dgemm_kernel_L4_M8_22) .align 5 -.Ldgemm_kernel_L4_M8_22a: +L(dgemm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_M2 @@ -955,13 +955,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Ldgemm_kernel_L4_M8_44 + b L(dgemm_kernel_L4_M8_44) .align 5 -.Ldgemm_kernel_L4_M8_32: +L(dgemm_kernel_L4_M8_32): tst counterL, #1 - ble .Ldgemm_kernel_L4_M8_40 + ble L(dgemm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_M2 @@ -972,46 +972,46 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Ldgemm_kernel_L4_M8_44 + b L(dgemm_kernel_L4_M8_44) -.Ldgemm_kernel_L4_M8_40: +L(dgemm_kernel_L4_M8_40): INIT8x4 -.Ldgemm_kernel_L4_M8_44: +L(dgemm_kernel_L4_M8_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L4_M8_100 + ble L(dgemm_kernel_L4_M8_100) .align 5 -.Ldgemm_kernel_L4_M8_46: +L(dgemm_kernel_L4_M8_46): KERNEL8x4_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L4_M8_46 + bne L(dgemm_kernel_L4_M8_46) -.Ldgemm_kernel_L4_M8_100: +L(dgemm_kernel_L4_M8_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVE8x4 -.Ldgemm_kernel_L4_M8_END: +L(dgemm_kernel_L4_M8_END): subs counterI, counterI, #1 - bne .Ldgemm_kernel_L4_M8_20 + bne L(dgemm_kernel_L4_M8_20) -.Ldgemm_kernel_L4_M4_BEGIN: +L(dgemm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Ldgemm_kernel_L4_END + ble L(dgemm_kernel_L4_END) tst counterI, #4 - ble .Ldgemm_kernel_L4_M2_BEGIN + ble L(dgemm_kernel_L4_M2_BEGIN) -.Ldgemm_kernel_L4_M4_20: +L(dgemm_kernel_L4_M4_20): INIT4x4 @@ -1019,10 +1019,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L4_M4_40 + ble L(dgemm_kernel_L4_M4_40) .align 5 -.Ldgemm_kernel_L4_M4_22: +L(dgemm_kernel_L4_M4_22): KERNEL4x4_SUB prfm PLDL1KEEP, [pB, #B_PRE_SIZE] @@ -1043,38 +1043,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prfm PLDL1KEEP, [pA, #A_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M4_22 + bgt L(dgemm_kernel_L4_M4_22) -.Ldgemm_kernel_L4_M4_40: +L(dgemm_kernel_L4_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L4_M4_100 + ble L(dgemm_kernel_L4_M4_100) -.Ldgemm_kernel_L4_M4_42: +L(dgemm_kernel_L4_M4_42): KERNEL4x4_SUB prfm PLDL1KEEP, [pB, #B_PRE_SIZE] prfm PLDL1KEEP, [pA, #A_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M4_42 + bgt L(dgemm_kernel_L4_M4_42) -.Ldgemm_kernel_L4_M4_100: +L(dgemm_kernel_L4_M4_100): SAVE4x4 -.Ldgemm_kernel_L4_M4_END: +L(dgemm_kernel_L4_M4_END): -.Ldgemm_kernel_L4_M2_BEGIN: +L(dgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L4_END + ble L(dgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L4_M1_BEGIN + ble L(dgemm_kernel_L4_M1_BEGIN) -.Ldgemm_kernel_L4_M2_20: +L(dgemm_kernel_L4_M2_20): INIT2x4 @@ -1082,10 +1082,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L4_M2_40 + ble L(dgemm_kernel_L4_M2_40) .align 5 -.Ldgemm_kernel_L4_M2_22: +L(dgemm_kernel_L4_M2_22): KERNEL2x4_SUB prfm PLDL1KEEP, [pB, #B_PRE_SIZE] @@ -1104,37 +1104,37 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M2_22 + bgt L(dgemm_kernel_L4_M2_22) -.Ldgemm_kernel_L4_M2_40: +L(dgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L4_M2_100 + ble L(dgemm_kernel_L4_M2_100) prfm PLDL1KEEP, [pA, #A_PRE_SIZE] prfm PLDL1KEEP, [pA, #A_PRE_SIZE+64] -.Ldgemm_kernel_L4_M2_42: +L(dgemm_kernel_L4_M2_42): KERNEL2x4_SUB prfm PLDL1KEEP, [pB, #B_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M2_42 + bgt L(dgemm_kernel_L4_M2_42) -.Ldgemm_kernel_L4_M2_100: +L(dgemm_kernel_L4_M2_100): SAVE2x4 -.Ldgemm_kernel_L4_M2_END: +L(dgemm_kernel_L4_M2_END): -.Ldgemm_kernel_L4_M1_BEGIN: +L(dgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L4_END + ble L(dgemm_kernel_L4_END) -.Ldgemm_kernel_L4_M1_20: +L(dgemm_kernel_L4_M1_20): INIT1x4 @@ -1142,10 +1142,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L4_M1_40 + ble L(dgemm_kernel_L4_M1_40) .align 5 -.Ldgemm_kernel_L4_M1_22: +L(dgemm_kernel_L4_M1_22): KERNEL1x4_SUB prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNEL1x4_SUB @@ -1163,46 +1163,46 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M1_22 + bgt L(dgemm_kernel_L4_M1_22) -.Ldgemm_kernel_L4_M1_40: +L(dgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L4_M1_100 + ble L(dgemm_kernel_L4_M1_100) prfm PLDL1KEEP, [pA, #A_PRE_SIZE] -.Ldgemm_kernel_L4_M1_42: +L(dgemm_kernel_L4_M1_42): KERNEL1x4_SUB prfm PLDL1KEEP, [pB, #B_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M1_42 + bgt L(dgemm_kernel_L4_M1_42) -.Ldgemm_kernel_L4_M1_100: +L(dgemm_kernel_L4_M1_100): SAVE1x4 -.Ldgemm_kernel_L4_END: +L(dgemm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 subs counterJ, counterJ , #1 // j-- - bgt .Ldgemm_kernel_L4_BEGIN + bgt L(dgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Ldgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(dgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Ldgemm_kernel_L999 // error, N was less than 4? + ble L(dgemm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Ldgemm_kernel_L1_BEGIN + ble L(dgemm_kernel_L1_BEGIN) mov pCRow0, pC add pCRow1, pCRow0, LDC @@ -1211,15 +1211,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Ldgemm_kernel_L2_M8_BEGIN: +L(dgemm_kernel_L2_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Ldgemm_kernel_L2_M4_BEGIN + ble L(dgemm_kernel_L2_M4_BEGIN) .align 5 -.Ldgemm_kernel_L2_M8_20: +L(dgemm_kernel_L2_M8_20): INIT8x2 @@ -1227,10 +1227,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldgemm_kernel_L2_M8_40 + ble L(dgemm_kernel_L2_M8_40) .align 5 -.Ldgemm_kernel_L2_M8_22: +L(dgemm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB prfm PLDL1KEEP, [pB, #B_PRE_SIZE] @@ -1244,41 +1244,41 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M8_22 + bgt L(dgemm_kernel_L2_M8_22) -.Ldgemm_kernel_L2_M8_40: +L(dgemm_kernel_L2_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M8_100 + ble L(dgemm_kernel_L2_M8_100) prfm PLDL1KEEP, [pB, #B_PRE_SIZE] prfm PLDL1KEEP, [pB, #B_PRE_SIZE+64] -.Ldgemm_kernel_L2_M8_42: +L(dgemm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M8_42 + bgt L(dgemm_kernel_L2_M8_42) -.Ldgemm_kernel_L2_M8_100: +L(dgemm_kernel_L2_M8_100): SAVE8x2 -.Ldgemm_kernel_L2_M8_END: +L(dgemm_kernel_L2_M8_END): subs counterI, counterI, #1 - bgt .Ldgemm_kernel_L2_M8_20 + bgt L(dgemm_kernel_L2_M8_20) -.Ldgemm_kernel_L2_M4_BEGIN: +L(dgemm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Ldgemm_kernel_L2_END + ble L(dgemm_kernel_L2_END) tst counterI, #4 // counterI = counterI / 2 - ble .Ldgemm_kernel_L2_M2_BEGIN + ble L(dgemm_kernel_L2_M2_BEGIN) -.Ldgemm_kernel_L2_M4_20: +L(dgemm_kernel_L2_M4_20): INIT4x2 @@ -1286,10 +1286,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldgemm_kernel_L2_M4_40 + ble L(dgemm_kernel_L2_M4_40) .align 5 -.Ldgemm_kernel_L2_M4_22: +L(dgemm_kernel_L2_M4_22): KERNEL4x2_SUB prfm PLDL1KEEP, [pA, #A_PRE_SIZE] KERNEL4x2_SUB @@ -1307,41 +1307,41 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M4_22 + bgt L(dgemm_kernel_L2_M4_22) -.Ldgemm_kernel_L2_M4_40: +L(dgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M4_100 + ble L(dgemm_kernel_L2_M4_100) prfm PLDL1KEEP, [pB, #B_PRE_SIZE] prfm PLDL1KEEP, [pB, #B_PRE_SIZE+64] -.Ldgemm_kernel_L2_M4_42: +L(dgemm_kernel_L2_M4_42): KERNEL4x2_SUB prfm PLDL1KEEP, [pA, #A_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M4_42 + bgt L(dgemm_kernel_L2_M4_42) -.Ldgemm_kernel_L2_M4_100: +L(dgemm_kernel_L2_M4_100): SAVE4x2 -.Ldgemm_kernel_L2_M4_END: +L(dgemm_kernel_L2_M4_END): -.Ldgemm_kernel_L2_M2_BEGIN: +L(dgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L2_END + ble L(dgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L2_M1_BEGIN + ble L(dgemm_kernel_L2_M1_BEGIN) -.Ldgemm_kernel_L2_M2_20: +L(dgemm_kernel_L2_M2_20): INIT2x2 @@ -1349,9 +1349,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldgemm_kernel_L2_M2_40 + ble L(dgemm_kernel_L2_M2_40) -.Ldgemm_kernel_L2_M2_22: +L(dgemm_kernel_L2_M2_22): KERNEL2x2_SUB prfm PLDL1KEEP, [pB, #B_PRE_SIZE] @@ -1368,37 +1368,37 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M2_22 + bgt L(dgemm_kernel_L2_M2_22) prfm PLDL1KEEP, [pA, #A_PRE_SIZE] prfm PLDL1KEEP, [pA, #A_PRE_SIZE+64] prfm PLDL1KEEP, [pB, #B_PRE_SIZE] prfm PLDL1KEEP, [pB, #B_PRE_SIZE+64] -.Ldgemm_kernel_L2_M2_40: +L(dgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M2_100 + ble L(dgemm_kernel_L2_M2_100) -.Ldgemm_kernel_L2_M2_42: +L(dgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M2_42 + bgt L(dgemm_kernel_L2_M2_42) -.Ldgemm_kernel_L2_M2_100: +L(dgemm_kernel_L2_M2_100): SAVE2x2 -.Ldgemm_kernel_L2_M2_END: +L(dgemm_kernel_L2_M2_END): -.Ldgemm_kernel_L2_M1_BEGIN: +L(dgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L2_END + ble L(dgemm_kernel_L2_END) -.Ldgemm_kernel_L2_M1_20: +L(dgemm_kernel_L2_M1_20): INIT1x2 @@ -1406,9 +1406,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Ldgemm_kernel_L2_M1_40 + ble L(dgemm_kernel_L2_M1_40) -.Ldgemm_kernel_L2_M1_22: +L(dgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB prfm PLDL1KEEP, [pB, #B_PRE_SIZE] @@ -1424,62 +1424,62 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M1_22 + bgt L(dgemm_kernel_L2_M1_22) prfm PLDL1KEEP, [pA, #A_PRE_SIZE] prfm PLDL1KEEP, [pB, #B_PRE_SIZE] prfm PLDL1KEEP, [pB, #B_PRE_SIZE+64] -.Ldgemm_kernel_L2_M1_40: +L(dgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M1_100 + ble L(dgemm_kernel_L2_M1_100) -.Ldgemm_kernel_L2_M1_42: +L(dgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M1_42 + bgt L(dgemm_kernel_L2_M1_42) -.Ldgemm_kernel_L2_M1_100: +L(dgemm_kernel_L2_M1_100): SAVE1x2 -.Ldgemm_kernel_L2_END: +L(dgemm_kernel_L2_END): add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 /******************************************************************************/ -.Ldgemm_kernel_L1_BEGIN: +L(dgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldgemm_kernel_L999 // done + ble L(dgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C add pC , pC , LDC // Update pC to point to next mov pA, origPA // pA = A -.Ldgemm_kernel_L1_M8_BEGIN: +L(dgemm_kernel_L1_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Ldgemm_kernel_L1_M4_BEGIN + ble L(dgemm_kernel_L1_M4_BEGIN) .align 5 -.Ldgemm_kernel_L1_M8_20: +L(dgemm_kernel_L1_M8_20): INIT8x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M8_40 + ble L(dgemm_kernel_L1_M8_40) .align 5 -.Ldgemm_kernel_L1_M8_22: +L(dgemm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -1493,51 +1493,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M8_22 + bgt L(dgemm_kernel_L1_M8_22) -.Ldgemm_kernel_L1_M8_40: +L(dgemm_kernel_L1_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M8_100 + ble L(dgemm_kernel_L1_M8_100) prfm PLDL1KEEP, [pB, #B_PRE_SIZE] -.Ldgemm_kernel_L1_M8_42: +L(dgemm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M8_42 + bgt L(dgemm_kernel_L1_M8_42) -.Ldgemm_kernel_L1_M8_100: +L(dgemm_kernel_L1_M8_100): SAVE8x1 -.Ldgemm_kernel_L1_M8_END: +L(dgemm_kernel_L1_M8_END): subs counterI, counterI, #1 - bgt .Ldgemm_kernel_L1_M8_20 + bgt L(dgemm_kernel_L1_M8_20) -.Ldgemm_kernel_L1_M4_BEGIN: +L(dgemm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Ldgemm_kernel_L1_END + ble L(dgemm_kernel_L1_END) tst counterI, #4 // counterI = counterI / 2 - ble .Ldgemm_kernel_L1_M2_BEGIN + ble L(dgemm_kernel_L1_M2_BEGIN) -.Ldgemm_kernel_L1_M4_20: +L(dgemm_kernel_L1_M4_20): INIT4x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M4_40 + ble L(dgemm_kernel_L1_M4_40) .align 5 -.Ldgemm_kernel_L1_M4_22: +L(dgemm_kernel_L1_M4_22): KERNEL4x1_SUB prfm PLDL1KEEP, [pA, #A_PRE_SIZE] KERNEL4x1_SUB @@ -1555,39 +1555,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M4_22 + bgt L(dgemm_kernel_L1_M4_22) -.Ldgemm_kernel_L1_M4_40: +L(dgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M4_100 + ble L(dgemm_kernel_L1_M4_100) prfm PLDL1KEEP, [pB, #B_PRE_SIZE] -.Ldgemm_kernel_L1_M4_42: +L(dgemm_kernel_L1_M4_42): KERNEL4x1_SUB prfm PLDL1KEEP, [pA, #A_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M4_42 + bgt L(dgemm_kernel_L1_M4_42) -.Ldgemm_kernel_L1_M4_100: +L(dgemm_kernel_L1_M4_100): SAVE4x1 -.Ldgemm_kernel_L1_M4_END: +L(dgemm_kernel_L1_M4_END): -.Ldgemm_kernel_L1_M2_BEGIN: +L(dgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L1_END + ble L(dgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L1_M1_BEGIN + ble L(dgemm_kernel_L1_M1_BEGIN) -.Ldgemm_kernel_L1_M2_20: +L(dgemm_kernel_L1_M2_20): INIT2x1 @@ -1595,9 +1595,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M2_40 + ble L(dgemm_kernel_L1_M2_40) -.Ldgemm_kernel_L1_M2_22: +L(dgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1614,36 +1614,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M2_22 + bgt L(dgemm_kernel_L1_M2_22) prfm PLDL1KEEP, [pA, #A_PRE_SIZE] prfm PLDL1KEEP, [pA, #A_PRE_SIZE+64] prfm PLDL1KEEP, [pB, #B_PRE_SIZE] -.Ldgemm_kernel_L1_M2_40: +L(dgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M2_100 + ble L(dgemm_kernel_L1_M2_100) -.Ldgemm_kernel_L1_M2_42: +L(dgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M2_42 + bgt L(dgemm_kernel_L1_M2_42) -.Ldgemm_kernel_L1_M2_100: +L(dgemm_kernel_L1_M2_100): SAVE2x1 -.Ldgemm_kernel_L1_M2_END: +L(dgemm_kernel_L1_M2_END): -.Ldgemm_kernel_L1_M1_BEGIN: +L(dgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L1_END + ble L(dgemm_kernel_L1_END) -.Ldgemm_kernel_L1_M1_20: +L(dgemm_kernel_L1_M1_20): INIT1x1 @@ -1651,10 +1651,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M1_40 + ble L(dgemm_kernel_L1_M1_40) -.Ldgemm_kernel_L1_M1_22: +L(dgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB prfm PLDL1KEEP, [pA, #A_PRE_SIZE] @@ -1668,32 +1668,32 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M1_22 + bgt L(dgemm_kernel_L1_M1_22) -.Ldgemm_kernel_L1_M1_40: +L(dgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M1_100 + ble L(dgemm_kernel_L1_M1_100) prfm PLDL1KEEP, [pA, #A_PRE_SIZE] prfm PLDL1KEEP, [pB, #B_PRE_SIZE] -.Ldgemm_kernel_L1_M1_42: +L(dgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M1_42 + bgt L(dgemm_kernel_L1_M1_42) -.Ldgemm_kernel_L1_M1_100: +L(dgemm_kernel_L1_M1_100): SAVE1x1 -.Ldgemm_kernel_L1_END: +L(dgemm_kernel_L1_END): -.Ldgemm_kernel_L999: +L(dgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dgemm_kernel_8x4_thunderx2t99.S b/kernel/arm64/dgemm_kernel_8x4_thunderx2t99.S index d1551ffeab..7525ee1078 100644 --- a/kernel/arm64/dgemm_kernel_8x4_thunderx2t99.S +++ b/kernel/arm64/dgemm_kernel_8x4_thunderx2t99.S @@ -958,12 +958,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Ldgemm_kernel_L2_BEGIN + ble L(dgemm_kernel_L2_BEGIN) /******************************************************************************/ .align 5 -.Ldgemm_kernel_L4_BEGIN: +L(dgemm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -973,21 +973,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L4_M8_BEGIN: +L(dgemm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Ldgemm_kernel_L4_M4_BEGIN + ble L(dgemm_kernel_L4_M4_BEGIN) .align 5 -.Ldgemm_kernel_L4_M8_20: +L(dgemm_kernel_L4_M8_20): mov pB, origPB asr counterL , origK, #7 // L = K / 128 cmp counterL , #2 // is there at least 4 to do? - blt .Ldgemm_kernel_L4_M8_32 + blt L(dgemm_kernel_L4_M8_32) KERNEL8x4_I KERNEL8x4_M2 @@ -999,18 +999,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1_M2_x1 subs counterL, counterL, #2 // subtract 2 - ble .Ldgemm_kernel_L4_M8_22a + ble L(dgemm_kernel_L4_M8_22a) .align 5 -.Ldgemm_kernel_L4_M8_22: +L(dgemm_kernel_L4_M8_22): KERNEL8x4_M1_M2_x64 subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M8_22 + bgt L(dgemm_kernel_L4_M8_22) .align 5 -.Ldgemm_kernel_L4_M8_22a: +L(dgemm_kernel_L4_M8_22a): KERNEL8x4_M1_M2_x32 KERNEL8x4_M1_M2_x16 @@ -1021,13 +1021,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Ldgemm_kernel_L4_M8_44 + b L(dgemm_kernel_L4_M8_44) .align 5 -.Ldgemm_kernel_L4_M8_32: +L(dgemm_kernel_L4_M8_32): tst counterL, #1 - ble .Ldgemm_kernel_L4_M8_40 + ble L(dgemm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_M2 @@ -1039,26 +1039,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Ldgemm_kernel_L4_M8_44 + b L(dgemm_kernel_L4_M8_44) -.Ldgemm_kernel_L4_M8_40: +L(dgemm_kernel_L4_M8_40): INIT8x4 -.Ldgemm_kernel_L4_M8_44: +L(dgemm_kernel_L4_M8_44): ands counterL , origK, #127 - ble .Ldgemm_kernel_L4_M8_100 + ble L(dgemm_kernel_L4_M8_100) .align 5 -.Ldgemm_kernel_L4_M8_46: +L(dgemm_kernel_L4_M8_46): KERNEL8x4_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L4_M8_46 + bne L(dgemm_kernel_L4_M8_46) -.Ldgemm_kernel_L4_M8_100: +L(dgemm_kernel_L4_M8_100): prfm PLDL2KEEP, [pCRow0, C_PRE_SIZE] prfm PLDL2KEEP, [pCRow1, C_PRE_SIZE] prfm PLDL2KEEP, [pCRow2, C_PRE_SIZE] @@ -1069,20 +1069,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SAVE8x4 -.Ldgemm_kernel_L4_M8_END: +L(dgemm_kernel_L4_M8_END): subs counterI, counterI, #1 - bne .Ldgemm_kernel_L4_M8_20 + bne L(dgemm_kernel_L4_M8_20) -.Ldgemm_kernel_L4_M4_BEGIN: +L(dgemm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Ldgemm_kernel_L4_END + ble L(dgemm_kernel_L4_END) tst counterI, #4 - ble .Ldgemm_kernel_L4_M2_BEGIN + ble L(dgemm_kernel_L4_M2_BEGIN) -.Ldgemm_kernel_L4_M4_20: +L(dgemm_kernel_L4_M4_20): INIT4x4 @@ -1090,10 +1090,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L4_M4_40 + ble L(dgemm_kernel_L4_M4_40) .align 5 -.Ldgemm_kernel_L4_M4_22: +L(dgemm_kernel_L4_M4_22): KERNEL4x4_SUB prfm PLDL1KEEP, [pB, B_PRE_SIZE] @@ -1114,38 +1114,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prfm PLDL1KEEP, [pA, A_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M4_22 + bgt L(dgemm_kernel_L4_M4_22) -.Ldgemm_kernel_L4_M4_40: +L(dgemm_kernel_L4_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L4_M4_100 + ble L(dgemm_kernel_L4_M4_100) -.Ldgemm_kernel_L4_M4_42: +L(dgemm_kernel_L4_M4_42): KERNEL4x4_SUB prfm PLDL1KEEP, [pB, B_PRE_SIZE] prfm PLDL1KEEP, [pA, A_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M4_42 + bgt L(dgemm_kernel_L4_M4_42) -.Ldgemm_kernel_L4_M4_100: +L(dgemm_kernel_L4_M4_100): SAVE4x4 -.Ldgemm_kernel_L4_M4_END: +L(dgemm_kernel_L4_M4_END): -.Ldgemm_kernel_L4_M2_BEGIN: +L(dgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L4_END + ble L(dgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L4_M1_BEGIN + ble L(dgemm_kernel_L4_M1_BEGIN) -.Ldgemm_kernel_L4_M2_20: +L(dgemm_kernel_L4_M2_20): INIT2x4 @@ -1153,10 +1153,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L4_M2_40 + ble L(dgemm_kernel_L4_M2_40) .align 5 -.Ldgemm_kernel_L4_M2_22: +L(dgemm_kernel_L4_M2_22): KERNEL2x4_SUB prfm PLDL1KEEP, [pB, B_PRE_SIZE] @@ -1175,37 +1175,37 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M2_22 + bgt L(dgemm_kernel_L4_M2_22) -.Ldgemm_kernel_L4_M2_40: +L(dgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L4_M2_100 + ble L(dgemm_kernel_L4_M2_100) prfm PLDL1KEEP, [pA, A_PRE_SIZE] prfm PLDL1KEEP, [pA, A_PRE_SIZE_64] -.Ldgemm_kernel_L4_M2_42: +L(dgemm_kernel_L4_M2_42): KERNEL2x4_SUB prfm PLDL1KEEP, [pB, B_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M2_42 + bgt L(dgemm_kernel_L4_M2_42) -.Ldgemm_kernel_L4_M2_100: +L(dgemm_kernel_L4_M2_100): SAVE2x4 -.Ldgemm_kernel_L4_M2_END: +L(dgemm_kernel_L4_M2_END): -.Ldgemm_kernel_L4_M1_BEGIN: +L(dgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L4_END + ble L(dgemm_kernel_L4_END) -.Ldgemm_kernel_L4_M1_20: +L(dgemm_kernel_L4_M1_20): INIT1x4 @@ -1213,10 +1213,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L4_M1_40 + ble L(dgemm_kernel_L4_M1_40) .align 5 -.Ldgemm_kernel_L4_M1_22: +L(dgemm_kernel_L4_M1_22): KERNEL1x4_SUB prfm PLDL1KEEP, [pB, B_PRE_SIZE] KERNEL1x4_SUB @@ -1234,46 +1234,46 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M1_22 + bgt L(dgemm_kernel_L4_M1_22) -.Ldgemm_kernel_L4_M1_40: +L(dgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L4_M1_100 + ble L(dgemm_kernel_L4_M1_100) prfm PLDL1KEEP, [pA, A_PRE_SIZE] -.Ldgemm_kernel_L4_M1_42: +L(dgemm_kernel_L4_M1_42): KERNEL1x4_SUB prfm PLDL1KEEP, [pB, B_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M1_42 + bgt L(dgemm_kernel_L4_M1_42) -.Ldgemm_kernel_L4_M1_100: +L(dgemm_kernel_L4_M1_100): SAVE1x4 -.Ldgemm_kernel_L4_END: +L(dgemm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 subs counterJ, counterJ , #1 // j-- - bgt .Ldgemm_kernel_L4_BEGIN + bgt L(dgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Ldgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(dgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Ldgemm_kernel_L999 // error, N was less than 4? + ble L(dgemm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Ldgemm_kernel_L1_BEGIN + ble L(dgemm_kernel_L1_BEGIN) mov pCRow0, pC add pCRow1, pCRow0, LDC @@ -1282,15 +1282,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Ldgemm_kernel_L2_M8_BEGIN: +L(dgemm_kernel_L2_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Ldgemm_kernel_L2_M4_BEGIN + ble L(dgemm_kernel_L2_M4_BEGIN) .align 5 -.Ldgemm_kernel_L2_M8_20: +L(dgemm_kernel_L2_M8_20): INIT8x2 @@ -1298,10 +1298,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldgemm_kernel_L2_M8_40 + ble L(dgemm_kernel_L2_M8_40) .align 5 -.Ldgemm_kernel_L2_M8_22: +L(dgemm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB prfm PLDL1KEEP, [pB, B_PRE_SIZE] @@ -1315,41 +1315,41 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M8_22 + bgt L(dgemm_kernel_L2_M8_22) -.Ldgemm_kernel_L2_M8_40: +L(dgemm_kernel_L2_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M8_100 + ble L(dgemm_kernel_L2_M8_100) prfm PLDL1KEEP, [pB, B_PRE_SIZE] prfm PLDL1KEEP, [pB, B_PRE_SIZE_64] -.Ldgemm_kernel_L2_M8_42: +L(dgemm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M8_42 + bgt L(dgemm_kernel_L2_M8_42) -.Ldgemm_kernel_L2_M8_100: +L(dgemm_kernel_L2_M8_100): SAVE8x2 -.Ldgemm_kernel_L2_M8_END: +L(dgemm_kernel_L2_M8_END): subs counterI, counterI, #1 - bgt .Ldgemm_kernel_L2_M8_20 + bgt L(dgemm_kernel_L2_M8_20) -.Ldgemm_kernel_L2_M4_BEGIN: +L(dgemm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Ldgemm_kernel_L2_END + ble L(dgemm_kernel_L2_END) tst counterI, #4 // counterI = counterI / 2 - ble .Ldgemm_kernel_L2_M2_BEGIN + ble L(dgemm_kernel_L2_M2_BEGIN) -.Ldgemm_kernel_L2_M4_20: +L(dgemm_kernel_L2_M4_20): INIT4x2 @@ -1357,10 +1357,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldgemm_kernel_L2_M4_40 + ble L(dgemm_kernel_L2_M4_40) .align 5 -.Ldgemm_kernel_L2_M4_22: +L(dgemm_kernel_L2_M4_22): KERNEL4x2_SUB prfm PLDL1KEEP, [pA, A_PRE_SIZE] KERNEL4x2_SUB @@ -1378,41 +1378,41 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M4_22 + bgt L(dgemm_kernel_L2_M4_22) -.Ldgemm_kernel_L2_M4_40: +L(dgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M4_100 + ble L(dgemm_kernel_L2_M4_100) prfm PLDL1KEEP, [pB, B_PRE_SIZE] prfm PLDL1KEEP, [pB, B_PRE_SIZE_64] -.Ldgemm_kernel_L2_M4_42: +L(dgemm_kernel_L2_M4_42): KERNEL4x2_SUB prfm PLDL1KEEP, [pA, A_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M4_42 + bgt L(dgemm_kernel_L2_M4_42) -.Ldgemm_kernel_L2_M4_100: +L(dgemm_kernel_L2_M4_100): SAVE4x2 -.Ldgemm_kernel_L2_M4_END: +L(dgemm_kernel_L2_M4_END): -.Ldgemm_kernel_L2_M2_BEGIN: +L(dgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L2_END + ble L(dgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L2_M1_BEGIN + ble L(dgemm_kernel_L2_M1_BEGIN) -.Ldgemm_kernel_L2_M2_20: +L(dgemm_kernel_L2_M2_20): INIT2x2 @@ -1420,9 +1420,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldgemm_kernel_L2_M2_40 + ble L(dgemm_kernel_L2_M2_40) -.Ldgemm_kernel_L2_M2_22: +L(dgemm_kernel_L2_M2_22): KERNEL2x2_SUB prfm PLDL1KEEP, [pB, B_PRE_SIZE] @@ -1439,37 +1439,37 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M2_22 + bgt L(dgemm_kernel_L2_M2_22) prfm PLDL1KEEP, [pA, A_PRE_SIZE] prfm PLDL1KEEP, [pA, A_PRE_SIZE_64] prfm PLDL1KEEP, [pB, B_PRE_SIZE] prfm PLDL1KEEP, [pB, B_PRE_SIZE_64] -.Ldgemm_kernel_L2_M2_40: +L(dgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M2_100 + ble L(dgemm_kernel_L2_M2_100) -.Ldgemm_kernel_L2_M2_42: +L(dgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M2_42 + bgt L(dgemm_kernel_L2_M2_42) -.Ldgemm_kernel_L2_M2_100: +L(dgemm_kernel_L2_M2_100): SAVE2x2 -.Ldgemm_kernel_L2_M2_END: +L(dgemm_kernel_L2_M2_END): -.Ldgemm_kernel_L2_M1_BEGIN: +L(dgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L2_END + ble L(dgemm_kernel_L2_END) -.Ldgemm_kernel_L2_M1_20: +L(dgemm_kernel_L2_M1_20): INIT1x2 @@ -1477,9 +1477,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Ldgemm_kernel_L2_M1_40 + ble L(dgemm_kernel_L2_M1_40) -.Ldgemm_kernel_L2_M1_22: +L(dgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB prfm PLDL1KEEP, [pB, B_PRE_SIZE] @@ -1495,62 +1495,62 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M1_22 + bgt L(dgemm_kernel_L2_M1_22) prfm PLDL1KEEP, [pA, A_PRE_SIZE] prfm PLDL1KEEP, [pB, B_PRE_SIZE] prfm PLDL1KEEP, [pB, B_PRE_SIZE_64] -.Ldgemm_kernel_L2_M1_40: +L(dgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M1_100 + ble L(dgemm_kernel_L2_M1_100) -.Ldgemm_kernel_L2_M1_42: +L(dgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M1_42 + bgt L(dgemm_kernel_L2_M1_42) -.Ldgemm_kernel_L2_M1_100: +L(dgemm_kernel_L2_M1_100): SAVE1x2 -.Ldgemm_kernel_L2_END: +L(dgemm_kernel_L2_END): add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 /******************************************************************************/ -.Ldgemm_kernel_L1_BEGIN: +L(dgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldgemm_kernel_L999 // done + ble L(dgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C add pC , pC , LDC // Update pC to point to next mov pA, origPA // pA = A -.Ldgemm_kernel_L1_M8_BEGIN: +L(dgemm_kernel_L1_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Ldgemm_kernel_L1_M4_BEGIN + ble L(dgemm_kernel_L1_M4_BEGIN) .align 5 -.Ldgemm_kernel_L1_M8_20: +L(dgemm_kernel_L1_M8_20): INIT8x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M8_40 + ble L(dgemm_kernel_L1_M8_40) .align 5 -.Ldgemm_kernel_L1_M8_22: +L(dgemm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -1564,51 +1564,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M8_22 + bgt L(dgemm_kernel_L1_M8_22) -.Ldgemm_kernel_L1_M8_40: +L(dgemm_kernel_L1_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M8_100 + ble L(dgemm_kernel_L1_M8_100) prfm PLDL1KEEP, [pB, B_PRE_SIZE] -.Ldgemm_kernel_L1_M8_42: +L(dgemm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M8_42 + bgt L(dgemm_kernel_L1_M8_42) -.Ldgemm_kernel_L1_M8_100: +L(dgemm_kernel_L1_M8_100): SAVE8x1 -.Ldgemm_kernel_L1_M8_END: +L(dgemm_kernel_L1_M8_END): subs counterI, counterI, #1 - bgt .Ldgemm_kernel_L1_M8_20 + bgt L(dgemm_kernel_L1_M8_20) -.Ldgemm_kernel_L1_M4_BEGIN: +L(dgemm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Ldgemm_kernel_L1_END + ble L(dgemm_kernel_L1_END) tst counterI, #4 // counterI = counterI / 2 - ble .Ldgemm_kernel_L1_M2_BEGIN + ble L(dgemm_kernel_L1_M2_BEGIN) -.Ldgemm_kernel_L1_M4_20: +L(dgemm_kernel_L1_M4_20): INIT4x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M4_40 + ble L(dgemm_kernel_L1_M4_40) .align 5 -.Ldgemm_kernel_L1_M4_22: +L(dgemm_kernel_L1_M4_22): KERNEL4x1_SUB prfm PLDL1KEEP, [pA, A_PRE_SIZE] KERNEL4x1_SUB @@ -1626,39 +1626,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M4_22 + bgt L(dgemm_kernel_L1_M4_22) -.Ldgemm_kernel_L1_M4_40: +L(dgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M4_100 + ble L(dgemm_kernel_L1_M4_100) prfm PLDL1KEEP, [pB, B_PRE_SIZE] -.Ldgemm_kernel_L1_M4_42: +L(dgemm_kernel_L1_M4_42): KERNEL4x1_SUB prfm PLDL1KEEP, [pA, A_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M4_42 + bgt L(dgemm_kernel_L1_M4_42) -.Ldgemm_kernel_L1_M4_100: +L(dgemm_kernel_L1_M4_100): SAVE4x1 -.Ldgemm_kernel_L1_M4_END: +L(dgemm_kernel_L1_M4_END): -.Ldgemm_kernel_L1_M2_BEGIN: +L(dgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L1_END + ble L(dgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L1_M1_BEGIN + ble L(dgemm_kernel_L1_M1_BEGIN) -.Ldgemm_kernel_L1_M2_20: +L(dgemm_kernel_L1_M2_20): INIT2x1 @@ -1666,9 +1666,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M2_40 + ble L(dgemm_kernel_L1_M2_40) -.Ldgemm_kernel_L1_M2_22: +L(dgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1685,36 +1685,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M2_22 + bgt L(dgemm_kernel_L1_M2_22) prfm PLDL1KEEP, [pA, A_PRE_SIZE] prfm PLDL1KEEP, [pA, A_PRE_SIZE_64] prfm PLDL1KEEP, [pB, B_PRE_SIZE] -.Ldgemm_kernel_L1_M2_40: +L(dgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M2_100 + ble L(dgemm_kernel_L1_M2_100) -.Ldgemm_kernel_L1_M2_42: +L(dgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M2_42 + bgt L(dgemm_kernel_L1_M2_42) -.Ldgemm_kernel_L1_M2_100: +L(dgemm_kernel_L1_M2_100): SAVE2x1 -.Ldgemm_kernel_L1_M2_END: +L(dgemm_kernel_L1_M2_END): -.Ldgemm_kernel_L1_M1_BEGIN: +L(dgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L1_END + ble L(dgemm_kernel_L1_END) -.Ldgemm_kernel_L1_M1_20: +L(dgemm_kernel_L1_M1_20): INIT1x1 @@ -1722,10 +1722,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M1_40 + ble L(dgemm_kernel_L1_M1_40) -.Ldgemm_kernel_L1_M1_22: +L(dgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB prfm PLDL1KEEP, [pA, A_PRE_SIZE] @@ -1739,32 +1739,32 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M1_22 + bgt L(dgemm_kernel_L1_M1_22) -.Ldgemm_kernel_L1_M1_40: +L(dgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M1_100 + ble L(dgemm_kernel_L1_M1_100) prfm PLDL1KEEP, [pA, A_PRE_SIZE] prfm PLDL1KEEP, [pB, B_PRE_SIZE] -.Ldgemm_kernel_L1_M1_42: +L(dgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M1_42 + bgt L(dgemm_kernel_L1_M1_42) -.Ldgemm_kernel_L1_M1_100: +L(dgemm_kernel_L1_M1_100): SAVE1x1 -.Ldgemm_kernel_L1_END: +L(dgemm_kernel_L1_END): -.Ldgemm_kernel_L999: +L(dgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dgemm_kernel_inter_sve_v2x8.S b/kernel/arm64/dgemm_kernel_inter_sve_v2x8.S index dfad8dbfea..20838d13ea 100644 --- a/kernel/arm64/dgemm_kernel_inter_sve_v2x8.S +++ b/kernel/arm64/dgemm_kernel_inter_sve_v2x8.S @@ -857,38 +857,38 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Ldgemm_kernel_L4_BEGIN + ble L(dgemm_kernel_L4_BEGIN) /******************************************************************************/ /* Repeat this as long as there are 8 left in N */ .align 5 -.Ldgemm_kernel_L8_BEGIN: +L(dgemm_kernel_L8_BEGIN): mov pCRow0, pC add pC, pC, LDC, lsl #3 // add 8 x LDC mov pA1, origPA // pA1 = start of A array -.Ldgemm_kernel_L8_Mv2_BEGIN: +L(dgemm_kernel_L8_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 // Check if M < 2*SVE_LEN - blt .Ldgemm_kernel_L8_Mv1_BEGIN + blt L(dgemm_kernel_L8_Mv1_BEGIN) mov counterI, origM /* Until we have at least 2*SVE_LEN iters left in M, we do them with V2*8 kernel */ .align 5 -.Ldgemm_kernel_L8_Mv2_20: +L(dgemm_kernel_L8_Mv2_20): mov pB, origPB INITv2x8 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Ldgemm_kernel_L8_Mv2_32 + blt L(dgemm_kernel_L8_Mv2_32) KERNELv2x8_I KERNELv2x8_M2 @@ -900,10 +900,10 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Ldgemm_kernel_L8_Mv2_22a + ble L(dgemm_kernel_L8_Mv2_22a) .align 5 -.Ldgemm_kernel_L8_Mv2_22: +L(dgemm_kernel_L8_Mv2_22): KERNELv2x8_M1 KERNELv2x8_M2 @@ -915,10 +915,10 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M2 subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L8_Mv2_22 + bgt L(dgemm_kernel_L8_Mv2_22) .align 5 -.Ldgemm_kernel_L8_Mv2_22a: +L(dgemm_kernel_L8_Mv2_22a): KERNELv2x8_M1 KERNELv2x8_M2 @@ -929,13 +929,13 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M1 KERNELv2x8_E - b .Ldgemm_kernel_L8_Mv2_44 + b L(dgemm_kernel_L8_Mv2_44) .align 5 -.Ldgemm_kernel_L8_Mv2_32: +L(dgemm_kernel_L8_Mv2_32): tst counterL, #1 - ble .Ldgemm_kernel_L8_Mv2_40 + ble L(dgemm_kernel_L8_Mv2_40) KERNELv2x8_I KERNELv2x8_M2 @@ -947,54 +947,54 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_E - b .Ldgemm_kernel_L8_Mv2_44 + b L(dgemm_kernel_L8_Mv2_44) -.Ldgemm_kernel_L8_Mv2_40: +L(dgemm_kernel_L8_Mv2_40): INITv2x8 -.Ldgemm_kernel_L8_Mv2_44: +L(dgemm_kernel_L8_Mv2_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L8_Mv2_100 + ble L(dgemm_kernel_L8_Mv2_100) .align 5 -.Ldgemm_kernel_L8_Mv2_46: +L(dgemm_kernel_L8_Mv2_46): KERNELv2x8_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L8_Mv2_46 + bne L(dgemm_kernel_L8_Mv2_46) -.Ldgemm_kernel_L8_Mv2_100: +L(dgemm_kernel_L8_Mv2_100): SAVEv2x8 -.Ldgemm_kernel_L8_Mv2_END: +L(dgemm_kernel_L8_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Ldgemm_kernel_L8_Mv2_20 + bge L(dgemm_kernel_L8_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Ldgemm_kernel_L8_END + beq L(dgemm_kernel_L8_END) ////////////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x8 kernel. -.Ldgemm_kernel_L8_Mv1_BEGIN: +L(dgemm_kernel_L8_Mv1_BEGIN): whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension .align 5 -.Ldgemm_kernel_L8_Mv1_20: +L(dgemm_kernel_L8_Mv1_20): mov pB, origPB INITv1x8 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Ldgemm_kernel_L8_Mv1_32 + blt L(dgemm_kernel_L8_Mv1_32) KERNELv1x8_I KERNELv1x8_M2 @@ -1006,10 +1006,10 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Ldgemm_kernel_L8_Mv1_22a + ble L(dgemm_kernel_L8_Mv1_22a) .align 5 -.Ldgemm_kernel_L8_Mv1_22: +L(dgemm_kernel_L8_Mv1_22): KERNELv1x8_M1 KERNELv1x8_M2 @@ -1021,10 +1021,10 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M2 subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L8_Mv1_22 + bgt L(dgemm_kernel_L8_Mv1_22) .align 5 -.Ldgemm_kernel_L8_Mv1_22a: +L(dgemm_kernel_L8_Mv1_22a): KERNELv1x8_M1 KERNELv1x8_M2 @@ -1035,13 +1035,13 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M1 KERNELv1x8_E - b .Ldgemm_kernel_L8_Mv1_44 + b L(dgemm_kernel_L8_Mv1_44) .align 5 -.Ldgemm_kernel_L8_Mv1_32: +L(dgemm_kernel_L8_Mv1_32): tst counterL, #1 - ble .Ldgemm_kernel_L8_Mv1_40 + ble L(dgemm_kernel_L8_Mv1_40) KERNELv1x8_I KERNELv1x8_M2 @@ -1053,53 +1053,53 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_E - b .Ldgemm_kernel_L8_Mv1_44 + b L(dgemm_kernel_L8_Mv1_44) -.Ldgemm_kernel_L8_Mv1_40: +L(dgemm_kernel_L8_Mv1_40): INITv1x8 -.Ldgemm_kernel_L8_Mv1_44: +L(dgemm_kernel_L8_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L8_Mv1_100 + ble L(dgemm_kernel_L8_Mv1_100) .align 5 -.Ldgemm_kernel_L8_Mv1_46: +L(dgemm_kernel_L8_Mv1_46): KERNELv1x8_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L8_Mv1_46 + bne L(dgemm_kernel_L8_Mv1_46) -.Ldgemm_kernel_L8_Mv1_100: +L(dgemm_kernel_L8_Mv1_100): SAVEv1x8 -.Ldgemm_kernel_L8_Mv1_END: +L(dgemm_kernel_L8_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension - b.any .Ldgemm_kernel_L8_Mv1_20 + b.any L(dgemm_kernel_L8_Mv1_20) -.Ldgemm_kernel_L8_END: +L(dgemm_kernel_L8_END): lsl temp, origK, #6 add origPB, origPB, temp // B = B + K * 8 * 8 subs counterJ, counterJ , #1 // j-- - bgt .Ldgemm_kernel_L8_BEGIN + bgt L(dgemm_kernel_L8_BEGIN) /******************************************************************************/ /* Repeat the same thing if 4 left in N */ .align 5 -.Ldgemm_kernel_L4_BEGIN: +L(dgemm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #4 - ble .Ldgemm_kernel_L2_BEGIN + ble L(dgemm_kernel_L2_BEGIN) mov pCRow0, pC @@ -1108,26 +1108,26 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Ldgemm_kernel_L4_Mv2_BEGIN: +L(dgemm_kernel_L4_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Ldgemm_kernel_L4_Mv1_BEGIN + blt L(dgemm_kernel_L4_Mv1_BEGIN) mov counterI, origM .align 5 -.Ldgemm_kernel_L4_Mv2_20: +L(dgemm_kernel_L4_Mv2_20): mov pB, origPB INITv2x4 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L4_Mv2_44 + ble L(dgemm_kernel_L4_Mv2_44) .align 5 -.Ldgemm_kernel_L4_Mv2_22: +L(dgemm_kernel_L4_Mv2_22): KERNELv2x4_SUB KERNELv2x4_SUB @@ -1139,53 +1139,53 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_Mv2_22 + bgt L(dgemm_kernel_L4_Mv2_22) -.Ldgemm_kernel_L4_Mv2_44: +L(dgemm_kernel_L4_Mv2_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L4_Mv2_100 + ble L(dgemm_kernel_L4_Mv2_100) .align 5 -.Ldgemm_kernel_L4_Mv2_46: +L(dgemm_kernel_L4_Mv2_46): KERNELv2x4_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L4_Mv2_46 + bne L(dgemm_kernel_L4_Mv2_46) -.Ldgemm_kernel_L4_Mv2_100: +L(dgemm_kernel_L4_Mv2_100): SAVEv2x4 -.Ldgemm_kernel_L4_Mv2_END: +L(dgemm_kernel_L4_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Ldgemm_kernel_L4_Mv2_20 + bge L(dgemm_kernel_L4_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Ldgemm_kernel_L4_END + beq L(dgemm_kernel_L4_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x4 kernel. -.Ldgemm_kernel_L4_Mv1_BEGIN: +L(dgemm_kernel_L4_Mv1_BEGIN): whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension .align 5 -.Ldgemm_kernel_L4_Mv1_20: +L(dgemm_kernel_L4_Mv1_20): mov pB, origPB INITv1x4 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L4_Mv1_44 + ble L(dgemm_kernel_L4_Mv1_44) .align 5 -.Ldgemm_kernel_L4_Mv1_22: +L(dgemm_kernel_L4_Mv1_22): KERNELv1x4_SUB KERNELv1x4_SUB @@ -1197,34 +1197,34 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_Mv1_22 + bgt L(dgemm_kernel_L4_Mv1_22) -.Ldgemm_kernel_L4_Mv1_44: +L(dgemm_kernel_L4_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L4_Mv1_100 + ble L(dgemm_kernel_L4_Mv1_100) .align 5 -.Ldgemm_kernel_L4_Mv1_46: +L(dgemm_kernel_L4_Mv1_46): KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L4_Mv1_46 + bne L(dgemm_kernel_L4_Mv1_46) -.Ldgemm_kernel_L4_Mv1_100: +L(dgemm_kernel_L4_Mv1_100): SAVEv1x4 -.Ldgemm_kernel_L4_Mv1_END: +L(dgemm_kernel_L4_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldgemm_kernel_L4_Mv1_20 + b.any L(dgemm_kernel_L4_Mv1_20) -.Ldgemm_kernel_L4_END: +L(dgemm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 @@ -1232,11 +1232,11 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ /* Repeat the same thing if 2 left in N */ .align 5 -.Ldgemm_kernel_L2_BEGIN: +L(dgemm_kernel_L2_BEGIN): mov counterJ , origN tst counterJ , #2 - ble .Ldgemm_kernel_L1_BEGIN + ble L(dgemm_kernel_L1_BEGIN) mov pCRow0, pC @@ -1244,26 +1244,26 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Ldgemm_kernel_L2_Mv2_BEGIN: +L(dgemm_kernel_L2_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Ldgemm_kernel_L2_Mv1_BEGIN + blt L(dgemm_kernel_L2_Mv1_BEGIN) mov counterI, origM .align 5 -.Ldgemm_kernel_L2_Mv2_20: +L(dgemm_kernel_L2_Mv2_20): mov pB, origPB INITv2x2 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L2_Mv2_44 + ble L(dgemm_kernel_L2_Mv2_44) .align 5 -.Ldgemm_kernel_L2_Mv2_22: +L(dgemm_kernel_L2_Mv2_22): KERNELv2x2_SUB KERNELv2x2_SUB @@ -1275,54 +1275,54 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_Mv2_22 + bgt L(dgemm_kernel_L2_Mv2_22) -.Ldgemm_kernel_L2_Mv2_44: +L(dgemm_kernel_L2_Mv2_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L2_Mv2_100 + ble L(dgemm_kernel_L2_Mv2_100) .align 5 -.Ldgemm_kernel_L2_Mv2_46: +L(dgemm_kernel_L2_Mv2_46): KERNELv2x2_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L2_Mv2_46 + bne L(dgemm_kernel_L2_Mv2_46) -.Ldgemm_kernel_L2_Mv2_100: +L(dgemm_kernel_L2_Mv2_100): SAVEv2x2 -.Ldgemm_kernel_L2_Mv2_END: +L(dgemm_kernel_L2_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Ldgemm_kernel_L2_Mv2_20 + bge L(dgemm_kernel_L2_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Ldgemm_kernel_L2_END + beq L(dgemm_kernel_L2_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x2 kernel. -.Ldgemm_kernel_L2_Mv1_BEGIN: +L(dgemm_kernel_L2_Mv1_BEGIN): whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d .align 5 -.Ldgemm_kernel_L2_Mv1_20: +L(dgemm_kernel_L2_Mv1_20): mov pB, origPB INITv1x2 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L2_Mv1_44 + ble L(dgemm_kernel_L2_Mv1_44) .align 5 -.Ldgemm_kernel_L2_Mv1_22: +L(dgemm_kernel_L2_Mv1_22): KERNELv1x2_SUB KERNELv1x2_SUB @@ -1334,45 +1334,45 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_Mv1_22 + bgt L(dgemm_kernel_L2_Mv1_22) -.Ldgemm_kernel_L2_Mv1_44: +L(dgemm_kernel_L2_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L2_Mv1_100 + ble L(dgemm_kernel_L2_Mv1_100) .align 5 -.Ldgemm_kernel_L2_Mv1_46: +L(dgemm_kernel_L2_Mv1_46): KERNELv1x2_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L2_Mv1_46 + bne L(dgemm_kernel_L2_Mv1_46) -.Ldgemm_kernel_L2_Mv1_100: +L(dgemm_kernel_L2_Mv1_100): SAVEv1x2 -.Ldgemm_kernel_L2_Mv1_END: +L(dgemm_kernel_L2_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldgemm_kernel_L2_Mv1_20 + b.any L(dgemm_kernel_L2_Mv1_20) -.Ldgemm_kernel_L2_END: +L(dgemm_kernel_L2_END): add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 /******************************************************************************/ /* Repeat the same thing if 1 left in N */ .align 5 -.Ldgemm_kernel_L1_BEGIN: +L(dgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldgemm_kernel_L999 // done + ble L(dgemm_kernel_L999) // done mov pCRow0, pC @@ -1380,26 +1380,26 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Ldgemm_kernel_L1_Mv2_BEGIN: +L(dgemm_kernel_L1_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Ldgemm_kernel_L1_Mv1_BEGIN + blt L(dgemm_kernel_L1_Mv1_BEGIN) mov counterI, origM .align 5 -.Ldgemm_kernel_L1_Mv2_20: +L(dgemm_kernel_L1_Mv2_20): mov pB, origPB INITv2x1 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Ldgemm_kernel_L1_Mv2_44 + ble L(dgemm_kernel_L1_Mv2_44) .align 5 -.Ldgemm_kernel_L1_Mv2_22: +L(dgemm_kernel_L1_Mv2_22): KERNELv2x1_SUB KERNELv2x1_SUB @@ -1411,54 +1411,54 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv2_22 + bgt L(dgemm_kernel_L1_Mv2_22) -.Ldgemm_kernel_L1_Mv2_44: +L(dgemm_kernel_L1_Mv2_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L1_Mv2_100 + ble L(dgemm_kernel_L1_Mv2_100) .align 5 -.Ldgemm_kernel_L1_Mv2_46: +L(dgemm_kernel_L1_Mv2_46): KERNELv2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv2_46 + bgt L(dgemm_kernel_L1_Mv2_46) -.Ldgemm_kernel_L1_Mv2_100: +L(dgemm_kernel_L1_Mv2_100): SAVEv2x1 -.Ldgemm_kernel_L1_Mv2_END: +L(dgemm_kernel_L1_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Ldgemm_kernel_L1_Mv2_20 + bge L(dgemm_kernel_L1_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Ldgemm_kernel_L1_END + beq L(dgemm_kernel_L1_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x1 kernel. -.Ldgemm_kernel_L1_Mv1_BEGIN: +L(dgemm_kernel_L1_Mv1_BEGIN): whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d .align 5 -.Ldgemm_kernel_L1_Mv1_20: +L(dgemm_kernel_L1_Mv1_20): mov pB, origPB INITv1x1 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Ldgemm_kernel_L1_Mv1_44 + ble L(dgemm_kernel_L1_Mv1_44) .align 5 -.Ldgemm_kernel_L1_Mv1_22: +L(dgemm_kernel_L1_Mv1_22): KERNELv1x1_SUB KERNELv1x1_SUB @@ -1470,38 +1470,38 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv1_22 + bgt L(dgemm_kernel_L1_Mv1_22) -.Ldgemm_kernel_L1_Mv1_44: +L(dgemm_kernel_L1_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L1_Mv1_100 + ble L(dgemm_kernel_L1_Mv1_100) .align 5 -.Ldgemm_kernel_L1_Mv1_46: +L(dgemm_kernel_L1_Mv1_46): KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv1_46 + bgt L(dgemm_kernel_L1_Mv1_46) -.Ldgemm_kernel_L1_Mv1_100: +L(dgemm_kernel_L1_Mv1_100): SAVEv1x1 -.Ldgemm_kernel_L1_Mv1_END: +L(dgemm_kernel_L1_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldgemm_kernel_L1_Mv1_20 + b.any L(dgemm_kernel_L1_Mv1_20) -.Ldgemm_kernel_L1_END: +L(dgemm_kernel_L1_END): /******************************************************************************/ -.Ldgemm_kernel_L999: +L(dgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dgemm_kernel_sve_v1x8.S b/kernel/arm64/dgemm_kernel_sve_v1x8.S index bbbd0fd953..fd7b33b4d8 100644 --- a/kernel/arm64/dgemm_kernel_sve_v1x8.S +++ b/kernel/arm64/dgemm_kernel_sve_v1x8.S @@ -479,20 +479,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Ldgemm_kernel_L4_BEGIN + ble L(dgemm_kernel_L4_BEGIN) /******************************************************************************/ /* Repeat this as long as there are 8 left in N */ .align 5 -.Ldgemm_kernel_L8_BEGIN: +L(dgemm_kernel_L8_BEGIN): mov pCRow0, pC add pC, pC, LDC, lsl #3 // add 8 x LDC mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L8_Mv1_BEGIN: +L(dgemm_kernel_L8_Mv1_BEGIN): /* Loop over M is done in an SVE fashion. This has the benefit of the last M%SVE_LEN iterations being done in a single sweep */ mov counterI, #0 @@ -500,14 +500,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension .align 5 -.Ldgemm_kernel_L8_Mv1_20: +L(dgemm_kernel_L8_Mv1_20): mov pB, origPB INITv1x8 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Ldgemm_kernel_L8_Mv1_32 + blt L(dgemm_kernel_L8_Mv1_32) KERNELv1x8_I KERNELv1x8_M2 @@ -519,10 +519,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Ldgemm_kernel_L8_Mv1_22a + ble L(dgemm_kernel_L8_Mv1_22a) .align 5 -.Ldgemm_kernel_L8_Mv1_22: +L(dgemm_kernel_L8_Mv1_22): KERNELv1x8_M1 KERNELv1x8_M2 @@ -534,10 +534,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M2 subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L8_Mv1_22 + bgt L(dgemm_kernel_L8_Mv1_22) .align 5 -.Ldgemm_kernel_L8_Mv1_22a: +L(dgemm_kernel_L8_Mv1_22a): KERNELv1x8_M1 KERNELv1x8_M2 @@ -548,13 +548,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M1 KERNELv1x8_E - b .Ldgemm_kernel_L8_Mv1_44 + b L(dgemm_kernel_L8_Mv1_44) .align 5 -.Ldgemm_kernel_L8_Mv1_32: +L(dgemm_kernel_L8_Mv1_32): tst counterL, #1 - ble .Ldgemm_kernel_L8_Mv1_40 + ble L(dgemm_kernel_L8_Mv1_40) KERNELv1x8_I KERNELv1x8_M2 @@ -566,56 +566,56 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_E - b .Ldgemm_kernel_L8_Mv1_44 + b L(dgemm_kernel_L8_Mv1_44) -.Ldgemm_kernel_L8_Mv1_40: +L(dgemm_kernel_L8_Mv1_40): INITv1x8 -.Ldgemm_kernel_L8_Mv1_44: +L(dgemm_kernel_L8_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L8_Mv1_100 + ble L(dgemm_kernel_L8_Mv1_100) .align 5 -.Ldgemm_kernel_L8_Mv1_46: +L(dgemm_kernel_L8_Mv1_46): KERNELv1x8_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L8_Mv1_46 + bne L(dgemm_kernel_L8_Mv1_46) -.Ldgemm_kernel_L8_Mv1_100: +L(dgemm_kernel_L8_Mv1_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVEv1x8 -.Ldgemm_kernel_L8_Mv1_END: +L(dgemm_kernel_L8_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension - b.any .Ldgemm_kernel_L8_Mv1_20 + b.any L(dgemm_kernel_L8_Mv1_20) -.Ldgemm_kernel_L8_END: +L(dgemm_kernel_L8_END): lsl temp, origK, #6 add origPB, origPB, temp // B = B + K * 8 * 8 subs counterJ, counterJ , #1 // j-- - bgt .Ldgemm_kernel_L8_BEGIN + bgt L(dgemm_kernel_L8_BEGIN) /******************************************************************************/ /* Repeat the same thing if 4 left in N */ .align 5 -.Ldgemm_kernel_L4_BEGIN: +L(dgemm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #4 - ble .Ldgemm_kernel_L2_BEGIN + ble L(dgemm_kernel_L2_BEGIN) mov pCRow0, pC @@ -624,24 +624,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L4_Mv1_BEGIN: +L(dgemm_kernel_L4_Mv1_BEGIN): mov counterI, #0 whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d .align 5 -.Ldgemm_kernel_L4_Mv1_20: +L(dgemm_kernel_L4_Mv1_20): mov pB, origPB INITv1x4 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L4_Mv1_44 + ble L(dgemm_kernel_L4_Mv1_44) .align 5 -.Ldgemm_kernel_L4_Mv1_22: +L(dgemm_kernel_L4_Mv1_22): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x4_SUB @@ -657,38 +657,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_Mv1_22 + bgt L(dgemm_kernel_L4_Mv1_22) -.Ldgemm_kernel_L4_Mv1_44: +L(dgemm_kernel_L4_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L4_Mv1_100 + ble L(dgemm_kernel_L4_Mv1_100) .align 5 -.Ldgemm_kernel_L4_Mv1_46: +L(dgemm_kernel_L4_Mv1_46): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L4_Mv1_46 + bne L(dgemm_kernel_L4_Mv1_46) -.Ldgemm_kernel_L4_Mv1_100: +L(dgemm_kernel_L4_Mv1_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVEv1x4 -.Ldgemm_kernel_L4_Mv1_END: +L(dgemm_kernel_L4_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldgemm_kernel_L4_Mv1_20 + b.any L(dgemm_kernel_L4_Mv1_20) -.Ldgemm_kernel_L4_END: +L(dgemm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 @@ -696,11 +696,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /* Repeat the same thing if 2 left in N */ .align 5 -.Ldgemm_kernel_L2_BEGIN: +L(dgemm_kernel_L2_BEGIN): mov counterJ , origN tst counterJ , #2 - ble .Ldgemm_kernel_L1_BEGIN + ble L(dgemm_kernel_L1_BEGIN) mov pCRow0, pC @@ -708,24 +708,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L2_Mv1_BEGIN: +L(dgemm_kernel_L2_Mv1_BEGIN): mov counterI, #0 whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d .align 5 -.Ldgemm_kernel_L2_Mv1_20: +L(dgemm_kernel_L2_Mv1_20): mov pB, origPB INITv1x2 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L2_Mv1_44 + ble L(dgemm_kernel_L2_Mv1_44) .align 5 -.Ldgemm_kernel_L2_Mv1_22: +L(dgemm_kernel_L2_Mv1_22): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x2_SUB @@ -739,49 +739,49 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_Mv1_22 + bgt L(dgemm_kernel_L2_Mv1_22) -.Ldgemm_kernel_L2_Mv1_44: +L(dgemm_kernel_L2_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L2_Mv1_100 + ble L(dgemm_kernel_L2_Mv1_100) .align 5 -.Ldgemm_kernel_L2_Mv1_46: +L(dgemm_kernel_L2_Mv1_46): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x2_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L2_Mv1_46 + bne L(dgemm_kernel_L2_Mv1_46) -.Ldgemm_kernel_L2_Mv1_100: +L(dgemm_kernel_L2_Mv1_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVEv1x2 -.Ldgemm_kernel_L2_Mv1_END: +L(dgemm_kernel_L2_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldgemm_kernel_L2_Mv1_20 + b.any L(dgemm_kernel_L2_Mv1_20) -.Ldgemm_kernel_L2_END: +L(dgemm_kernel_L2_END): add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 /******************************************************************************/ /* Repeat the same thing if 1 left in N */ .align 5 -.Ldgemm_kernel_L1_BEGIN: +L(dgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldgemm_kernel_L999 // done + ble L(dgemm_kernel_L999) // done mov pCRow0, pC @@ -789,24 +789,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L1_Mv1_BEGIN: +L(dgemm_kernel_L1_Mv1_BEGIN): mov counterI, #0 whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d .align 5 -.Ldgemm_kernel_L1_Mv1_20: +L(dgemm_kernel_L1_Mv1_20): mov pB, origPB INITv1x1 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Ldgemm_kernel_L1_Mv1_44 + ble L(dgemm_kernel_L1_Mv1_44) .align 5 -.Ldgemm_kernel_L1_Mv1_22: +L(dgemm_kernel_L1_Mv1_22): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x1_SUB @@ -819,42 +819,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv1_22 + bgt L(dgemm_kernel_L1_Mv1_22) -.Ldgemm_kernel_L1_Mv1_44: +L(dgemm_kernel_L1_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L1_Mv1_100 + ble L(dgemm_kernel_L1_Mv1_100) .align 5 -.Ldgemm_kernel_L1_Mv1_46: +L(dgemm_kernel_L1_Mv1_46): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv1_46 + bgt L(dgemm_kernel_L1_Mv1_46) -.Ldgemm_kernel_L1_Mv1_100: +L(dgemm_kernel_L1_Mv1_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVEv1x1 -.Ldgemm_kernel_L1_Mv1_END: +L(dgemm_kernel_L1_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldgemm_kernel_L1_Mv1_20 + b.any L(dgemm_kernel_L1_Mv1_20) -.Ldgemm_kernel_L1_END: +L(dgemm_kernel_L1_END): /******************************************************************************/ -.Ldgemm_kernel_L999: +L(dgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dgemm_kernel_sve_v2x8.S b/kernel/arm64/dgemm_kernel_sve_v2x8.S index d978a3315b..7296a5c578 100644 --- a/kernel/arm64/dgemm_kernel_sve_v2x8.S +++ b/kernel/arm64/dgemm_kernel_sve_v2x8.S @@ -882,24 +882,24 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Ldgemm_kernel_L4_BEGIN + ble L(dgemm_kernel_L4_BEGIN) /******************************************************************************/ /* Repeat this as long as there are 8 left in N */ .align 5 -.Ldgemm_kernel_L8_BEGIN: +L(dgemm_kernel_L8_BEGIN): mov pCRow0, pC add pC, pC, LDC, lsl #3 // add 8 x LDC mov pA1, origPA // pA1 = start of A array -.Ldgemm_kernel_L8_Mv2_BEGIN: +L(dgemm_kernel_L8_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 // Check if M < 2*SVE_LEN - blt .Ldgemm_kernel_L8_Mv1_BEGIN + blt L(dgemm_kernel_L8_Mv1_BEGIN) mov counterI, origM @@ -908,14 +908,14 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ add pA2, pA1, temp, lsl #3 // pA1 = start of A array .align 5 -.Ldgemm_kernel_L8_Mv2_20: +L(dgemm_kernel_L8_Mv2_20): mov pB, origPB INITv2x8 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Ldgemm_kernel_L8_Mv2_32 + blt L(dgemm_kernel_L8_Mv2_32) KERNELv2x8_I KERNELv2x8_M2 @@ -927,10 +927,10 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Ldgemm_kernel_L8_Mv2_22a + ble L(dgemm_kernel_L8_Mv2_22a) .align 5 -.Ldgemm_kernel_L8_Mv2_22: +L(dgemm_kernel_L8_Mv2_22): KERNELv2x8_M1 KERNELv2x8_M2 @@ -942,10 +942,10 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M2 subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L8_Mv2_22 + bgt L(dgemm_kernel_L8_Mv2_22) .align 5 -.Ldgemm_kernel_L8_Mv2_22a: +L(dgemm_kernel_L8_Mv2_22a): KERNELv2x8_M1 KERNELv2x8_M2 @@ -956,13 +956,13 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M1 KERNELv2x8_E - b .Ldgemm_kernel_L8_Mv2_44 + b L(dgemm_kernel_L8_Mv2_44) .align 5 -.Ldgemm_kernel_L8_Mv2_32: +L(dgemm_kernel_L8_Mv2_32): tst counterL, #1 - ble .Ldgemm_kernel_L8_Mv2_40 + ble L(dgemm_kernel_L8_Mv2_40) KERNELv2x8_I KERNELv2x8_M2 @@ -974,57 +974,57 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_E - b .Ldgemm_kernel_L8_Mv2_44 + b L(dgemm_kernel_L8_Mv2_44) -.Ldgemm_kernel_L8_Mv2_40: +L(dgemm_kernel_L8_Mv2_40): INITv2x8 -.Ldgemm_kernel_L8_Mv2_44: +L(dgemm_kernel_L8_Mv2_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L8_Mv2_100 + ble L(dgemm_kernel_L8_Mv2_100) .align 5 -.Ldgemm_kernel_L8_Mv2_46: +L(dgemm_kernel_L8_Mv2_46): KERNELv2x8_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L8_Mv2_46 + bne L(dgemm_kernel_L8_Mv2_46) -.Ldgemm_kernel_L8_Mv2_100: +L(dgemm_kernel_L8_Mv2_100): SAVEv2x8 mov pA1, pA2 // pA1 = pA2 mul temp, vec_len, origK // generate address of pA2 add pA2, pA1, temp, lsl #3 // -.Ldgemm_kernel_L8_Mv2_END: +L(dgemm_kernel_L8_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Ldgemm_kernel_L8_Mv2_20 + bge L(dgemm_kernel_L8_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Ldgemm_kernel_L8_END + beq L(dgemm_kernel_L8_END) ////////////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x8 kernel. -.Ldgemm_kernel_L8_Mv1_BEGIN: +L(dgemm_kernel_L8_Mv1_BEGIN): whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension .align 5 -.Ldgemm_kernel_L8_Mv1_20: +L(dgemm_kernel_L8_Mv1_20): mov pB, origPB INITv1x8 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Ldgemm_kernel_L8_Mv1_32 + blt L(dgemm_kernel_L8_Mv1_32) KERNELv1x8_I KERNELv1x8_M2 @@ -1036,10 +1036,10 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Ldgemm_kernel_L8_Mv1_22a + ble L(dgemm_kernel_L8_Mv1_22a) .align 5 -.Ldgemm_kernel_L8_Mv1_22: +L(dgemm_kernel_L8_Mv1_22): KERNELv1x8_M1 KERNELv1x8_M2 @@ -1051,10 +1051,10 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M2 subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L8_Mv1_22 + bgt L(dgemm_kernel_L8_Mv1_22) .align 5 -.Ldgemm_kernel_L8_Mv1_22a: +L(dgemm_kernel_L8_Mv1_22a): KERNELv1x8_M1 KERNELv1x8_M2 @@ -1065,13 +1065,13 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M1 KERNELv1x8_E - b .Ldgemm_kernel_L8_Mv1_44 + b L(dgemm_kernel_L8_Mv1_44) .align 5 -.Ldgemm_kernel_L8_Mv1_32: +L(dgemm_kernel_L8_Mv1_32): tst counterL, #1 - ble .Ldgemm_kernel_L8_Mv1_40 + ble L(dgemm_kernel_L8_Mv1_40) KERNELv1x8_I KERNELv1x8_M2 @@ -1083,53 +1083,53 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_E - b .Ldgemm_kernel_L8_Mv1_44 + b L(dgemm_kernel_L8_Mv1_44) -.Ldgemm_kernel_L8_Mv1_40: +L(dgemm_kernel_L8_Mv1_40): INITv1x8 -.Ldgemm_kernel_L8_Mv1_44: +L(dgemm_kernel_L8_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L8_Mv1_100 + ble L(dgemm_kernel_L8_Mv1_100) .align 5 -.Ldgemm_kernel_L8_Mv1_46: +L(dgemm_kernel_L8_Mv1_46): KERNELv1x8_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L8_Mv1_46 + bne L(dgemm_kernel_L8_Mv1_46) -.Ldgemm_kernel_L8_Mv1_100: +L(dgemm_kernel_L8_Mv1_100): SAVEv1x8 -.Ldgemm_kernel_L8_Mv1_END: +L(dgemm_kernel_L8_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension - b.any .Ldgemm_kernel_L8_Mv1_20 + b.any L(dgemm_kernel_L8_Mv1_20) -.Ldgemm_kernel_L8_END: +L(dgemm_kernel_L8_END): lsl temp, origK, #6 add origPB, origPB, temp // B = B + K * 8 * 8 subs counterJ, counterJ , #1 // j-- - bgt .Ldgemm_kernel_L8_BEGIN + bgt L(dgemm_kernel_L8_BEGIN) /******************************************************************************/ /* Repeat the same thing if 4 left in N */ .align 5 -.Ldgemm_kernel_L4_BEGIN: +L(dgemm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #4 - ble .Ldgemm_kernel_L2_BEGIN + ble L(dgemm_kernel_L2_BEGIN) mov pCRow0, pC @@ -1138,11 +1138,11 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Ldgemm_kernel_L4_Mv2_BEGIN: +L(dgemm_kernel_L4_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Ldgemm_kernel_L4_Mv1_BEGIN + blt L(dgemm_kernel_L4_Mv1_BEGIN) mov counterI, origM @@ -1150,17 +1150,17 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ add pA2, pA1, temp, lsl #3 // pA1 = start of A array .align 5 -.Ldgemm_kernel_L4_Mv2_20: +L(dgemm_kernel_L4_Mv2_20): mov pB, origPB INITv2x4 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L4_Mv2_44 + ble L(dgemm_kernel_L4_Mv2_44) .align 5 -.Ldgemm_kernel_L4_Mv2_22: +L(dgemm_kernel_L4_Mv2_22): KERNELv2x4_SUB KERNELv2x4_SUB @@ -1172,56 +1172,56 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_Mv2_22 + bgt L(dgemm_kernel_L4_Mv2_22) -.Ldgemm_kernel_L4_Mv2_44: +L(dgemm_kernel_L4_Mv2_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L4_Mv2_100 + ble L(dgemm_kernel_L4_Mv2_100) .align 5 -.Ldgemm_kernel_L4_Mv2_46: +L(dgemm_kernel_L4_Mv2_46): KERNELv2x4_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L4_Mv2_46 + bne L(dgemm_kernel_L4_Mv2_46) -.Ldgemm_kernel_L4_Mv2_100: +L(dgemm_kernel_L4_Mv2_100): SAVEv2x4 mov pA1, pA2 // pA1 = pA2 mul temp, vec_len, origK // generate address of pA2 add pA2, pA1, temp, lsl #3 // -.Ldgemm_kernel_L4_Mv2_END: +L(dgemm_kernel_L4_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Ldgemm_kernel_L4_Mv2_20 + bge L(dgemm_kernel_L4_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Ldgemm_kernel_L4_END + beq L(dgemm_kernel_L4_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x4 kernel. -.Ldgemm_kernel_L4_Mv1_BEGIN: +L(dgemm_kernel_L4_Mv1_BEGIN): whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension .align 5 -.Ldgemm_kernel_L4_Mv1_20: +L(dgemm_kernel_L4_Mv1_20): mov pB, origPB INITv1x4 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L4_Mv1_44 + ble L(dgemm_kernel_L4_Mv1_44) .align 5 -.Ldgemm_kernel_L4_Mv1_22: +L(dgemm_kernel_L4_Mv1_22): KERNELv1x4_SUB KERNELv1x4_SUB @@ -1233,34 +1233,34 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_Mv1_22 + bgt L(dgemm_kernel_L4_Mv1_22) -.Ldgemm_kernel_L4_Mv1_44: +L(dgemm_kernel_L4_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L4_Mv1_100 + ble L(dgemm_kernel_L4_Mv1_100) .align 5 -.Ldgemm_kernel_L4_Mv1_46: +L(dgemm_kernel_L4_Mv1_46): KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L4_Mv1_46 + bne L(dgemm_kernel_L4_Mv1_46) -.Ldgemm_kernel_L4_Mv1_100: +L(dgemm_kernel_L4_Mv1_100): SAVEv1x4 -.Ldgemm_kernel_L4_Mv1_END: +L(dgemm_kernel_L4_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldgemm_kernel_L4_Mv1_20 + b.any L(dgemm_kernel_L4_Mv1_20) -.Ldgemm_kernel_L4_END: +L(dgemm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 @@ -1268,11 +1268,11 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ /* Repeat the same thing if 2 left in N */ .align 5 -.Ldgemm_kernel_L2_BEGIN: +L(dgemm_kernel_L2_BEGIN): mov counterJ , origN tst counterJ , #2 - ble .Ldgemm_kernel_L1_BEGIN + ble L(dgemm_kernel_L1_BEGIN) mov pCRow0, pC @@ -1280,11 +1280,11 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Ldgemm_kernel_L2_Mv2_BEGIN: +L(dgemm_kernel_L2_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Ldgemm_kernel_L2_Mv1_BEGIN + blt L(dgemm_kernel_L2_Mv1_BEGIN) mov counterI, origM @@ -1292,17 +1292,17 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ add pA2, pA1, temp, lsl #3 // pA1 = start of A array .align 5 -.Ldgemm_kernel_L2_Mv2_20: +L(dgemm_kernel_L2_Mv2_20): mov pB, origPB INITv2x2 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L2_Mv2_44 + ble L(dgemm_kernel_L2_Mv2_44) .align 5 -.Ldgemm_kernel_L2_Mv2_22: +L(dgemm_kernel_L2_Mv2_22): KERNELv2x2_SUB KERNELv2x2_SUB @@ -1314,57 +1314,57 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_Mv2_22 + bgt L(dgemm_kernel_L2_Mv2_22) -.Ldgemm_kernel_L2_Mv2_44: +L(dgemm_kernel_L2_Mv2_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L2_Mv2_100 + ble L(dgemm_kernel_L2_Mv2_100) .align 5 -.Ldgemm_kernel_L2_Mv2_46: +L(dgemm_kernel_L2_Mv2_46): KERNELv2x2_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L2_Mv2_46 + bne L(dgemm_kernel_L2_Mv2_46) -.Ldgemm_kernel_L2_Mv2_100: +L(dgemm_kernel_L2_Mv2_100): SAVEv2x2 mov pA1, pA2 // pA1 = pA2 mul temp, vec_len, origK // generate address of pA2 add pA2, pA1, temp, lsl #3 // -.Ldgemm_kernel_L2_Mv2_END: +L(dgemm_kernel_L2_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Ldgemm_kernel_L2_Mv2_20 + bge L(dgemm_kernel_L2_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Ldgemm_kernel_L2_END + beq L(dgemm_kernel_L2_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x2 kernel. -.Ldgemm_kernel_L2_Mv1_BEGIN: +L(dgemm_kernel_L2_Mv1_BEGIN): whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d .align 5 -.Ldgemm_kernel_L2_Mv1_20: +L(dgemm_kernel_L2_Mv1_20): mov pB, origPB INITv1x2 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L2_Mv1_44 + ble L(dgemm_kernel_L2_Mv1_44) .align 5 -.Ldgemm_kernel_L2_Mv1_22: +L(dgemm_kernel_L2_Mv1_22): KERNELv1x2_SUB KERNELv1x2_SUB @@ -1376,45 +1376,45 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_Mv1_22 + bgt L(dgemm_kernel_L2_Mv1_22) -.Ldgemm_kernel_L2_Mv1_44: +L(dgemm_kernel_L2_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L2_Mv1_100 + ble L(dgemm_kernel_L2_Mv1_100) .align 5 -.Ldgemm_kernel_L2_Mv1_46: +L(dgemm_kernel_L2_Mv1_46): KERNELv1x2_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L2_Mv1_46 + bne L(dgemm_kernel_L2_Mv1_46) -.Ldgemm_kernel_L2_Mv1_100: +L(dgemm_kernel_L2_Mv1_100): SAVEv1x2 -.Ldgemm_kernel_L2_Mv1_END: +L(dgemm_kernel_L2_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldgemm_kernel_L2_Mv1_20 + b.any L(dgemm_kernel_L2_Mv1_20) -.Ldgemm_kernel_L2_END: +L(dgemm_kernel_L2_END): add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 /******************************************************************************/ /* Repeat the same thing if 1 left in N */ .align 5 -.Ldgemm_kernel_L1_BEGIN: +L(dgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldgemm_kernel_L999 // done + ble L(dgemm_kernel_L999) // done mov pCRow0, pC @@ -1422,11 +1422,11 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Ldgemm_kernel_L1_Mv2_BEGIN: +L(dgemm_kernel_L1_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Ldgemm_kernel_L1_Mv1_BEGIN + blt L(dgemm_kernel_L1_Mv1_BEGIN) mov counterI, origM @@ -1435,17 +1435,17 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ .align 5 -.Ldgemm_kernel_L1_Mv2_20: +L(dgemm_kernel_L1_Mv2_20): mov pB, origPB INITv2x1 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Ldgemm_kernel_L1_Mv2_44 + ble L(dgemm_kernel_L1_Mv2_44) .align 5 -.Ldgemm_kernel_L1_Mv2_22: +L(dgemm_kernel_L1_Mv2_22): KERNELv2x1_SUB KERNELv2x1_SUB @@ -1457,57 +1457,57 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv2_22 + bgt L(dgemm_kernel_L1_Mv2_22) -.Ldgemm_kernel_L1_Mv2_44: +L(dgemm_kernel_L1_Mv2_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L1_Mv2_100 + ble L(dgemm_kernel_L1_Mv2_100) .align 5 -.Ldgemm_kernel_L1_Mv2_46: +L(dgemm_kernel_L1_Mv2_46): KERNELv2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv2_46 + bgt L(dgemm_kernel_L1_Mv2_46) -.Ldgemm_kernel_L1_Mv2_100: +L(dgemm_kernel_L1_Mv2_100): SAVEv2x1 mov pA1, pA2 // pA1 = pA2 mul temp, vec_len, origK // generate address of pA2 add pA2, pA1, temp, lsl #3 // -.Ldgemm_kernel_L1_Mv2_END: +L(dgemm_kernel_L1_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Ldgemm_kernel_L1_Mv2_20 + bge L(dgemm_kernel_L1_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Ldgemm_kernel_L1_END + beq L(dgemm_kernel_L1_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x1 kernel. -.Ldgemm_kernel_L1_Mv1_BEGIN: +L(dgemm_kernel_L1_Mv1_BEGIN): whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d .align 5 -.Ldgemm_kernel_L1_Mv1_20: +L(dgemm_kernel_L1_Mv1_20): mov pB, origPB INITv1x1 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Ldgemm_kernel_L1_Mv1_44 + ble L(dgemm_kernel_L1_Mv1_44) .align 5 -.Ldgemm_kernel_L1_Mv1_22: +L(dgemm_kernel_L1_Mv1_22): KERNELv1x1_SUB KERNELv1x1_SUB @@ -1519,38 +1519,38 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv1_22 + bgt L(dgemm_kernel_L1_Mv1_22) -.Ldgemm_kernel_L1_Mv1_44: +L(dgemm_kernel_L1_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L1_Mv1_100 + ble L(dgemm_kernel_L1_Mv1_100) .align 5 -.Ldgemm_kernel_L1_Mv1_46: +L(dgemm_kernel_L1_Mv1_46): KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv1_46 + bgt L(dgemm_kernel_L1_Mv1_46) -.Ldgemm_kernel_L1_Mv1_100: +L(dgemm_kernel_L1_Mv1_100): SAVEv1x1 -.Ldgemm_kernel_L1_Mv1_END: +L(dgemm_kernel_L1_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldgemm_kernel_L1_Mv1_20 + b.any L(dgemm_kernel_L1_Mv1_20) -.Ldgemm_kernel_L1_END: +L(dgemm_kernel_L1_END): /******************************************************************************/ -.Ldgemm_kernel_L999: +L(dgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dgemm_ncopy_4.S b/kernel/arm64/dgemm_ncopy_4.S index 29d274d931..11e2d23a11 100644 --- a/kernel/arm64/dgemm_ncopy_4.S +++ b/kernel/arm64/dgemm_ncopy_4.S @@ -192,14 +192,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. lsl LDA, LDA, #3 // LDA = LDA * SIZE -.Ldgemm_ncopy_L4_BEGIN: +L(dgemm_ncopy_L4_BEGIN): asr J, N, #2 // J = N / 4 cmp J, #0 - ble .Ldgemm_ncopy_L2_BEGIN + ble L(dgemm_ncopy_L2_BEGIN) .align 5 -.Ldgemm_ncopy_L4_M4_BEGIN: +L(dgemm_ncopy_L4_M4_BEGIN): mov A01, A00 add A02, A01, LDA @@ -209,128 +209,128 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #2 // I = M / 4 cmp I, #0 - ble .Ldgemm_ncopy_L4_M4_40 + ble L(dgemm_ncopy_L4_M4_40) .align 5 -.Ldgemm_ncopy_L4_M4_20: +L(dgemm_ncopy_L4_M4_20): COPY4x4 subs I , I , #1 - bne .Ldgemm_ncopy_L4_M4_20 + bne L(dgemm_ncopy_L4_M4_20) -.Ldgemm_ncopy_L4_M4_40: +L(dgemm_ncopy_L4_M4_40): and I, M , #3 cmp I, #0 - ble .Ldgemm_ncopy_L4_M4_END + ble L(dgemm_ncopy_L4_M4_END) .align 5 -.Ldgemm_ncopy_L4_M4_60: +L(dgemm_ncopy_L4_M4_60): COPY1x4 subs I , I , #1 - bne .Ldgemm_ncopy_L4_M4_60 + bne L(dgemm_ncopy_L4_M4_60) -.Ldgemm_ncopy_L4_M4_END: +L(dgemm_ncopy_L4_M4_END): subs J , J, #1 // j-- - bne .Ldgemm_ncopy_L4_M4_BEGIN + bne L(dgemm_ncopy_L4_M4_BEGIN) /*********************************************************************************************/ -.Ldgemm_ncopy_L2_BEGIN: +L(dgemm_ncopy_L2_BEGIN): tst N, #3 - ble .Ldgemm_ncopy_L999 + ble L(dgemm_ncopy_L999) tst N, #2 - ble .Ldgemm_ncopy_L1_BEGIN + ble L(dgemm_ncopy_L1_BEGIN) -.Ldgemm_ncopy_L2_M4_BEGIN: +L(dgemm_ncopy_L2_M4_BEGIN): mov A01, A00 add A02, A01, LDA add A00, A02, LDA asr I, M, #2 // I = M / 4 cmp I, #0 - ble .Ldgemm_ncopy_L2_M4_40 + ble L(dgemm_ncopy_L2_M4_40) .align 5 -.Ldgemm_ncopy_L2_M4_20: +L(dgemm_ncopy_L2_M4_20): COPY4x2 subs I , I , #1 - bne .Ldgemm_ncopy_L2_M4_20 + bne L(dgemm_ncopy_L2_M4_20) -.Ldgemm_ncopy_L2_M4_40: +L(dgemm_ncopy_L2_M4_40): and I, M , #3 cmp I, #0 - ble .Ldgemm_ncopy_L2_M4_END + ble L(dgemm_ncopy_L2_M4_END) .align 5 -.Ldgemm_ncopy_L2_M4_60: +L(dgemm_ncopy_L2_M4_60): COPY1x2 subs I , I , #1 - bne .Ldgemm_ncopy_L2_M4_60 + bne L(dgemm_ncopy_L2_M4_60) -.Ldgemm_ncopy_L2_M4_END: +L(dgemm_ncopy_L2_M4_END): /*********************************************************************************************/ -.Ldgemm_ncopy_L1_BEGIN: +L(dgemm_ncopy_L1_BEGIN): tst N, #1 - ble .Ldgemm_ncopy_L999 + ble L(dgemm_ncopy_L999) -.Ldgemm_ncopy_L1_M4_BEGIN: +L(dgemm_ncopy_L1_M4_BEGIN): mov A01, A00 asr I, M, #2 // I = M / 4 cmp I, #0 - ble .Ldgemm_ncopy_L1_M4_40 + ble L(dgemm_ncopy_L1_M4_40) .align 5 -.Ldgemm_ncopy_L1_M4_20: +L(dgemm_ncopy_L1_M4_20): COPY4x1 subs I , I , #1 - bne .Ldgemm_ncopy_L1_M4_20 + bne L(dgemm_ncopy_L1_M4_20) -.Ldgemm_ncopy_L1_M4_40: +L(dgemm_ncopy_L1_M4_40): and I, M , #3 cmp I, #0 - ble .Ldgemm_ncopy_L1_M4_END + ble L(dgemm_ncopy_L1_M4_END) .align 5 -.Ldgemm_ncopy_L1_M4_60: +L(dgemm_ncopy_L1_M4_60): COPY1x1 subs I , I , #1 - bne .Ldgemm_ncopy_L1_M4_60 + bne L(dgemm_ncopy_L1_M4_60) -.Ldgemm_ncopy_L1_M4_END: +L(dgemm_ncopy_L1_M4_END): -.Ldgemm_ncopy_L999: +L(dgemm_ncopy_L999): mov x0, #0 RESTORE_REGS diff --git a/kernel/arm64/dgemm_ncopy_8.S b/kernel/arm64/dgemm_ncopy_8.S index 366424830d..f4dc62f2d3 100644 --- a/kernel/arm64/dgemm_ncopy_8.S +++ b/kernel/arm64/dgemm_ncopy_8.S @@ -353,13 +353,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. lsl LDA, LDA, #3 // LDA = LDA * SIZE -.Ldgemm_ncopy_L8_BEGIN: +L(dgemm_ncopy_L8_BEGIN): asr J, N, #3 // J = N / 8 cmp J, #0 - ble .Ldgemm_ncopy_L4_BEGIN + ble L(dgemm_ncopy_L4_BEGIN) -.Ldgemm_ncopy_L8_M8_BEGIN: +L(dgemm_ncopy_L8_M8_BEGIN): mov A01, A00 add A02, A01, LDA @@ -374,46 +374,46 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #3 // I = M / 8 cmp I, #0 - ble .Ldgemm_ncopy_L8_M8_40 + ble L(dgemm_ncopy_L8_M8_40) -.Ldgemm_ncopy_L8_M8_20: +L(dgemm_ncopy_L8_M8_20): COPY8x8 subs I , I , #1 - bne .Ldgemm_ncopy_L8_M8_20 + bne L(dgemm_ncopy_L8_M8_20) -.Ldgemm_ncopy_L8_M8_40: +L(dgemm_ncopy_L8_M8_40): and I, M , #7 cmp I, #0 - ble .Ldgemm_ncopy_L8_M8_END + ble L(dgemm_ncopy_L8_M8_END) -.Ldgemm_ncopy_L8_M8_60: +L(dgemm_ncopy_L8_M8_60): COPY1x8 subs I , I , #1 - bne .Ldgemm_ncopy_L8_M8_60 + bne L(dgemm_ncopy_L8_M8_60) -.Ldgemm_ncopy_L8_M8_END: +L(dgemm_ncopy_L8_M8_END): subs J , J, #1 // j-- - bne .Ldgemm_ncopy_L8_M8_BEGIN + bne L(dgemm_ncopy_L8_M8_BEGIN) /*********************************************************************************************/ -.Ldgemm_ncopy_L4_BEGIN: +L(dgemm_ncopy_L4_BEGIN): tst N, #7 - ble .Ldgemm_ncopy_L999 + ble L(dgemm_ncopy_L999) tst N, #4 - ble .Ldgemm_ncopy_L2_BEGIN + ble L(dgemm_ncopy_L2_BEGIN) -.Ldgemm_ncopy_L4_M8_BEGIN: +L(dgemm_ncopy_L4_M8_BEGIN): mov A01, A00 add A02, A01, LDA @@ -423,118 +423,118 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #3 // I = M / 8 cmp I, #0 - ble .Ldgemm_ncopy_L4_M8_40 + ble L(dgemm_ncopy_L4_M8_40) -.Ldgemm_ncopy_L4_M8_20: +L(dgemm_ncopy_L4_M8_20): COPY8x4 subs I , I , #1 - bne .Ldgemm_ncopy_L4_M8_20 + bne L(dgemm_ncopy_L4_M8_20) -.Ldgemm_ncopy_L4_M8_40: +L(dgemm_ncopy_L4_M8_40): and I, M , #7 cmp I, #0 - ble .Ldgemm_ncopy_L4_M8_END + ble L(dgemm_ncopy_L4_M8_END) -.Ldgemm_ncopy_L4_M8_60: +L(dgemm_ncopy_L4_M8_60): COPY1x4 subs I , I , #1 - bne .Ldgemm_ncopy_L4_M8_60 + bne L(dgemm_ncopy_L4_M8_60) -.Ldgemm_ncopy_L4_M8_END: +L(dgemm_ncopy_L4_M8_END): /*********************************************************************************************/ -.Ldgemm_ncopy_L2_BEGIN: +L(dgemm_ncopy_L2_BEGIN): tst N, #3 - ble .Ldgemm_ncopy_L999 + ble L(dgemm_ncopy_L999) tst N, #2 - ble .Ldgemm_ncopy_L1_BEGIN + ble L(dgemm_ncopy_L1_BEGIN) -.Ldgemm_ncopy_L2_M8_BEGIN: +L(dgemm_ncopy_L2_M8_BEGIN): mov A01, A00 add A02, A01, LDA add A00, A02, LDA asr I, M, #3 // I = M / 8 cmp I, #0 - ble .Ldgemm_ncopy_L2_M8_40 + ble L(dgemm_ncopy_L2_M8_40) -.Ldgemm_ncopy_L2_M8_20: +L(dgemm_ncopy_L2_M8_20): COPY8x2 subs I , I , #1 - bne .Ldgemm_ncopy_L2_M8_20 + bne L(dgemm_ncopy_L2_M8_20) -.Ldgemm_ncopy_L2_M8_40: +L(dgemm_ncopy_L2_M8_40): and I, M , #7 cmp I, #0 - ble .Ldgemm_ncopy_L2_M8_END + ble L(dgemm_ncopy_L2_M8_END) -.Ldgemm_ncopy_L2_M8_60: +L(dgemm_ncopy_L2_M8_60): COPY1x2 subs I , I , #1 - bne .Ldgemm_ncopy_L2_M8_60 + bne L(dgemm_ncopy_L2_M8_60) -.Ldgemm_ncopy_L2_M8_END: +L(dgemm_ncopy_L2_M8_END): /*********************************************************************************************/ -.Ldgemm_ncopy_L1_BEGIN: +L(dgemm_ncopy_L1_BEGIN): tst N, #1 - ble .Ldgemm_ncopy_L999 + ble L(dgemm_ncopy_L999) -.Ldgemm_ncopy_L1_M8_BEGIN: +L(dgemm_ncopy_L1_M8_BEGIN): mov A01, A00 asr I, M, #3 // I = M / 8 cmp I, #0 - ble .Ldgemm_ncopy_L1_M8_40 + ble L(dgemm_ncopy_L1_M8_40) -.Ldgemm_ncopy_L1_M8_20: +L(dgemm_ncopy_L1_M8_20): COPY8x1 subs I , I , #1 - bne .Ldgemm_ncopy_L1_M8_20 + bne L(dgemm_ncopy_L1_M8_20) -.Ldgemm_ncopy_L1_M8_40: +L(dgemm_ncopy_L1_M8_40): and I, M , #7 cmp I, #0 - ble .Ldgemm_ncopy_L1_M8_END + ble L(dgemm_ncopy_L1_M8_END) -.Ldgemm_ncopy_L1_M8_60: +L(dgemm_ncopy_L1_M8_60): COPY1x1 subs I , I , #1 - bne .Ldgemm_ncopy_L1_M8_60 + bne L(dgemm_ncopy_L1_M8_60) -.Ldgemm_ncopy_L1_M8_END: +L(dgemm_ncopy_L1_M8_END): -.Ldgemm_ncopy_L999: +L(dgemm_ncopy_L999): mov x0, #0 RESTORE_REGS diff --git a/kernel/arm64/dgemm_tcopy_4.S b/kernel/arm64/dgemm_tcopy_4.S index 7c9135287f..8ace3a67be 100644 --- a/kernel/arm64/dgemm_tcopy_4.S +++ b/kernel/arm64/dgemm_tcopy_4.S @@ -247,13 +247,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. lsl M4, M, #5 // M4 = M * 4 * SIZE -.Ldgemm_tcopy_L4_BEGIN: +L(dgemm_tcopy_L4_BEGIN): asr J, M, #2 // J = M / 4 cmp J, #0 - ble .Ldgemm_tcopy_L2_BEGIN + ble L(dgemm_tcopy_L2_BEGIN) .align 5 -.Ldgemm_tcopy_L4_M4_BEGIN: +L(dgemm_tcopy_L4_M4_BEGIN): mov A01, A add A02, A01, LDA @@ -266,51 +266,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, N, #2 // I = N / 4 cmp I, #0 - ble .Ldgemm_tcopy_L4_M4_40 + ble L(dgemm_tcopy_L4_M4_40) .align 5 -.Ldgemm_tcopy_L4_M4_20: +L(dgemm_tcopy_L4_M4_20): COPY4x4 subs I , I , #1 - bne .Ldgemm_tcopy_L4_M4_20 + bne L(dgemm_tcopy_L4_M4_20) -.Ldgemm_tcopy_L4_M4_40: +L(dgemm_tcopy_L4_M4_40): tst N , #2 - ble .Ldgemm_tcopy_L4_M4_60 + ble L(dgemm_tcopy_L4_M4_60) COPY2x4 -.Ldgemm_tcopy_L4_M4_60: +L(dgemm_tcopy_L4_M4_60): tst N, #1 - ble .Ldgemm_tcopy_L4_M4_END + ble L(dgemm_tcopy_L4_M4_END) COPY1x4 -.Ldgemm_tcopy_L4_M4_END: +L(dgemm_tcopy_L4_M4_END): subs J , J, #1 // j-- - bne .Ldgemm_tcopy_L4_M4_BEGIN + bne L(dgemm_tcopy_L4_M4_BEGIN) /*********************************************************************************************/ -.Ldgemm_tcopy_L2_BEGIN: +L(dgemm_tcopy_L2_BEGIN): tst M, #3 - ble .Ldgemm_tcopy_L999 + ble L(dgemm_tcopy_L999) tst M, #2 - ble .Ldgemm_tcopy_L1_BEGIN + ble L(dgemm_tcopy_L1_BEGIN) -.Ldgemm_tcopy_L2_M4_BEGIN: +L(dgemm_tcopy_L2_M4_BEGIN): mov A01, A add A02, A01, LDA add A, A02, LDA @@ -320,80 +320,80 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, N, #2 // I = N / 4 cmp I, #0 - ble .Ldgemm_tcopy_L2_M4_40 + ble L(dgemm_tcopy_L2_M4_40) .align 5 -.Ldgemm_tcopy_L2_M4_20: +L(dgemm_tcopy_L2_M4_20): COPY4x2 subs I , I , #1 - bne .Ldgemm_tcopy_L2_M4_20 + bne L(dgemm_tcopy_L2_M4_20) -.Ldgemm_tcopy_L2_M4_40: +L(dgemm_tcopy_L2_M4_40): tst N , #2 - ble .Ldgemm_tcopy_L2_M4_60 + ble L(dgemm_tcopy_L2_M4_60) COPY2x2 -.Ldgemm_tcopy_L2_M4_60: +L(dgemm_tcopy_L2_M4_60): tst N , #1 - ble .Ldgemm_tcopy_L2_M4_END + ble L(dgemm_tcopy_L2_M4_END) COPY1x2 -.Ldgemm_tcopy_L2_M4_END: +L(dgemm_tcopy_L2_M4_END): /*********************************************************************************************/ -.Ldgemm_tcopy_L1_BEGIN: +L(dgemm_tcopy_L1_BEGIN): tst M, #1 - ble .Ldgemm_tcopy_L999 + ble L(dgemm_tcopy_L999) -.Ldgemm_tcopy_L1_M4_BEGIN: +L(dgemm_tcopy_L1_M4_BEGIN): mov A01, A // A01 = A mov B01, B asr I, N, #2 // I = M / 4 cmp I, #0 - ble .Ldgemm_tcopy_L1_M4_40 + ble L(dgemm_tcopy_L1_M4_40) .align 5 -.Ldgemm_tcopy_L1_M4_20: +L(dgemm_tcopy_L1_M4_20): COPY4x1 subs I , I , #1 - bne .Ldgemm_tcopy_L1_M4_20 + bne L(dgemm_tcopy_L1_M4_20) -.Ldgemm_tcopy_L1_M4_40: +L(dgemm_tcopy_L1_M4_40): tst N , #2 - ble .Ldgemm_tcopy_L1_M4_60 + ble L(dgemm_tcopy_L1_M4_60) COPY2x1 -.Ldgemm_tcopy_L1_M4_60: +L(dgemm_tcopy_L1_M4_60): tst N , #1 - ble .Ldgemm_tcopy_L1_M4_END + ble L(dgemm_tcopy_L1_M4_END) COPY1x1 -.Ldgemm_tcopy_L1_M4_END: +L(dgemm_tcopy_L1_M4_END): -.Ldgemm_tcopy_L999: +L(dgemm_tcopy_L999): mov x0, #0 // set return value RESTORE_REGS ret diff --git a/kernel/arm64/dgemm_tcopy_8.S b/kernel/arm64/dgemm_tcopy_8.S index 7e5bf6080f..a04f71d4d7 100644 --- a/kernel/arm64/dgemm_tcopy_8.S +++ b/kernel/arm64/dgemm_tcopy_8.S @@ -453,13 +453,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. lsl M8, M, #6 // M8 = M * 8 * SIZE -.Ldgemm_tcopy_L8_BEGIN: +L(dgemm_tcopy_L8_BEGIN): asr J, M, #3 // J = M / 4 cmp J, #0 - ble .Ldgemm_tcopy_L4_BEGIN + ble L(dgemm_tcopy_L4_BEGIN) .align 5 -.Ldgemm_tcopy_L8_M8_BEGIN: +L(dgemm_tcopy_L8_M8_BEGIN): mov A01, A add A02, A01, LDA @@ -476,53 +476,53 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, N, #3 // I = N / 8 cmp I, #0 - ble .Ldgemm_tcopy_L8_M8_40 + ble L(dgemm_tcopy_L8_M8_40) .align 5 -.Ldgemm_tcopy_L8_M8_20: +L(dgemm_tcopy_L8_M8_20): COPY8x8 subs I , I , #1 - bne .Ldgemm_tcopy_L8_M8_20 + bne L(dgemm_tcopy_L8_M8_20) -.Ldgemm_tcopy_L8_M8_40: +L(dgemm_tcopy_L8_M8_40): tst N , #4 - ble .Ldgemm_tcopy_L8_M8_60 + ble L(dgemm_tcopy_L8_M8_60) COPY4x8 -.Ldgemm_tcopy_L8_M8_60: +L(dgemm_tcopy_L8_M8_60): tst N , #2 - ble .Ldgemm_tcopy_L8_M8_80 + ble L(dgemm_tcopy_L8_M8_80) COPY2x8 -.Ldgemm_tcopy_L8_M8_80: +L(dgemm_tcopy_L8_M8_80): tst N, #1 - ble .Ldgemm_tcopy_L8_M8_END + ble L(dgemm_tcopy_L8_M8_END) COPY1x8 -.Ldgemm_tcopy_L8_M8_END: +L(dgemm_tcopy_L8_M8_END): subs J , J, #1 // j-- - bne .Ldgemm_tcopy_L8_M8_BEGIN + bne L(dgemm_tcopy_L8_M8_BEGIN) /*********************************************************************************************/ -.Ldgemm_tcopy_L4_BEGIN: +L(dgemm_tcopy_L4_BEGIN): tst M, #7 - ble .Ldgemm_tcopy_L999 + ble L(dgemm_tcopy_L999) tst M, #4 - ble .Ldgemm_tcopy_L2_BEGIN + ble L(dgemm_tcopy_L2_BEGIN) -.Ldgemm_tcopy_L4_M8_BEGIN: +L(dgemm_tcopy_L4_M8_BEGIN): mov A01, A add A02, A01, LDA @@ -535,51 +535,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, N, #3 // I = N / 8 cmp I, #0 - ble .Ldgemm_tcopy_L4_M8_40 + ble L(dgemm_tcopy_L4_M8_40) .align 5 -.Ldgemm_tcopy_L4_M8_20: +L(dgemm_tcopy_L4_M8_20): COPY8x4 subs I , I , #1 - bne .Ldgemm_tcopy_L4_M8_20 + bne L(dgemm_tcopy_L4_M8_20) -.Ldgemm_tcopy_L4_M8_40: +L(dgemm_tcopy_L4_M8_40): tst N , #4 - ble .Ldgemm_tcopy_L4_M8_60 + ble L(dgemm_tcopy_L4_M8_60) COPY4x4 -.Ldgemm_tcopy_L4_M8_60: +L(dgemm_tcopy_L4_M8_60): tst N , #2 - ble .Ldgemm_tcopy_L4_M8_80 + ble L(dgemm_tcopy_L4_M8_80) COPY2x4 -.Ldgemm_tcopy_L4_M8_80: +L(dgemm_tcopy_L4_M8_80): tst N, #1 - ble .Ldgemm_tcopy_L4_M8_END + ble L(dgemm_tcopy_L4_M8_END) COPY1x4 -.Ldgemm_tcopy_L4_M8_END: +L(dgemm_tcopy_L4_M8_END): /*********************************************************************************************/ -.Ldgemm_tcopy_L2_BEGIN: +L(dgemm_tcopy_L2_BEGIN): tst M, #3 - ble .Ldgemm_tcopy_L999 + ble L(dgemm_tcopy_L999) tst M, #2 - ble .Ldgemm_tcopy_L1_BEGIN + ble L(dgemm_tcopy_L1_BEGIN) -.Ldgemm_tcopy_L2_M8_BEGIN: +L(dgemm_tcopy_L2_M8_BEGIN): mov A01, A add A02, A01, LDA add A, A02, LDA @@ -589,90 +589,90 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, N, #3 // I = N / 8 cmp I, #0 - ble .Ldgemm_tcopy_L2_M8_40 + ble L(dgemm_tcopy_L2_M8_40) .align 5 -.Ldgemm_tcopy_L2_M8_20: +L(dgemm_tcopy_L2_M8_20): COPY8x2 subs I , I , #1 - bne .Ldgemm_tcopy_L2_M8_20 + bne L(dgemm_tcopy_L2_M8_20) -.Ldgemm_tcopy_L2_M8_40: +L(dgemm_tcopy_L2_M8_40): tst N , #4 - ble .Ldgemm_tcopy_L2_M8_60 + ble L(dgemm_tcopy_L2_M8_60) COPY4x2 -.Ldgemm_tcopy_L2_M8_60: +L(dgemm_tcopy_L2_M8_60): tst N , #2 - ble .Ldgemm_tcopy_L2_M8_80 + ble L(dgemm_tcopy_L2_M8_80) COPY2x2 -.Ldgemm_tcopy_L2_M8_80: +L(dgemm_tcopy_L2_M8_80): tst N , #1 - ble .Ldgemm_tcopy_L2_M8_END + ble L(dgemm_tcopy_L2_M8_END) COPY1x2 -.Ldgemm_tcopy_L2_M8_END: +L(dgemm_tcopy_L2_M8_END): /*********************************************************************************************/ -.Ldgemm_tcopy_L1_BEGIN: +L(dgemm_tcopy_L1_BEGIN): tst M, #1 - ble .Ldgemm_tcopy_L999 + ble L(dgemm_tcopy_L999) -.Ldgemm_tcopy_L1_M8_BEGIN: +L(dgemm_tcopy_L1_M8_BEGIN): mov A01, A // A01 = A mov B01, B asr I, N, #3 // I = M / 8 cmp I, #0 - ble .Ldgemm_tcopy_L1_M8_40 + ble L(dgemm_tcopy_L1_M8_40) .align 5 -.Ldgemm_tcopy_L1_M8_20: +L(dgemm_tcopy_L1_M8_20): COPY8x1 subs I , I , #1 - bne .Ldgemm_tcopy_L1_M8_20 + bne L(dgemm_tcopy_L1_M8_20) -.Ldgemm_tcopy_L1_M8_40: +L(dgemm_tcopy_L1_M8_40): tst N , #4 - ble .Ldgemm_tcopy_L1_M8_60 + ble L(dgemm_tcopy_L1_M8_60) COPY4x1 -.Ldgemm_tcopy_L1_M8_60: +L(dgemm_tcopy_L1_M8_60): tst N , #2 - ble .Ldgemm_tcopy_L1_M8_80 + ble L(dgemm_tcopy_L1_M8_80) COPY2x1 -.Ldgemm_tcopy_L1_M8_80: +L(dgemm_tcopy_L1_M8_80): tst N , #1 - ble .Ldgemm_tcopy_L1_M8_END + ble L(dgemm_tcopy_L1_M8_END) COPY1x1 -.Ldgemm_tcopy_L1_M8_END: +L(dgemm_tcopy_L1_M8_END): -.Ldgemm_tcopy_L999: +L(dgemm_tcopy_L999): mov x0, #0 // set return value RESTORE_REGS ret diff --git a/kernel/arm64/dot.S b/kernel/arm64/dot.S index 3f01385fed..e5df75fd5f 100644 --- a/kernel/arm64/dot.S +++ b/kernel/arm64/dot.S @@ -158,51 +158,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif cmp N, xzr - ble .Ldot_kernel_L999 + ble L(dot_kernel_L999) cmp INC_X, #1 - bne .Ldot_kernel_S_BEGIN + bne L(dot_kernel_S_BEGIN) cmp INC_Y, #1 - bne .Ldot_kernel_S_BEGIN + bne L(dot_kernel_S_BEGIN) -.Ldot_kernel_F_BEGIN: +L(dot_kernel_F_BEGIN): asr I, N, #2 cmp I, xzr - beq .Ldot_kernel_F1 + beq L(dot_kernel_F1) -.Ldot_kernel_F4: +L(dot_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Ldot_kernel_F4 + bne L(dot_kernel_F4) KERNEL_F4_FINALIZE -.Ldot_kernel_F1: +L(dot_kernel_F1): ands I, N, #3 - ble .Ldot_kernel_L999 + ble L(dot_kernel_L999) -.Ldot_kernel_F10: +L(dot_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Ldot_kernel_F10 + bne L(dot_kernel_F10) ret -.Ldot_kernel_S_BEGIN: +L(dot_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Ldot_kernel_S1 + ble L(dot_kernel_S1) -.Ldot_kernel_S4: +L(dot_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -210,21 +210,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Ldot_kernel_S4 + bne L(dot_kernel_S4) -.Ldot_kernel_S1: +L(dot_kernel_S1): ands I, N, #3 - ble .Ldot_kernel_L999 + ble L(dot_kernel_L999) -.Ldot_kernel_S10: +L(dot_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Ldot_kernel_S10 + bne L(dot_kernel_S10) -.Ldot_kernel_L999: +L(dot_kernel_L999): ret diff --git a/kernel/arm64/dtrmm_kernel_4x4.S b/kernel/arm64/dtrmm_kernel_4x4.S index b528aeb182..fdfb56e6a4 100644 --- a/kernel/arm64/dtrmm_kernel_4x4.S +++ b/kernel/arm64/dtrmm_kernel_4x4.S @@ -549,11 +549,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Ldtrmm_kernel_L2_BEGIN + ble L(dtrmm_kernel_L2_BEGIN) /******************************************************************************/ -.Ldtrmm_kernel_L4_BEGIN: +L(dtrmm_kernel_L4_BEGIN): mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #2 @@ -563,14 +563,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldtrmm_kernel_L4_M4_BEGIN: +L(dtrmm_kernel_L4_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Ldtrmm_kernel_L4_M2_BEGIN + ble L(dtrmm_kernel_L4_M2_BEGIN) -.Ldtrmm_kernel_L4_M4_20: +L(dtrmm_kernel_L4_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -591,57 +591,57 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Ldtrmm_kernel_L4_M4_32 + blt L(dtrmm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Ldtrmm_kernel_L4_M4_22a + ble L(dtrmm_kernel_L4_M4_22a) .align 5 -.Ldtrmm_kernel_L4_M4_22: +L(dtrmm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M4_22 + bgt L(dtrmm_kernel_L4_M4_22) -.Ldtrmm_kernel_L4_M4_22a: +L(dtrmm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Ldtrmm_kernel_L4_M4_44 + b L(dtrmm_kernel_L4_M4_44) -.Ldtrmm_kernel_L4_M4_32: +L(dtrmm_kernel_L4_M4_32): tst counterL, #1 - ble .Ldtrmm_kernel_L4_M4_40 + ble L(dtrmm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Ldtrmm_kernel_L4_M4_44 + b L(dtrmm_kernel_L4_M4_44) -.Ldtrmm_kernel_L4_M4_40: +L(dtrmm_kernel_L4_M4_40): INIT4x4 -.Ldtrmm_kernel_L4_M4_44: +L(dtrmm_kernel_L4_M4_44): ands counterL , tempK, #1 - ble .Ldtrmm_kernel_L4_M4_100 + ble L(dtrmm_kernel_L4_M4_100) -.Ldtrmm_kernel_L4_M4_46: +L(dtrmm_kernel_L4_M4_46): KERNEL4x4_SUB -.Ldtrmm_kernel_L4_M4_100: +L(dtrmm_kernel_L4_M4_100): SAVE4x4 @@ -660,20 +660,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Ldtrmm_kernel_L4_M4_END: +L(dtrmm_kernel_L4_M4_END): subs counterI, counterI, #1 - bne .Ldtrmm_kernel_L4_M4_20 + bne L(dtrmm_kernel_L4_M4_20) -.Ldtrmm_kernel_L4_M2_BEGIN: +L(dtrmm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldtrmm_kernel_L4_END + ble L(dtrmm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L4_M1_BEGIN + ble L(dtrmm_kernel_L4_M1_BEGIN) -.Ldtrmm_kernel_L4_M2_20: +L(dtrmm_kernel_L4_M2_20): INIT2x4 @@ -697,9 +697,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L4_M2_40 + ble L(dtrmm_kernel_L4_M2_40) -.Ldtrmm_kernel_L4_M2_22: +L(dtrmm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -712,22 +712,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M2_22 + bgt L(dtrmm_kernel_L4_M2_22) -.Ldtrmm_kernel_L4_M2_40: +L(dtrmm_kernel_L4_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L4_M2_100 + ble L(dtrmm_kernel_L4_M2_100) -.Ldtrmm_kernel_L4_M2_42: +L(dtrmm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M2_42 + bgt L(dtrmm_kernel_L4_M2_42) -.Ldtrmm_kernel_L4_M2_100: +L(dtrmm_kernel_L4_M2_100): SAVE2x4 @@ -747,15 +747,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Ldtrmm_kernel_L4_M2_END: +L(dtrmm_kernel_L4_M2_END): -.Ldtrmm_kernel_L4_M1_BEGIN: +L(dtrmm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldtrmm_kernel_L4_END + ble L(dtrmm_kernel_L4_END) -.Ldtrmm_kernel_L4_M1_20: +L(dtrmm_kernel_L4_M1_20): INIT1x4 @@ -779,9 +779,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L4_M1_40 + ble L(dtrmm_kernel_L4_M1_40) -.Ldtrmm_kernel_L4_M1_22: +L(dtrmm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -793,22 +793,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M1_22 + bgt L(dtrmm_kernel_L4_M1_22) -.Ldtrmm_kernel_L4_M1_40: +L(dtrmm_kernel_L4_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L4_M1_100 + ble L(dtrmm_kernel_L4_M1_100) -.Ldtrmm_kernel_L4_M1_42: +L(dtrmm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M1_42 + bgt L(dtrmm_kernel_L4_M1_42) -.Ldtrmm_kernel_L4_M1_100: +L(dtrmm_kernel_L4_M1_100): SAVE1x4 @@ -828,7 +828,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Ldtrmm_kernel_L4_END: +L(dtrmm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 @@ -838,19 +838,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Ldtrmm_kernel_L4_BEGIN + bgt L(dtrmm_kernel_L4_BEGIN) /******************************************************************************/ -.Ldtrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(dtrmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Ldtrmm_kernel_L999 // error, N was less than 4? + ble L(dtrmm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Ldtrmm_kernel_L1_BEGIN + ble L(dtrmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -863,14 +863,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Ldtrmm_kernel_L2_M4_BEGIN: +L(dtrmm_kernel_L2_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI,#0 - ble .Ldtrmm_kernel_L2_M2_BEGIN + ble L(dtrmm_kernel_L2_M2_BEGIN) -.Ldtrmm_kernel_L2_M4_20: +L(dtrmm_kernel_L2_M4_20): INIT4x2 @@ -894,10 +894,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldtrmm_kernel_L2_M4_40 + ble L(dtrmm_kernel_L2_M4_40) .align 5 -.Ldtrmm_kernel_L2_M4_22: +L(dtrmm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -909,22 +909,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M4_22 + bgt L(dtrmm_kernel_L2_M4_22) -.Ldtrmm_kernel_L2_M4_40: +L(dtrmm_kernel_L2_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L2_M4_100 + ble L(dtrmm_kernel_L2_M4_100) -.Ldtrmm_kernel_L2_M4_42: +L(dtrmm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M4_42 + bgt L(dtrmm_kernel_L2_M4_42) -.Ldtrmm_kernel_L2_M4_100: +L(dtrmm_kernel_L2_M4_100): SAVE4x2 @@ -944,22 +944,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Ldtrmm_kernel_L2_M4_END: +L(dtrmm_kernel_L2_M4_END): subs counterI, counterI, #1 - bgt .Ldtrmm_kernel_L2_M4_20 + bgt L(dtrmm_kernel_L2_M4_20) -.Ldtrmm_kernel_L2_M2_BEGIN: +L(dtrmm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldtrmm_kernel_L2_END + ble L(dtrmm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L2_M1_BEGIN + ble L(dtrmm_kernel_L2_M1_BEGIN) -.Ldtrmm_kernel_L2_M2_20: +L(dtrmm_kernel_L2_M2_20): INIT2x2 @@ -983,9 +983,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldtrmm_kernel_L2_M2_40 + ble L(dtrmm_kernel_L2_M2_40) -.Ldtrmm_kernel_L2_M2_22: +L(dtrmm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -998,22 +998,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M2_22 + bgt L(dtrmm_kernel_L2_M2_22) -.Ldtrmm_kernel_L2_M2_40: +L(dtrmm_kernel_L2_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L2_M2_100 + ble L(dtrmm_kernel_L2_M2_100) -.Ldtrmm_kernel_L2_M2_42: +L(dtrmm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M2_42 + bgt L(dtrmm_kernel_L2_M2_42) -.Ldtrmm_kernel_L2_M2_100: +L(dtrmm_kernel_L2_M2_100): SAVE2x2 @@ -1033,15 +1033,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Ldtrmm_kernel_L2_M2_END: +L(dtrmm_kernel_L2_M2_END): -.Ldtrmm_kernel_L2_M1_BEGIN: +L(dtrmm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldtrmm_kernel_L2_END + ble L(dtrmm_kernel_L2_END) -.Ldtrmm_kernel_L2_M1_20: +L(dtrmm_kernel_L2_M1_20): INIT1x2 @@ -1065,9 +1065,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Ldtrmm_kernel_L2_M1_40 + ble L(dtrmm_kernel_L2_M1_40) -.Ldtrmm_kernel_L2_M1_22: +L(dtrmm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1079,22 +1079,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M1_22 + bgt L(dtrmm_kernel_L2_M1_22) -.Ldtrmm_kernel_L2_M1_40: +L(dtrmm_kernel_L2_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L2_M1_100 + ble L(dtrmm_kernel_L2_M1_100) -.Ldtrmm_kernel_L2_M1_42: +L(dtrmm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M1_42 + bgt L(dtrmm_kernel_L2_M1_42) -.Ldtrmm_kernel_L2_M1_100: +L(dtrmm_kernel_L2_M1_100): SAVE1x2 @@ -1114,7 +1114,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Ldtrmm_kernel_L2_END: +L(dtrmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -1122,11 +1122,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Ldtrmm_kernel_L1_BEGIN: +L(dtrmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldtrmm_kernel_L999 // done + ble L(dtrmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1138,14 +1138,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Ldtrmm_kernel_L1_M4_BEGIN: +L(dtrmm_kernel_L1_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Ldtrmm_kernel_L1_M2_BEGIN + ble L(dtrmm_kernel_L1_M2_BEGIN) -.Ldtrmm_kernel_L1_M4_20: +L(dtrmm_kernel_L1_M4_20): INIT4x1 @@ -1169,10 +1169,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L1_M4_40 + ble L(dtrmm_kernel_L1_M4_40) .align 5 -.Ldtrmm_kernel_L1_M4_22: +L(dtrmm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1184,22 +1184,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M4_22 + bgt L(dtrmm_kernel_L1_M4_22) -.Ldtrmm_kernel_L1_M4_40: +L(dtrmm_kernel_L1_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L1_M4_100 + ble L(dtrmm_kernel_L1_M4_100) -.Ldtrmm_kernel_L1_M4_42: +L(dtrmm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M4_42 + bgt L(dtrmm_kernel_L1_M4_42) -.Ldtrmm_kernel_L1_M4_100: +L(dtrmm_kernel_L1_M4_100): SAVE4x1 @@ -1220,22 +1220,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Ldtrmm_kernel_L1_M4_END: +L(dtrmm_kernel_L1_M4_END): subs counterI, counterI, #1 - bgt .Ldtrmm_kernel_L1_M4_20 + bgt L(dtrmm_kernel_L1_M4_20) -.Ldtrmm_kernel_L1_M2_BEGIN: +L(dtrmm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldtrmm_kernel_L1_END + ble L(dtrmm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L1_M1_BEGIN + ble L(dtrmm_kernel_L1_M1_BEGIN) -.Ldtrmm_kernel_L1_M2_20: +L(dtrmm_kernel_L1_M2_20): INIT2x1 @@ -1259,9 +1259,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L1_M2_40 + ble L(dtrmm_kernel_L1_M2_40) -.Ldtrmm_kernel_L1_M2_22: +L(dtrmm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1274,22 +1274,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M2_22 + bgt L(dtrmm_kernel_L1_M2_22) -.Ldtrmm_kernel_L1_M2_40: +L(dtrmm_kernel_L1_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L1_M2_100 + ble L(dtrmm_kernel_L1_M2_100) -.Ldtrmm_kernel_L1_M2_42: +L(dtrmm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M2_42 + bgt L(dtrmm_kernel_L1_M2_42) -.Ldtrmm_kernel_L1_M2_100: +L(dtrmm_kernel_L1_M2_100): SAVE2x1 @@ -1309,15 +1309,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Ldtrmm_kernel_L1_M2_END: +L(dtrmm_kernel_L1_M2_END): -.Ldtrmm_kernel_L1_M1_BEGIN: +L(dtrmm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldtrmm_kernel_L1_END + ble L(dtrmm_kernel_L1_END) -.Ldtrmm_kernel_L1_M1_20: +L(dtrmm_kernel_L1_M1_20): INIT1x1 @@ -1341,9 +1341,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L1_M1_40 + ble L(dtrmm_kernel_L1_M1_40) -.Ldtrmm_kernel_L1_M1_22: +L(dtrmm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1355,30 +1355,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M1_22 + bgt L(dtrmm_kernel_L1_M1_22) -.Ldtrmm_kernel_L1_M1_40: +L(dtrmm_kernel_L1_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L1_M1_100 + ble L(dtrmm_kernel_L1_M1_100) -.Ldtrmm_kernel_L1_M1_42: +L(dtrmm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M1_42 + bgt L(dtrmm_kernel_L1_M1_42) -.Ldtrmm_kernel_L1_M1_100: +L(dtrmm_kernel_L1_M1_100): SAVE1x1 -.Ldtrmm_kernel_L1_END: +L(dtrmm_kernel_L1_END): -.Ldtrmm_kernel_L999: +L(dtrmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dtrmm_kernel_4x8.S b/kernel/arm64/dtrmm_kernel_4x8.S index 47956dec57..969ffae452 100644 --- a/kernel/arm64/dtrmm_kernel_4x8.S +++ b/kernel/arm64/dtrmm_kernel_4x8.S @@ -900,11 +900,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Ldtrmm_kernel_L4_BEGIN + ble L(dtrmm_kernel_L4_BEGIN) /******************************************************************************/ -.Ldtrmm_kernel_L8_BEGIN: +L(dtrmm_kernel_L8_BEGIN): mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #3 @@ -915,14 +915,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldtrmm_kernel_L8_M4_BEGIN: +L(dtrmm_kernel_L8_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Ldtrmm_kernel_L8_M2_BEGIN + ble L(dtrmm_kernel_L8_M2_BEGIN) -.Ldtrmm_kernel_L8_M4_20: +L(dtrmm_kernel_L8_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -944,57 +944,57 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL, tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Ldtrmm_kernel_L8_M4_32 + blt L(dtrmm_kernel_L8_M4_32) KERNEL4x8_I // do one in the K KERNEL4x8_M2 // do another in the K subs counterL, counterL, #2 - ble .Ldtrmm_kernel_L8_M4_22a + ble L(dtrmm_kernel_L8_M4_22a) .align 5 -.Ldtrmm_kernel_L8_M4_22: +L(dtrmm_kernel_L8_M4_22): KERNEL4x8_M1 KERNEL4x8_M2 subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L8_M4_22 + bgt L(dtrmm_kernel_L8_M4_22) -.Ldtrmm_kernel_L8_M4_22a: +L(dtrmm_kernel_L8_M4_22a): KERNEL4x8_M1 KERNEL4x8_E - b .Ldtrmm_kernel_L8_M4_44 + b L(dtrmm_kernel_L8_M4_44) -.Ldtrmm_kernel_L8_M4_32: +L(dtrmm_kernel_L8_M4_32): tst counterL, #1 - ble .Ldtrmm_kernel_L8_M4_40 + ble L(dtrmm_kernel_L8_M4_40) KERNEL4x8_I KERNEL4x8_E - b .Ldtrmm_kernel_L8_M4_44 + b L(dtrmm_kernel_L8_M4_44) -.Ldtrmm_kernel_L8_M4_40: +L(dtrmm_kernel_L8_M4_40): INIT4x8 -.Ldtrmm_kernel_L8_M4_44: +L(dtrmm_kernel_L8_M4_44): ands counterL, tempK, #1 - ble .Ldtrmm_kernel_L8_M4_100 + ble L(dtrmm_kernel_L8_M4_100) -.Ldtrmm_kernel_L8_M4_46: +L(dtrmm_kernel_L8_M4_46): KERNEL4x8_SUB -.Ldtrmm_kernel_L8_M4_100: +L(dtrmm_kernel_L8_M4_100): SAVE4x8 @@ -1014,20 +1014,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Ldtrmm_kernel_L8_M4_END: +L(dtrmm_kernel_L8_M4_END): subs counterI, counterI, #1 - bne .Ldtrmm_kernel_L8_M4_20 + bne L(dtrmm_kernel_L8_M4_20) -.Ldtrmm_kernel_L8_M2_BEGIN: +L(dtrmm_kernel_L8_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldtrmm_kernel_L8_END + ble L(dtrmm_kernel_L8_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L8_M1_BEGIN + ble L(dtrmm_kernel_L8_M1_BEGIN) -.Ldtrmm_kernel_L8_M2_20: +L(dtrmm_kernel_L8_M2_20): INIT2x8 @@ -1051,9 +1051,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL, tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L8_M2_40 + ble L(dtrmm_kernel_L8_M2_40) -.Ldtrmm_kernel_L8_M2_22: +L(dtrmm_kernel_L8_M2_22): KERNEL2x8_SUB KERNEL2x8_SUB @@ -1066,22 +1066,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L8_M2_22 + bgt L(dtrmm_kernel_L8_M2_22) -.Ldtrmm_kernel_L8_M2_40: +L(dtrmm_kernel_L8_M2_40): ands counterL, tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L8_M2_100 + ble L(dtrmm_kernel_L8_M2_100) -.Ldtrmm_kernel_L8_M2_42: +L(dtrmm_kernel_L8_M2_42): KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L8_M2_42 + bgt L(dtrmm_kernel_L8_M2_42) -.Ldtrmm_kernel_L8_M2_100: +L(dtrmm_kernel_L8_M2_100): SAVE2x8 @@ -1102,15 +1102,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Ldtrmm_kernel_L8_M2_END: +L(dtrmm_kernel_L8_M2_END): -.Ldtrmm_kernel_L8_M1_BEGIN: +L(dtrmm_kernel_L8_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldtrmm_kernel_L8_END + ble L(dtrmm_kernel_L8_END) -.Ldtrmm_kernel_L8_M1_20: +L(dtrmm_kernel_L8_M1_20): INIT1x8 @@ -1134,9 +1134,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL, tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L8_M1_40 + ble L(dtrmm_kernel_L8_M1_40) -.Ldtrmm_kernel_L8_M1_22: +L(dtrmm_kernel_L8_M1_22): KERNEL1x8_SUB KERNEL1x8_SUB KERNEL1x8_SUB @@ -1148,22 +1148,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L8_M1_22 + bgt L(dtrmm_kernel_L8_M1_22) -.Ldtrmm_kernel_L8_M1_40: +L(dtrmm_kernel_L8_M1_40): ands counterL, tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L8_M1_100 + ble L(dtrmm_kernel_L8_M1_100) -.Ldtrmm_kernel_L8_M1_42: +L(dtrmm_kernel_L8_M1_42): KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L8_M1_42 + bgt L(dtrmm_kernel_L8_M1_42) -.Ldtrmm_kernel_L8_M1_100: +L(dtrmm_kernel_L8_M1_100): SAVE1x8 @@ -1183,7 +1183,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Ldtrmm_kernel_L8_END: +L(dtrmm_kernel_L8_END): lsl temp, origK, #6 add origPB, origPB, temp // B = B + K * 8 * 8 @@ -1193,19 +1193,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Ldtrmm_kernel_L8_BEGIN + bgt L(dtrmm_kernel_L8_BEGIN) /******************************************************************************/ -.Ldtrmm_kernel_L4_BEGIN: +L(dtrmm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #7 - ble .Ldtrmm_kernel_L999 + ble L(dtrmm_kernel_L999) tst counterJ , #4 - ble .Ldtrmm_kernel_L2_BEGIN + ble L(dtrmm_kernel_L2_BEGIN) mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #2 @@ -1216,14 +1216,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldtrmm_kernel_L4_M4_BEGIN: +L(dtrmm_kernel_L4_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Ldtrmm_kernel_L4_M2_BEGIN + ble L(dtrmm_kernel_L4_M2_BEGIN) -.Ldtrmm_kernel_L4_M4_20: +L(dtrmm_kernel_L4_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1244,57 +1244,57 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL, tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Ldtrmm_kernel_L4_M4_32 + blt L(dtrmm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Ldtrmm_kernel_L4_M4_22a + ble L(dtrmm_kernel_L4_M4_22a) .align 5 -.Ldtrmm_kernel_L4_M4_22: +L(dtrmm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M4_22 + bgt L(dtrmm_kernel_L4_M4_22) -.Ldtrmm_kernel_L4_M4_22a: +L(dtrmm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Ldtrmm_kernel_L4_M4_44 + b L(dtrmm_kernel_L4_M4_44) -.Ldtrmm_kernel_L4_M4_32: +L(dtrmm_kernel_L4_M4_32): tst counterL, #1 - ble .Ldtrmm_kernel_L4_M4_40 + ble L(dtrmm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Ldtrmm_kernel_L4_M4_44 + b L(dtrmm_kernel_L4_M4_44) -.Ldtrmm_kernel_L4_M4_40: +L(dtrmm_kernel_L4_M4_40): INIT4x4 -.Ldtrmm_kernel_L4_M4_44: +L(dtrmm_kernel_L4_M4_44): ands counterL , tempK, #1 - ble .Ldtrmm_kernel_L4_M4_100 + ble L(dtrmm_kernel_L4_M4_100) -.Ldtrmm_kernel_L4_M4_46: +L(dtrmm_kernel_L4_M4_46): KERNEL4x4_SUB -.Ldtrmm_kernel_L4_M4_100: +L(dtrmm_kernel_L4_M4_100): SAVE4x4 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) @@ -1312,20 +1312,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Ldtrmm_kernel_L4_M4_END: +L(dtrmm_kernel_L4_M4_END): subs counterI, counterI, #1 - bne .Ldtrmm_kernel_L4_M4_20 + bne L(dtrmm_kernel_L4_M4_20) -.Ldtrmm_kernel_L4_M2_BEGIN: +L(dtrmm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldtrmm_kernel_L4_END + ble L(dtrmm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L4_M1_BEGIN + ble L(dtrmm_kernel_L4_M1_BEGIN) -.Ldtrmm_kernel_L4_M2_20: +L(dtrmm_kernel_L4_M2_20): INIT2x4 @@ -1348,9 +1348,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L4_M2_40 + ble L(dtrmm_kernel_L4_M2_40) -.Ldtrmm_kernel_L4_M2_22: +L(dtrmm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1363,22 +1363,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M2_22 + bgt L(dtrmm_kernel_L4_M2_22) -.Ldtrmm_kernel_L4_M2_40: +L(dtrmm_kernel_L4_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L4_M2_100 + ble L(dtrmm_kernel_L4_M2_100) -.Ldtrmm_kernel_L4_M2_42: +L(dtrmm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M2_42 + bgt L(dtrmm_kernel_L4_M2_42) -.Ldtrmm_kernel_L4_M2_100: +L(dtrmm_kernel_L4_M2_100): SAVE2x4 @@ -1397,15 +1397,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #2 #endif -.Ldtrmm_kernel_L4_M2_END: +L(dtrmm_kernel_L4_M2_END): -.Ldtrmm_kernel_L4_M1_BEGIN: +L(dtrmm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldtrmm_kernel_L4_END + ble L(dtrmm_kernel_L4_END) -.Ldtrmm_kernel_L4_M1_20: +L(dtrmm_kernel_L4_M1_20): INIT1x4 @@ -1428,9 +1428,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L4_M1_40 + ble L(dtrmm_kernel_L4_M1_40) -.Ldtrmm_kernel_L4_M1_22: +L(dtrmm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1442,22 +1442,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M1_22 + bgt L(dtrmm_kernel_L4_M1_22) -.Ldtrmm_kernel_L4_M1_40: +L(dtrmm_kernel_L4_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L4_M1_100 + ble L(dtrmm_kernel_L4_M1_100) -.Ldtrmm_kernel_L4_M1_42: +L(dtrmm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M1_42 + bgt L(dtrmm_kernel_L4_M1_42) -.Ldtrmm_kernel_L4_M1_100: +L(dtrmm_kernel_L4_M1_100): SAVE1x4 @@ -1476,7 +1476,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #1 #endif -.Ldtrmm_kernel_L4_END: +L(dtrmm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 @@ -1486,14 +1486,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Ldtrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(dtrmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Ldtrmm_kernel_L999 // error, N was less than 4? + ble L(dtrmm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Ldtrmm_kernel_L1_BEGIN + ble L(dtrmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1505,14 +1505,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Ldtrmm_kernel_L2_M4_BEGIN: +L(dtrmm_kernel_L2_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI,#0 - ble .Ldtrmm_kernel_L2_M2_BEGIN + ble L(dtrmm_kernel_L2_M2_BEGIN) -.Ldtrmm_kernel_L2_M4_20: +L(dtrmm_kernel_L2_M4_20): INIT4x2 @@ -1535,10 +1535,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldtrmm_kernel_L2_M4_40 + ble L(dtrmm_kernel_L2_M4_40) .align 5 -.Ldtrmm_kernel_L2_M4_22: +L(dtrmm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1550,22 +1550,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M4_22 + bgt L(dtrmm_kernel_L2_M4_22) -.Ldtrmm_kernel_L2_M4_40: +L(dtrmm_kernel_L2_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L2_M4_100 + ble L(dtrmm_kernel_L2_M4_100) -.Ldtrmm_kernel_L2_M4_42: +L(dtrmm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M4_42 + bgt L(dtrmm_kernel_L2_M4_42) -.Ldtrmm_kernel_L2_M4_100: +L(dtrmm_kernel_L2_M4_100): SAVE4x2 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) @@ -1584,22 +1584,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Ldtrmm_kernel_L2_M4_END: +L(dtrmm_kernel_L2_M4_END): subs counterI, counterI, #1 - bgt .Ldtrmm_kernel_L2_M4_20 + bgt L(dtrmm_kernel_L2_M4_20) -.Ldtrmm_kernel_L2_M2_BEGIN: +L(dtrmm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldtrmm_kernel_L2_END + ble L(dtrmm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L2_M1_BEGIN + ble L(dtrmm_kernel_L2_M1_BEGIN) -.Ldtrmm_kernel_L2_M2_20: +L(dtrmm_kernel_L2_M2_20): INIT2x2 @@ -1622,9 +1622,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldtrmm_kernel_L2_M2_40 + ble L(dtrmm_kernel_L2_M2_40) -.Ldtrmm_kernel_L2_M2_22: +L(dtrmm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1637,22 +1637,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M2_22 + bgt L(dtrmm_kernel_L2_M2_22) -.Ldtrmm_kernel_L2_M2_40: +L(dtrmm_kernel_L2_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L2_M2_100 + ble L(dtrmm_kernel_L2_M2_100) -.Ldtrmm_kernel_L2_M2_42: +L(dtrmm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M2_42 + bgt L(dtrmm_kernel_L2_M2_42) -.Ldtrmm_kernel_L2_M2_100: +L(dtrmm_kernel_L2_M2_100): SAVE2x2 @@ -1671,15 +1671,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #2 #endif -.Ldtrmm_kernel_L2_M2_END: +L(dtrmm_kernel_L2_M2_END): -.Ldtrmm_kernel_L2_M1_BEGIN: +L(dtrmm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldtrmm_kernel_L2_END + ble L(dtrmm_kernel_L2_END) -.Ldtrmm_kernel_L2_M1_20: +L(dtrmm_kernel_L2_M1_20): INIT1x2 @@ -1702,9 +1702,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Ldtrmm_kernel_L2_M1_40 + ble L(dtrmm_kernel_L2_M1_40) -.Ldtrmm_kernel_L2_M1_22: +L(dtrmm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1716,22 +1716,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M1_22 + bgt L(dtrmm_kernel_L2_M1_22) -.Ldtrmm_kernel_L2_M1_40: +L(dtrmm_kernel_L2_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L2_M1_100 + ble L(dtrmm_kernel_L2_M1_100) -.Ldtrmm_kernel_L2_M1_42: +L(dtrmm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M1_42 + bgt L(dtrmm_kernel_L2_M1_42) -.Ldtrmm_kernel_L2_M1_100: +L(dtrmm_kernel_L2_M1_100): SAVE1x2 @@ -1750,7 +1750,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #1 #endif -.Ldtrmm_kernel_L2_END: +L(dtrmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -1758,11 +1758,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Ldtrmm_kernel_L1_BEGIN: +L(dtrmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldtrmm_kernel_L999 // done + ble L(dtrmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1773,14 +1773,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = A -.Ldtrmm_kernel_L1_M4_BEGIN: +L(dtrmm_kernel_L1_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Ldtrmm_kernel_L1_M2_BEGIN + ble L(dtrmm_kernel_L1_M2_BEGIN) -.Ldtrmm_kernel_L1_M4_20: +L(dtrmm_kernel_L1_M4_20): INIT4x1 @@ -1802,10 +1802,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L1_M4_40 + ble L(dtrmm_kernel_L1_M4_40) .align 5 -.Ldtrmm_kernel_L1_M4_22: +L(dtrmm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1817,22 +1817,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M4_22 + bgt L(dtrmm_kernel_L1_M4_22) -.Ldtrmm_kernel_L1_M4_40: +L(dtrmm_kernel_L1_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L1_M4_100 + ble L(dtrmm_kernel_L1_M4_100) -.Ldtrmm_kernel_L1_M4_42: +L(dtrmm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M4_42 + bgt L(dtrmm_kernel_L1_M4_42) -.Ldtrmm_kernel_L1_M4_100: +L(dtrmm_kernel_L1_M4_100): SAVE4x1 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) @@ -1851,22 +1851,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Ldtrmm_kernel_L1_M4_END: +L(dtrmm_kernel_L1_M4_END): subs counterI, counterI, #1 - bgt .Ldtrmm_kernel_L1_M4_20 + bgt L(dtrmm_kernel_L1_M4_20) -.Ldtrmm_kernel_L1_M2_BEGIN: +L(dtrmm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldtrmm_kernel_L1_END + ble L(dtrmm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L1_M1_BEGIN + ble L(dtrmm_kernel_L1_M1_BEGIN) -.Ldtrmm_kernel_L1_M2_20: +L(dtrmm_kernel_L1_M2_20): INIT2x1 @@ -1889,9 +1889,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L1_M2_40 + ble L(dtrmm_kernel_L1_M2_40) -.Ldtrmm_kernel_L1_M2_22: +L(dtrmm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1904,22 +1904,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M2_22 + bgt L(dtrmm_kernel_L1_M2_22) -.Ldtrmm_kernel_L1_M2_40: +L(dtrmm_kernel_L1_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L1_M2_100 + ble L(dtrmm_kernel_L1_M2_100) -.Ldtrmm_kernel_L1_M2_42: +L(dtrmm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M2_42 + bgt L(dtrmm_kernel_L1_M2_42) -.Ldtrmm_kernel_L1_M2_100: +L(dtrmm_kernel_L1_M2_100): SAVE2x1 @@ -1938,15 +1938,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #2 #endif -.Ldtrmm_kernel_L1_M2_END: +L(dtrmm_kernel_L1_M2_END): -.Ldtrmm_kernel_L1_M1_BEGIN: +L(dtrmm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldtrmm_kernel_L1_END + ble L(dtrmm_kernel_L1_END) -.Ldtrmm_kernel_L1_M1_20: +L(dtrmm_kernel_L1_M1_20): INIT1x1 @@ -1969,9 +1969,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L1_M1_40 + ble L(dtrmm_kernel_L1_M1_40) -.Ldtrmm_kernel_L1_M1_22: +L(dtrmm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1983,30 +1983,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M1_22 + bgt L(dtrmm_kernel_L1_M1_22) -.Ldtrmm_kernel_L1_M1_40: +L(dtrmm_kernel_L1_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L1_M1_100 + ble L(dtrmm_kernel_L1_M1_100) -.Ldtrmm_kernel_L1_M1_42: +L(dtrmm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M1_42 + bgt L(dtrmm_kernel_L1_M1_42) -.Ldtrmm_kernel_L1_M1_100: +L(dtrmm_kernel_L1_M1_100): SAVE1x1 -.Ldtrmm_kernel_L1_END: +L(dtrmm_kernel_L1_END): -.Ldtrmm_kernel_L999: +L(dtrmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dtrmm_kernel_6x8_cortexa72.S b/kernel/arm64/dtrmm_kernel_6x8_cortexa72.S index 749e58a809..5cdf668652 100644 --- a/kernel/arm64/dtrmm_kernel_6x8_cortexa72.S +++ b/kernel/arm64/dtrmm_kernel_6x8_cortexa72.S @@ -715,9 +715,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 - cbz counterJ, .Ldtrmm6x8_L4_BEGIN + cbz counterJ, L(dtrmm6x8_L4_BEGIN) -.Ldtrmm6x8_L8_BEGIN: +L(dtrmm6x8_L8_BEGIN): mov pCRow0, pC add pC, pC, LDC, lsl #3 #if defined(LEFT) @@ -727,9 +727,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov temp, #6 udiv counterI, origM, temp - cbz counterI, .Ldtrmm6x8_L8_Mrem + cbz counterI, L(dtrmm6x8_L8_Mrem) -.Ldtrmm6x8_L8_M6: +L(dtrmm6x8_L8_M6): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -751,12 +751,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT6x8 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L8_M6s -.Ldtrmm6x8_L8_M6l: + cbz counterL, L(dtrmm6x8_L8_M6s) +L(dtrmm6x8_L8_M6l): KERNEL6x8_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L8_M6l -.Ldtrmm6x8_L8_M6s: + bgt L(dtrmm6x8_L8_M6l) +L(dtrmm6x8_L8_M6s): SAVE6x8 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -777,14 +777,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterI, counterI, #1 - bgt .Ldtrmm6x8_L8_M6 + bgt L(dtrmm6x8_L8_M6) -.Ldtrmm6x8_L8_Mrem: +L(dtrmm6x8_L8_Mrem): mov temp, #6 udiv counterI, origM, temp msub counterI, counterI, temp, origM cmp counterI, #4 - blt .Ldtrmm6x8_L8_M2c + blt L(dtrmm6x8_L8_M2c) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -806,12 +806,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT4x8 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L8_M4s -.Ldtrmm6x8_L8_M4l: + cbz counterL, L(dtrmm6x8_L8_M4s) +L(dtrmm6x8_L8_M4l): KERNEL4x8_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L8_M4l -.Ldtrmm6x8_L8_M4s: + bgt L(dtrmm6x8_L8_M4l) +L(dtrmm6x8_L8_M4s): SAVE4x8 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -832,9 +832,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif sub counterI, counterI, #4 -.Ldtrmm6x8_L8_M2c: +L(dtrmm6x8_L8_M2c): cmp counterI, #2 - blt .Ldtrmm6x8_L8_M1c + blt L(dtrmm6x8_L8_M1c) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -856,12 +856,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT2x8 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L8_M2s -.Ldtrmm6x8_L8_M2l: + cbz counterL, L(dtrmm6x8_L8_M2s) +L(dtrmm6x8_L8_M2l): KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L8_M2l -.Ldtrmm6x8_L8_M2s: + bgt L(dtrmm6x8_L8_M2l) +L(dtrmm6x8_L8_M2s): SAVE2x8 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -882,8 +882,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif sub counterI, counterI, #2 -.Ldtrmm6x8_L8_M1c: - cbz counterI, .Ldtrmm6x8_L8_END +L(dtrmm6x8_L8_M1c): + cbz counterI, L(dtrmm6x8_L8_END) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -905,12 +905,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT1x8 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L8_M1s -.Ldtrmm6x8_L8_M1l: + cbz counterL, L(dtrmm6x8_L8_M1s) +L(dtrmm6x8_L8_M1l): KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L8_M1l -.Ldtrmm6x8_L8_M1s: + bgt L(dtrmm6x8_L8_M1l) +L(dtrmm6x8_L8_M1s): SAVE1x8 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -930,20 +930,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Ldtrmm6x8_L8_END: +L(dtrmm6x8_L8_END): lsl temp, origK, #6 add origPB, origPB, temp #if !defined(LEFT) add tempOffset, tempOffset, #8 #endif subs counterJ, counterJ, #1 - bgt .Ldtrmm6x8_L8_BEGIN + bgt L(dtrmm6x8_L8_BEGIN) -.Ldtrmm6x8_L4_BEGIN: +L(dtrmm6x8_L4_BEGIN): tst origN, #7 - beq .Ldtrmm6x8_L999 + beq L(dtrmm6x8_L999) tst origN, #4 - beq .Ldtrmm6x8_L2_BEGIN + beq L(dtrmm6x8_L2_BEGIN) mov pCRow0, pC add pC, pC, LDC, lsl #2 @@ -954,8 +954,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov temp, #6 udiv counterI, origM, temp - cbz counterI, .Ldtrmm6x8_L4_Mrem -.Ldtrmm6x8_L4_M6: + cbz counterI, L(dtrmm6x8_L4_Mrem) +L(dtrmm6x8_L4_M6): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -977,12 +977,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT6x4 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L4_M6s -.Ldtrmm6x8_L4_M6l: + cbz counterL, L(dtrmm6x8_L4_M6s) +L(dtrmm6x8_L4_M6l): KERNEL6x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L4_M6l -.Ldtrmm6x8_L4_M6s: + bgt L(dtrmm6x8_L4_M6l) +L(dtrmm6x8_L4_M6s): SAVE6x4 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1003,13 +1003,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterI, counterI, #1 - bgt .Ldtrmm6x8_L4_M6 -.Ldtrmm6x8_L4_Mrem: + bgt L(dtrmm6x8_L4_M6) +L(dtrmm6x8_L4_Mrem): mov temp, #6 udiv counterI, origM, temp msub counterI, counterI, temp, origM cmp counterI, #4 - blt .Ldtrmm6x8_L4_M2c + blt L(dtrmm6x8_L4_M2c) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -1031,12 +1031,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT4x4 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L4_M4s -.Ldtrmm6x8_L4_M4l: + cbz counterL, L(dtrmm6x8_L4_M4s) +L(dtrmm6x8_L4_M4l): KERNEL4x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L4_M4l -.Ldtrmm6x8_L4_M4s: + bgt L(dtrmm6x8_L4_M4l) +L(dtrmm6x8_L4_M4s): SAVE4x4 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1057,9 +1057,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif sub counterI, counterI, #4 -.Ldtrmm6x8_L4_M2c: +L(dtrmm6x8_L4_M2c): cmp counterI, #2 - blt .Ldtrmm6x8_L4_M1c + blt L(dtrmm6x8_L4_M1c) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -1081,12 +1081,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT2x4 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L4_M2s -.Ldtrmm6x8_L4_M2l: + cbz counterL, L(dtrmm6x8_L4_M2s) +L(dtrmm6x8_L4_M2l): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L4_M2l -.Ldtrmm6x8_L4_M2s: + bgt L(dtrmm6x8_L4_M2l) +L(dtrmm6x8_L4_M2s): SAVE2x4 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1107,8 +1107,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif sub counterI, counterI, #2 -.Ldtrmm6x8_L4_M1c: - cbz counterI, .Ldtrmm6x8_L4_END +L(dtrmm6x8_L4_M1c): + cbz counterI, L(dtrmm6x8_L4_END) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -1130,12 +1130,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT1x4 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L4_M1s -.Ldtrmm6x8_L4_M1l: + cbz counterL, L(dtrmm6x8_L4_M1s) +L(dtrmm6x8_L4_M1l): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L4_M1l -.Ldtrmm6x8_L4_M1s: + bgt L(dtrmm6x8_L4_M1l) +L(dtrmm6x8_L4_M1s): SAVE1x4 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1155,15 +1155,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Ldtrmm6x8_L4_END: +L(dtrmm6x8_L4_END): add origPB, origPB, origK, lsl #5 #if !defined(LEFT) add tempOffset, tempOffset, #4 #endif -.Ldtrmm6x8_L2_BEGIN: +L(dtrmm6x8_L2_BEGIN): tst origN, #2 - beq .Ldtrmm6x8_L1_BEGIN + beq L(dtrmm6x8_L1_BEGIN) mov pCRow0, pC add pC, pC, LDC, lsl #1 #if defined(LEFT) @@ -1172,8 +1172,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA mov temp, #6 udiv counterI, origM, temp - cbz counterI, .Ldtrmm6x8_L2_Mrem -.Ldtrmm6x8_L2_M6: + cbz counterI, L(dtrmm6x8_L2_Mrem) +L(dtrmm6x8_L2_M6): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -1195,12 +1195,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT6x2 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L2_M6s -.Ldtrmm6x8_L2_M6l: + cbz counterL, L(dtrmm6x8_L2_M6s) +L(dtrmm6x8_L2_M6l): KERNEL6x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L2_M6l -.Ldtrmm6x8_L2_M6s: + bgt L(dtrmm6x8_L2_M6l) +L(dtrmm6x8_L2_M6s): SAVE6x2 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1221,13 +1221,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterI, counterI, #1 - bgt .Ldtrmm6x8_L2_M6 -.Ldtrmm6x8_L2_Mrem: + bgt L(dtrmm6x8_L2_M6) +L(dtrmm6x8_L2_Mrem): mov temp, #6 udiv counterI, origM, temp msub counterI, counterI, temp, origM cmp counterI, #4 - blt .Ldtrmm6x8_L2_M2c + blt L(dtrmm6x8_L2_M2c) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -1249,12 +1249,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT4x2 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L2_M4s -.Ldtrmm6x8_L2_M4l: + cbz counterL, L(dtrmm6x8_L2_M4s) +L(dtrmm6x8_L2_M4l): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L2_M4l -.Ldtrmm6x8_L2_M4s: + bgt L(dtrmm6x8_L2_M4l) +L(dtrmm6x8_L2_M4s): SAVE4x2 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1275,9 +1275,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif sub counterI, counterI, #4 -.Ldtrmm6x8_L2_M2c: +L(dtrmm6x8_L2_M2c): cmp counterI, #2 - blt .Ldtrmm6x8_L2_M1c + blt L(dtrmm6x8_L2_M1c) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -1299,12 +1299,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT2x2 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L2_M2s -.Ldtrmm6x8_L2_M2l: + cbz counterL, L(dtrmm6x8_L2_M2s) +L(dtrmm6x8_L2_M2l): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L2_M2l -.Ldtrmm6x8_L2_M2s: + bgt L(dtrmm6x8_L2_M2l) +L(dtrmm6x8_L2_M2s): SAVE2x2 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1325,8 +1325,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif sub counterI, counterI, #2 -.Ldtrmm6x8_L2_M1c: - cbz counterI, .Ldtrmm6x8_L2_END +L(dtrmm6x8_L2_M1c): + cbz counterI, L(dtrmm6x8_L2_END) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -1348,12 +1348,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT1x2 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L2_M1s -.Ldtrmm6x8_L2_M1l: + cbz counterL, L(dtrmm6x8_L2_M1s) +L(dtrmm6x8_L2_M1l): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L2_M1l -.Ldtrmm6x8_L2_M1s: + bgt L(dtrmm6x8_L2_M1l) +L(dtrmm6x8_L2_M1s): SAVE1x2 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1373,15 +1373,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Ldtrmm6x8_L2_END: +L(dtrmm6x8_L2_END): add origPB, origPB, origK, lsl #4 #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif -.Ldtrmm6x8_L1_BEGIN: +L(dtrmm6x8_L1_BEGIN): tst origN, #1 - beq .Ldtrmm6x8_L999 + beq L(dtrmm6x8_L999) mov pCRow0, pC #if defined(LEFT) mov tempOffset, offset @@ -1389,8 +1389,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA mov temp, #6 udiv counterI, origM, temp - cbz counterI, .Ldtrmm6x8_L1_Mrem -.Ldtrmm6x8_L1_M6: + cbz counterI, L(dtrmm6x8_L1_Mrem) +L(dtrmm6x8_L1_M6): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -1412,12 +1412,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT6x1 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L1_M6s -.Ldtrmm6x8_L1_M6l: + cbz counterL, L(dtrmm6x8_L1_M6s) +L(dtrmm6x8_L1_M6l): KERNEL6x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L1_M6l -.Ldtrmm6x8_L1_M6s: + bgt L(dtrmm6x8_L1_M6l) +L(dtrmm6x8_L1_M6s): SAVE6x1 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1438,13 +1438,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterI, counterI, #1 - bgt .Ldtrmm6x8_L1_M6 -.Ldtrmm6x8_L1_Mrem: + bgt L(dtrmm6x8_L1_M6) +L(dtrmm6x8_L1_Mrem): mov temp, #6 udiv counterI, origM, temp msub counterI, counterI, temp, origM cmp counterI, #4 - blt .Ldtrmm6x8_L1_M2c + blt L(dtrmm6x8_L1_M2c) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -1466,12 +1466,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT4x1 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L1_M4s -.Ldtrmm6x8_L1_M4l: + cbz counterL, L(dtrmm6x8_L1_M4s) +L(dtrmm6x8_L1_M4l): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L1_M4l -.Ldtrmm6x8_L1_M4s: + bgt L(dtrmm6x8_L1_M4l) +L(dtrmm6x8_L1_M4s): SAVE4x1 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1492,9 +1492,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif sub counterI, counterI, #4 -.Ldtrmm6x8_L1_M2c: +L(dtrmm6x8_L1_M2c): cmp counterI, #2 - blt .Ldtrmm6x8_L1_M1c + blt L(dtrmm6x8_L1_M1c) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -1516,12 +1516,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT2x1 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L1_M2s -.Ldtrmm6x8_L1_M2l: + cbz counterL, L(dtrmm6x8_L1_M2s) +L(dtrmm6x8_L1_M2l): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L1_M2l -.Ldtrmm6x8_L1_M2s: + bgt L(dtrmm6x8_L1_M2l) +L(dtrmm6x8_L1_M2s): SAVE2x1 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1542,8 +1542,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif sub counterI, counterI, #2 -.Ldtrmm6x8_L1_M1c: - cbz counterI, .Ldtrmm6x8_L999 +L(dtrmm6x8_L1_M1c): + cbz counterI, L(dtrmm6x8_L999) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -1565,12 +1565,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT1x1 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L1_M1s -.Ldtrmm6x8_L1_M1l: + cbz counterL, L(dtrmm6x8_L1_M1s) +L(dtrmm6x8_L1_M1l): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L1_M1l -.Ldtrmm6x8_L1_M1s: + bgt L(dtrmm6x8_L1_M1l) +L(dtrmm6x8_L1_M1s): SAVE1x1 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1590,7 +1590,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Ldtrmm6x8_L999: +L(dtrmm6x8_L999): mov x0, #0 ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dtrmm_kernel_8x4.S b/kernel/arm64/dtrmm_kernel_8x4.S index 3d953266ca..45c861dff7 100644 --- a/kernel/arm64/dtrmm_kernel_8x4.S +++ b/kernel/arm64/dtrmm_kernel_8x4.S @@ -830,11 +830,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Ldtrmm_kernel_L2_BEGIN + ble L(dtrmm_kernel_L2_BEGIN) /******************************************************************************/ -.Ldtrmm_kernel_L4_BEGIN: +L(dtrmm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -848,15 +848,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = start of A array -.Ldtrmm_kernel_L4_M8_BEGIN: +L(dtrmm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Ldtrmm_kernel_L4_M4_BEGIN + ble L(dtrmm_kernel_L4_M4_BEGIN) .align 5 -.Ldtrmm_kernel_L4_M8_20: +L(dtrmm_kernel_L4_M8_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -878,7 +878,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Ldtrmm_kernel_L4_M8_32 + blt L(dtrmm_kernel_L4_M8_32) KERNEL8x4_I // do one in the K KERNEL8x4_M2 // do another in the K @@ -890,10 +890,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M2 subs counterL, counterL, #2 // subtract 2 - ble .Ldtrmm_kernel_L4_M8_22a + ble L(dtrmm_kernel_L4_M8_22a) .align 5 -.Ldtrmm_kernel_L4_M8_22: +L(dtrmm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 @@ -905,10 +905,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M8_22 + bgt L(dtrmm_kernel_L4_M8_22) .align 5 -.Ldtrmm_kernel_L4_M8_22a: +L(dtrmm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_M2 @@ -919,13 +919,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Ldtrmm_kernel_L4_M8_44 + b L(dtrmm_kernel_L4_M8_44) .align 5 -.Ldtrmm_kernel_L4_M8_32: +L(dtrmm_kernel_L4_M8_32): tst counterL, #1 - ble .Ldtrmm_kernel_L4_M8_40 + ble L(dtrmm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_M2 @@ -936,26 +936,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Ldtrmm_kernel_L4_M8_44 + b L(dtrmm_kernel_L4_M8_44) -.Ldtrmm_kernel_L4_M8_40: +L(dtrmm_kernel_L4_M8_40): INIT8x4 -.Ldtrmm_kernel_L4_M8_44: +L(dtrmm_kernel_L4_M8_44): ands counterL , tempK, #7 - ble .Ldtrmm_kernel_L4_M8_100 + ble L(dtrmm_kernel_L4_M8_100) .align 5 -.Ldtrmm_kernel_L4_M8_46: +L(dtrmm_kernel_L4_M8_46): KERNEL8x4_SUB subs counterL, counterL, #1 - bne .Ldtrmm_kernel_L4_M8_46 + bne L(dtrmm_kernel_L4_M8_46) -.Ldtrmm_kernel_L4_M8_100: +L(dtrmm_kernel_L4_M8_100): SAVE8x4 @@ -978,20 +978,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] -.Ldtrmm_kernel_L4_M8_END: +L(dtrmm_kernel_L4_M8_END): subs counterI, counterI, #1 - bne .Ldtrmm_kernel_L4_M8_20 + bne L(dtrmm_kernel_L4_M8_20) -.Ldtrmm_kernel_L4_M4_BEGIN: +L(dtrmm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Ldtrmm_kernel_L4_END + ble L(dtrmm_kernel_L4_END) tst counterI, #4 - ble .Ldtrmm_kernel_L4_M2_BEGIN + ble L(dtrmm_kernel_L4_M2_BEGIN) -.Ldtrmm_kernel_L4_M4_20: +L(dtrmm_kernel_L4_M4_20): INIT4x4 @@ -1014,9 +1014,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L4_M4_40 + ble L(dtrmm_kernel_L4_M4_40) -.Ldtrmm_kernel_L4_M4_22: +L(dtrmm_kernel_L4_M4_22): KERNEL4x4_SUB KERNEL4x4_SUB @@ -1029,22 +1029,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M4_22 + bgt L(dtrmm_kernel_L4_M4_22) -.Ldtrmm_kernel_L4_M4_40: +L(dtrmm_kernel_L4_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L4_M4_100 + ble L(dtrmm_kernel_L4_M4_100) -.Ldtrmm_kernel_L4_M4_42: +L(dtrmm_kernel_L4_M4_42): KERNEL4x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M4_42 + bgt L(dtrmm_kernel_L4_M4_42) -.Ldtrmm_kernel_L4_M4_100: +L(dtrmm_kernel_L4_M4_100): SAVE4x4 @@ -1063,19 +1063,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Ldtrmm_kernel_L4_M4_END: +L(dtrmm_kernel_L4_M4_END): -.Ldtrmm_kernel_L4_M2_BEGIN: +L(dtrmm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldtrmm_kernel_L4_END + ble L(dtrmm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L4_M1_BEGIN + ble L(dtrmm_kernel_L4_M1_BEGIN) -.Ldtrmm_kernel_L4_M2_20: +L(dtrmm_kernel_L4_M2_20): INIT2x4 @@ -1098,9 +1098,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L4_M2_40 + ble L(dtrmm_kernel_L4_M2_40) -.Ldtrmm_kernel_L4_M2_22: +L(dtrmm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1113,22 +1113,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M2_22 + bgt L(dtrmm_kernel_L4_M2_22) -.Ldtrmm_kernel_L4_M2_40: +L(dtrmm_kernel_L4_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L4_M2_100 + ble L(dtrmm_kernel_L4_M2_100) -.Ldtrmm_kernel_L4_M2_42: +L(dtrmm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M2_42 + bgt L(dtrmm_kernel_L4_M2_42) -.Ldtrmm_kernel_L4_M2_100: +L(dtrmm_kernel_L4_M2_100): SAVE2x4 @@ -1148,15 +1148,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Ldtrmm_kernel_L4_M2_END: +L(dtrmm_kernel_L4_M2_END): -.Ldtrmm_kernel_L4_M1_BEGIN: +L(dtrmm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldtrmm_kernel_L4_END + ble L(dtrmm_kernel_L4_END) -.Ldtrmm_kernel_L4_M1_20: +L(dtrmm_kernel_L4_M1_20): INIT1x4 @@ -1180,9 +1180,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L4_M1_40 + ble L(dtrmm_kernel_L4_M1_40) -.Ldtrmm_kernel_L4_M1_22: +L(dtrmm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1194,22 +1194,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M1_22 + bgt L(dtrmm_kernel_L4_M1_22) -.Ldtrmm_kernel_L4_M1_40: +L(dtrmm_kernel_L4_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L4_M1_100 + ble L(dtrmm_kernel_L4_M1_100) -.Ldtrmm_kernel_L4_M1_42: +L(dtrmm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M1_42 + bgt L(dtrmm_kernel_L4_M1_42) -.Ldtrmm_kernel_L4_M1_100: +L(dtrmm_kernel_L4_M1_100): SAVE1x4 @@ -1229,7 +1229,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Ldtrmm_kernel_L4_END: +L(dtrmm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 @@ -1239,19 +1239,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Ldtrmm_kernel_L4_BEGIN + bgt L(dtrmm_kernel_L4_BEGIN) /******************************************************************************/ -.Ldtrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(dtrmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Ldtrmm_kernel_L999 // error, N was less than 4? + ble L(dtrmm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Ldtrmm_kernel_L1_BEGIN + ble L(dtrmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1262,14 +1262,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = A -.Ldtrmm_kernel_L2_M8_BEGIN: +L(dtrmm_kernel_L2_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Ldtrmm_kernel_L2_M4_BEGIN + ble L(dtrmm_kernel_L2_M4_BEGIN) -.Ldtrmm_kernel_L2_M8_20: +L(dtrmm_kernel_L2_M8_20): INIT8x2 @@ -1293,10 +1293,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldtrmm_kernel_L2_M8_40 + ble L(dtrmm_kernel_L2_M8_40) .align 5 -.Ldtrmm_kernel_L2_M8_22: +L(dtrmm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB KERNEL8x2_SUB @@ -1308,22 +1308,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M8_22 + bgt L(dtrmm_kernel_L2_M8_22) -.Ldtrmm_kernel_L2_M8_40: +L(dtrmm_kernel_L2_M8_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L2_M8_100 + ble L(dtrmm_kernel_L2_M8_100) -.Ldtrmm_kernel_L2_M8_42: +L(dtrmm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M8_42 + bgt L(dtrmm_kernel_L2_M8_42) -.Ldtrmm_kernel_L2_M8_100: +L(dtrmm_kernel_L2_M8_100): SAVE8x2 @@ -1343,21 +1343,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #8 #endif -.Ldtrmm_kernel_L2_M8_END: +L(dtrmm_kernel_L2_M8_END): subs counterI, counterI, #1 - bgt .Ldtrmm_kernel_L2_M8_20 + bgt L(dtrmm_kernel_L2_M8_20) -.Ldtrmm_kernel_L2_M4_BEGIN: +L(dtrmm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Ldtrmm_kernel_L2_END + ble L(dtrmm_kernel_L2_END) tst counterI, #4 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L2_M2_BEGIN + ble L(dtrmm_kernel_L2_M2_BEGIN) -.Ldtrmm_kernel_L2_M4_20: +L(dtrmm_kernel_L2_M4_20): INIT4x2 @@ -1381,10 +1381,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldtrmm_kernel_L2_M4_40 + ble L(dtrmm_kernel_L2_M4_40) .align 5 -.Ldtrmm_kernel_L2_M4_22: +L(dtrmm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1396,22 +1396,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M4_22 + bgt L(dtrmm_kernel_L2_M4_22) -.Ldtrmm_kernel_L2_M4_40: +L(dtrmm_kernel_L2_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L2_M4_100 + ble L(dtrmm_kernel_L2_M4_100) -.Ldtrmm_kernel_L2_M4_42: +L(dtrmm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M4_42 + bgt L(dtrmm_kernel_L2_M4_42) -.Ldtrmm_kernel_L2_M4_100: +L(dtrmm_kernel_L2_M4_100): SAVE4x2 @@ -1431,19 +1431,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Ldtrmm_kernel_L2_M4_END: +L(dtrmm_kernel_L2_M4_END): -.Ldtrmm_kernel_L2_M2_BEGIN: +L(dtrmm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldtrmm_kernel_L2_END + ble L(dtrmm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L2_M1_BEGIN + ble L(dtrmm_kernel_L2_M1_BEGIN) -.Ldtrmm_kernel_L2_M2_20: +L(dtrmm_kernel_L2_M2_20): INIT2x2 @@ -1467,9 +1467,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldtrmm_kernel_L2_M2_40 + ble L(dtrmm_kernel_L2_M2_40) -.Ldtrmm_kernel_L2_M2_22: +L(dtrmm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1482,22 +1482,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M2_22 + bgt L(dtrmm_kernel_L2_M2_22) -.Ldtrmm_kernel_L2_M2_40: +L(dtrmm_kernel_L2_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L2_M2_100 + ble L(dtrmm_kernel_L2_M2_100) -.Ldtrmm_kernel_L2_M2_42: +L(dtrmm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M2_42 + bgt L(dtrmm_kernel_L2_M2_42) -.Ldtrmm_kernel_L2_M2_100: +L(dtrmm_kernel_L2_M2_100): SAVE2x2 @@ -1517,15 +1517,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Ldtrmm_kernel_L2_M2_END: +L(dtrmm_kernel_L2_M2_END): -.Ldtrmm_kernel_L2_M1_BEGIN: +L(dtrmm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldtrmm_kernel_L2_END + ble L(dtrmm_kernel_L2_END) -.Ldtrmm_kernel_L2_M1_20: +L(dtrmm_kernel_L2_M1_20): INIT1x2 @@ -1549,9 +1549,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Ldtrmm_kernel_L2_M1_40 + ble L(dtrmm_kernel_L2_M1_40) -.Ldtrmm_kernel_L2_M1_22: +L(dtrmm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1563,22 +1563,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M1_22 + bgt L(dtrmm_kernel_L2_M1_22) -.Ldtrmm_kernel_L2_M1_40: +L(dtrmm_kernel_L2_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L2_M1_100 + ble L(dtrmm_kernel_L2_M1_100) -.Ldtrmm_kernel_L2_M1_42: +L(dtrmm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M1_42 + bgt L(dtrmm_kernel_L2_M1_42) -.Ldtrmm_kernel_L2_M1_100: +L(dtrmm_kernel_L2_M1_100): SAVE1x2 @@ -1598,7 +1598,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Ldtrmm_kernel_L2_END: +L(dtrmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -1606,11 +1606,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Ldtrmm_kernel_L1_BEGIN: +L(dtrmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldtrmm_kernel_L999 // done + ble L(dtrmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C add pC , pC , LDC // Update pC to point to next @@ -1620,14 +1620,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = A -.Ldtrmm_kernel_L1_M8_BEGIN: +L(dtrmm_kernel_L1_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Ldtrmm_kernel_L1_M4_BEGIN + ble L(dtrmm_kernel_L1_M4_BEGIN) -.Ldtrmm_kernel_L1_M8_20: +L(dtrmm_kernel_L1_M8_20): INIT8x1 @@ -1651,10 +1651,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L1_M8_40 + ble L(dtrmm_kernel_L1_M8_40) .align 5 -.Ldtrmm_kernel_L1_M8_22: +L(dtrmm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -1666,22 +1666,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M8_22 + bgt L(dtrmm_kernel_L1_M8_22) -.Ldtrmm_kernel_L1_M8_40: +L(dtrmm_kernel_L1_M8_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L1_M8_100 + ble L(dtrmm_kernel_L1_M8_100) -.Ldtrmm_kernel_L1_M8_42: +L(dtrmm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M8_42 + bgt L(dtrmm_kernel_L1_M8_42) -.Ldtrmm_kernel_L1_M8_100: +L(dtrmm_kernel_L1_M8_100): SAVE8x1 @@ -1701,21 +1701,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #8 #endif -.Ldtrmm_kernel_L1_M8_END: +L(dtrmm_kernel_L1_M8_END): subs counterI, counterI, #1 - bgt .Ldtrmm_kernel_L1_M8_20 + bgt L(dtrmm_kernel_L1_M8_20) -.Ldtrmm_kernel_L1_M4_BEGIN: +L(dtrmm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Ldtrmm_kernel_L1_END + ble L(dtrmm_kernel_L1_END) tst counterI, #4 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L1_M2_BEGIN + ble L(dtrmm_kernel_L1_M2_BEGIN) -.Ldtrmm_kernel_L1_M4_20: +L(dtrmm_kernel_L1_M4_20): INIT4x1 @@ -1738,10 +1738,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L1_M4_40 + ble L(dtrmm_kernel_L1_M4_40) .align 5 -.Ldtrmm_kernel_L1_M4_22: +L(dtrmm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1753,22 +1753,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M4_22 + bgt L(dtrmm_kernel_L1_M4_22) -.Ldtrmm_kernel_L1_M4_40: +L(dtrmm_kernel_L1_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L1_M4_100 + ble L(dtrmm_kernel_L1_M4_100) -.Ldtrmm_kernel_L1_M4_42: +L(dtrmm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M4_42 + bgt L(dtrmm_kernel_L1_M4_42) -.Ldtrmm_kernel_L1_M4_100: +L(dtrmm_kernel_L1_M4_100): SAVE4x1 @@ -1788,18 +1788,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Ldtrmm_kernel_L1_M4_END: +L(dtrmm_kernel_L1_M4_END): -.Ldtrmm_kernel_L1_M2_BEGIN: +L(dtrmm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldtrmm_kernel_L1_END + ble L(dtrmm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L1_M1_BEGIN + ble L(dtrmm_kernel_L1_M1_BEGIN) -.Ldtrmm_kernel_L1_M2_20: +L(dtrmm_kernel_L1_M2_20): INIT2x1 @@ -1823,9 +1823,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L1_M2_40 + ble L(dtrmm_kernel_L1_M2_40) -.Ldtrmm_kernel_L1_M2_22: +L(dtrmm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1838,22 +1838,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M2_22 + bgt L(dtrmm_kernel_L1_M2_22) -.Ldtrmm_kernel_L1_M2_40: +L(dtrmm_kernel_L1_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L1_M2_100 + ble L(dtrmm_kernel_L1_M2_100) -.Ldtrmm_kernel_L1_M2_42: +L(dtrmm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M2_42 + bgt L(dtrmm_kernel_L1_M2_42) -.Ldtrmm_kernel_L1_M2_100: +L(dtrmm_kernel_L1_M2_100): SAVE2x1 @@ -1873,15 +1873,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Ldtrmm_kernel_L1_M2_END: +L(dtrmm_kernel_L1_M2_END): -.Ldtrmm_kernel_L1_M1_BEGIN: +L(dtrmm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldtrmm_kernel_L1_END + ble L(dtrmm_kernel_L1_END) -.Ldtrmm_kernel_L1_M1_20: +L(dtrmm_kernel_L1_M1_20): INIT1x1 @@ -1905,9 +1905,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L1_M1_40 + ble L(dtrmm_kernel_L1_M1_40) -.Ldtrmm_kernel_L1_M1_22: +L(dtrmm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1919,30 +1919,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M1_22 + bgt L(dtrmm_kernel_L1_M1_22) -.Ldtrmm_kernel_L1_M1_40: +L(dtrmm_kernel_L1_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L1_M1_100 + ble L(dtrmm_kernel_L1_M1_100) -.Ldtrmm_kernel_L1_M1_42: +L(dtrmm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M1_42 + bgt L(dtrmm_kernel_L1_M1_42) -.Ldtrmm_kernel_L1_M1_100: +L(dtrmm_kernel_L1_M1_100): SAVE1x1 -.Ldtrmm_kernel_L1_END: +L(dtrmm_kernel_L1_END): -.Ldtrmm_kernel_L999: +L(dtrmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dtrmm_kernel_sve_v1x8.S b/kernel/arm64/dtrmm_kernel_sve_v1x8.S index 1f8c9b20fa..644d227138 100644 --- a/kernel/arm64/dtrmm_kernel_sve_v1x8.S +++ b/kernel/arm64/dtrmm_kernel_sve_v1x8.S @@ -472,13 +472,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Ldtrmm_kernel_L4_BEGIN + ble L(dtrmm_kernel_L4_BEGIN) /******************************************************************************/ /* Repeat this as long as there are 8 left in N */ .align 5 -.Ldtrmm_kernel_L8_BEGIN: +L(dtrmm_kernel_L8_BEGIN): mov pCRow0, pC add pC, pC, LDC, lsl #3 // add 8 x LDC @@ -489,7 +489,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldtrmm_kernel_L8_Mv1_BEGIN: +L(dtrmm_kernel_L8_Mv1_BEGIN): /* Loop over M is done in an SVE fashion. This has the benefit of the last M%SVE_LEN iterations being done in a single sweep */ mov counterI, #0 @@ -497,7 +497,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension .align 5 -.Ldtrmm_kernel_L8_Mv1_20: +L(dtrmm_kernel_L8_Mv1_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -521,7 +521,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Ldtrmm_kernel_L8_Mv1_32 + blt L(dtrmm_kernel_L8_Mv1_32) KERNELv1x8_I KERNELv1x8_M2 @@ -533,10 +533,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Ldtrmm_kernel_L8_Mv1_22a + ble L(dtrmm_kernel_L8_Mv1_22a) .align 5 -.Ldtrmm_kernel_L8_Mv1_22: +L(dtrmm_kernel_L8_Mv1_22): KERNELv1x8_M1 KERNELv1x8_M2 @@ -548,10 +548,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M2 subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L8_Mv1_22 + bgt L(dtrmm_kernel_L8_Mv1_22) .align 5 -.Ldtrmm_kernel_L8_Mv1_22a: +L(dtrmm_kernel_L8_Mv1_22a): KERNELv1x8_M1 KERNELv1x8_M2 @@ -562,13 +562,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M1 KERNELv1x8_E - b .Ldtrmm_kernel_L8_Mv1_44 + b L(dtrmm_kernel_L8_Mv1_44) .align 5 -.Ldtrmm_kernel_L8_Mv1_32: +L(dtrmm_kernel_L8_Mv1_32): tst counterL, #1 - ble .Ldtrmm_kernel_L8_Mv1_40 + ble L(dtrmm_kernel_L8_Mv1_40) KERNELv1x8_I KERNELv1x8_M2 @@ -580,26 +580,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_E - b .Ldtrmm_kernel_L8_Mv1_44 + b L(dtrmm_kernel_L8_Mv1_44) -.Ldtrmm_kernel_L8_Mv1_40: +L(dtrmm_kernel_L8_Mv1_40): INITv1x8 -.Ldtrmm_kernel_L8_Mv1_44: +L(dtrmm_kernel_L8_Mv1_44): ands counterL , tempK, #7 - ble .Ldtrmm_kernel_L8_Mv1_100 + ble L(dtrmm_kernel_L8_Mv1_100) .align 5 -.Ldtrmm_kernel_L8_Mv1_46: +L(dtrmm_kernel_L8_Mv1_46): KERNELv1x8_SUB subs counterL, counterL, #1 - bne .Ldtrmm_kernel_L8_Mv1_46 + bne L(dtrmm_kernel_L8_Mv1_46) -.Ldtrmm_kernel_L8_Mv1_100: +L(dtrmm_kernel_L8_Mv1_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] @@ -622,14 +622,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, lanes #endif -.Ldtrmm_kernel_L8_Mv1_END: +L(dtrmm_kernel_L8_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldtrmm_kernel_L8_Mv1_20 + b.any L(dtrmm_kernel_L8_Mv1_20) -.Ldtrmm_kernel_L8_END: +L(dtrmm_kernel_L8_END): lsl temp, origK, #6 add origPB, origPB, temp // B = B + K * 8 * 8 @@ -639,17 +639,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Ldtrmm_kernel_L8_BEGIN + bgt L(dtrmm_kernel_L8_BEGIN) /******************************************************************************/ /* Repeat the same thing if 4 left in N */ .align 5 -.Ldtrmm_kernel_L4_BEGIN: +L(dtrmm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #4 - ble .Ldtrmm_kernel_L2_BEGIN + ble L(dtrmm_kernel_L2_BEGIN) #if defined(LEFT) mov tempOffset, offset @@ -661,14 +661,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldtrmm_kernel_L4_Mv1_BEGIN: +L(dtrmm_kernel_L4_Mv1_BEGIN): mov counterI, #0 whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d .align 5 -.Ldtrmm_kernel_L4_Mv1_20: +L(dtrmm_kernel_L4_Mv1_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -692,10 +692,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldtrmm_kernel_L4_Mv1_44 + ble L(dtrmm_kernel_L4_Mv1_44) .align 5 -.Ldtrmm_kernel_L4_Mv1_22: +L(dtrmm_kernel_L4_Mv1_22): KERNELv1x4_SUB KERNELv1x4_SUB @@ -707,22 +707,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_Mv1_22 + bgt L(dtrmm_kernel_L4_Mv1_22) -.Ldtrmm_kernel_L4_Mv1_44: +L(dtrmm_kernel_L4_Mv1_44): ands counterL , tempK, #7 - ble .Ldtrmm_kernel_L4_Mv1_100 + ble L(dtrmm_kernel_L4_Mv1_100) .align 5 -.Ldtrmm_kernel_L4_Mv1_46: +L(dtrmm_kernel_L4_Mv1_46): KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Ldtrmm_kernel_L4_Mv1_46 + bne L(dtrmm_kernel_L4_Mv1_46) -.Ldtrmm_kernel_L4_Mv1_100: +L(dtrmm_kernel_L4_Mv1_100): SAVEv1x4 @@ -742,15 +742,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, lanes #endif -.Ldtrmm_kernel_L4_Mv1_END: +L(dtrmm_kernel_L4_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldtrmm_kernel_L4_Mv1_20 + b.any L(dtrmm_kernel_L4_Mv1_20) -.Ldtrmm_kernel_L4_END: +L(dtrmm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 #if !defined(LEFT) @@ -761,11 +761,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /* Repeat the same thing if 2 left in N */ .align 5 -.Ldtrmm_kernel_L2_BEGIN: +L(dtrmm_kernel_L2_BEGIN): mov counterJ , origN tst counterJ , #2 - ble .Ldtrmm_kernel_L1_BEGIN + ble L(dtrmm_kernel_L1_BEGIN) mov pCRow0, pC @@ -777,14 +777,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldtrmm_kernel_L2_Mv1_BEGIN: +L(dtrmm_kernel_L2_Mv1_BEGIN): mov counterI, #0 whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d .align 5 -.Ldtrmm_kernel_L2_Mv1_20: +L(dtrmm_kernel_L2_Mv1_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -808,10 +808,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldtrmm_kernel_L2_Mv1_44 + ble L(dtrmm_kernel_L2_Mv1_44) .align 5 -.Ldtrmm_kernel_L2_Mv1_22: +L(dtrmm_kernel_L2_Mv1_22): KERNELv1x2_SUB KERNELv1x2_SUB @@ -823,22 +823,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_Mv1_22 + bgt L(dtrmm_kernel_L2_Mv1_22) -.Ldtrmm_kernel_L2_Mv1_44: +L(dtrmm_kernel_L2_Mv1_44): ands counterL , tempK, #7 - ble .Ldtrmm_kernel_L2_Mv1_100 + ble L(dtrmm_kernel_L2_Mv1_100) .align 5 -.Ldtrmm_kernel_L2_Mv1_46: +L(dtrmm_kernel_L2_Mv1_46): KERNELv1x2_SUB subs counterL, counterL, #1 - bne .Ldtrmm_kernel_L2_Mv1_46 + bne L(dtrmm_kernel_L2_Mv1_46) -.Ldtrmm_kernel_L2_Mv1_100: +L(dtrmm_kernel_L2_Mv1_100): SAVEv1x2 @@ -859,15 +859,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.Ldtrmm_kernel_L2_Mv1_END: +L(dtrmm_kernel_L2_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldtrmm_kernel_L2_Mv1_20 + b.any L(dtrmm_kernel_L2_Mv1_20) -.Ldtrmm_kernel_L2_END: +L(dtrmm_kernel_L2_END): add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 #if !defined(LEFT) add tempOffset, tempOffset, #2 @@ -877,11 +877,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /* Repeat the same thing if 1 left in N */ .align 5 -.Ldtrmm_kernel_L1_BEGIN: +L(dtrmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldtrmm_kernel_L999 // done + ble L(dtrmm_kernel_L999) // done mov pCRow0, pC @@ -893,14 +893,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldtrmm_kernel_L1_Mv1_BEGIN: +L(dtrmm_kernel_L1_Mv1_BEGIN): mov counterI, #0 whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d .align 5 -.Ldtrmm_kernel_L1_Mv1_20: +L(dtrmm_kernel_L1_Mv1_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -924,10 +924,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Ldtrmm_kernel_L1_Mv1_44 + ble L(dtrmm_kernel_L1_Mv1_44) .align 5 -.Ldtrmm_kernel_L1_Mv1_22: +L(dtrmm_kernel_L1_Mv1_22): KERNELv1x1_SUB KERNELv1x1_SUB @@ -939,22 +939,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_Mv1_22 + bgt L(dtrmm_kernel_L1_Mv1_22) -.Ldtrmm_kernel_L1_Mv1_44: +L(dtrmm_kernel_L1_Mv1_44): ands counterL , tempK, #7 - ble .Ldtrmm_kernel_L1_Mv1_100 + ble L(dtrmm_kernel_L1_Mv1_100) .align 5 -.Ldtrmm_kernel_L1_Mv1_46: +L(dtrmm_kernel_L1_Mv1_46): KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_Mv1_46 + bgt L(dtrmm_kernel_L1_Mv1_46) -.Ldtrmm_kernel_L1_Mv1_100: +L(dtrmm_kernel_L1_Mv1_100): SAVEv1x1 @@ -976,19 +976,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Ldtrmm_kernel_L1_Mv1_END: +L(dtrmm_kernel_L1_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldtrmm_kernel_L1_Mv1_20 + b.any L(dtrmm_kernel_L1_Mv1_20) -.Ldtrmm_kernel_L1_END: +L(dtrmm_kernel_L1_END): /******************************************************************************/ -.Ldtrmm_kernel_L999: +L(dtrmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dznrm2_thunderx2t99_fast.c b/kernel/arm64/dznrm2_thunderx2t99_fast.c index 90558c2a02..37339cd847 100644 --- a/kernel/arm64/dznrm2_thunderx2t99_fast.c +++ b/kernel/arm64/dznrm2_thunderx2t99_fast.c @@ -44,6 +44,12 @@ extern int blas_level1_thread_with_return_value(int mode, BLASLONG m, BLASLONG n #define TMPF "d16" #define SSQ "d0" +#if defined(__APPLE__) +#define L(x) "L" #x +#else +#define L(x) ".L" #x +#endif + #if !defined(COMPLEX) #define N_DIV_SHIFT "5" #define N_REM_MASK "31" @@ -143,62 +149,62 @@ static double nrm2_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x) " fmov d6, xzr \n" " fmov d7, xzr \n" " cmp "N", xzr \n" - " ble .Lnrm2_kernel_L999 \n" + " ble "L(nrm2_kernel_L999)" \n" " cmp "INC_X", xzr \n" - " ble .Lnrm2_kernel_L999 \n" + " ble "L(nrm2_kernel_L999)" \n" " cmp "INC_X", #1 \n" - " bne .Lnrm2_kernel_S_BEGIN \n" + " bne "L(nrm2_kernel_S_BEGIN)" \n" - ".Lnrm2_kernel_F_BEGIN: \n" + L(nrm2_kernel_F_BEGIN)": \n" " lsl "INC_X", "INC_X", #"INC_SHIFT" \n" " asr "J", "N", #"N_DIV_SHIFT" \n" " cmp "J", xzr \n" - " beq .Lnrm2_kernel_F1 \n" + " beq "L(nrm2_kernel_F1)" \n" /* https://github.com/llvm/llvm-project/issues/149547 */ #if !(defined(__clang__) && defined(OS_WINDOWS)) " .align 5 \n" #endif - ".Lnrm2_kernel_F: \n" + L(nrm2_kernel_F)": \n" " "KERNEL_F" \n" " subs "J", "J", #1 \n" - " bne .Lnrm2_kernel_F \n" + " bne "L(nrm2_kernel_F)" \n" " "KERNEL_F_FINALIZE" \n" - ".Lnrm2_kernel_F1: \n" + L(nrm2_kernel_F1)": \n" " ands "J", "N", #"N_REM_MASK" \n" - " ble .Lnrm2_kernel_L999 \n" + " ble "L(nrm2_kernel_L999)" \n" - ".Lnrm2_kernel_F10: \n" + L(nrm2_kernel_F10)": \n" " "KERNEL_F1" \n" " subs "J", "J", #1 \n" - " bne .Lnrm2_kernel_F10 \n" - " b .Lnrm2_kernel_L999 \n" + " bne "L(nrm2_kernel_F10)" \n" + " b "L(nrm2_kernel_L999)" \n" - ".Lnrm2_kernel_S_BEGIN: \n" + L(nrm2_kernel_S_BEGIN)": \n" " lsl "INC_X", "INC_X", #"INC_SHIFT" \n" " asr "J", "N", #2 \n" " cmp "J", xzr \n" - " ble .Lnrm2_kernel_S1 \n" + " ble "L(nrm2_kernel_S1)" \n" - ".Lnrm2_kernel_S4: \n" + L(nrm2_kernel_S4)": \n" " "KERNEL_F1" \n" " "KERNEL_F1" \n" " "KERNEL_F1" \n" " "KERNEL_F1" \n" " subs "J", "J", #1 \n" - " bne .Lnrm2_kernel_S4 \n" + " bne "L(nrm2_kernel_S4)" \n" - ".Lnrm2_kernel_S1: \n" + L(nrm2_kernel_S1)": \n" " ands "J", "N", #3 \n" - " ble .Lnrm2_kernel_L999 \n" + " ble "L(nrm2_kernel_L999)" \n" - ".Lnrm2_kernel_S10: \n" + L(nrm2_kernel_S10)": \n" " "KERNEL_F1" \n" " subs "J", "J", #1 \n" - " bne .Lnrm2_kernel_S10 \n" + " bne "L(nrm2_kernel_S10)" \n" - ".Lnrm2_kernel_L999: \n" + L(nrm2_kernel_L999)": \n" " "KERNEL_FINALIZE" \n" " str "SSQ", [%[RET_]] \n" diff --git a/kernel/arm64/gemv_n.S b/kernel/arm64/gemv_n.S index 658551f4f0..ef791e22e7 100644 --- a/kernel/arm64/gemv_n.S +++ b/kernel/arm64/gemv_n.S @@ -203,18 +203,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SAVE_REGS cmp N, xzr - ble .Lgemv_n_kernel_L999 + ble L(gemv_n_kernel_L999) cmp M, xzr - ble .Lgemv_n_kernel_L999 + ble L(gemv_n_kernel_L999) lsl LDA, LDA, #SHZ lsl INC_X, INC_X, #SHZ mov J, N cmp INC_Y, #1 - bne .Lgemv_n_kernel_S_BEGIN + bne L(gemv_n_kernel_S_BEGIN) -.Lgemv_n_kernel_F_LOOP: +L(gemv_n_kernel_F_LOOP): ld1 TEMPV, [X], INC_X fmul TEMP, ALPHA, TEMP @@ -229,57 +229,57 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov Y_IPTR, Y mov Y_OPTR, Y -.Lgemv_n_kernel_F32: +L(gemv_n_kernel_F32): asr I, M, #5 cmp I, xzr - beq .Lgemv_n_kernel_F4 + beq L(gemv_n_kernel_F4) -.Lgemv_n_kernel_F320: +L(gemv_n_kernel_F320): KERNEL_F16 KERNEL_F16 subs I, I, #1 - bne .Lgemv_n_kernel_F320 + bne L(gemv_n_kernel_F320) -.Lgemv_n_kernel_F4: +L(gemv_n_kernel_F4): ands I, M, #31 asr I, I, #2 cmp I, xzr - beq .Lgemv_n_kernel_F1 + beq L(gemv_n_kernel_F1) -.Lgemv_n_kernel_F40: +L(gemv_n_kernel_F40): KERNEL_F4 subs I, I, #1 - bne .Lgemv_n_kernel_F40 + bne L(gemv_n_kernel_F40) -.Lgemv_n_kernel_F1: +L(gemv_n_kernel_F1): ands I, M, #3 - ble .Lgemv_n_kernel_F_END + ble L(gemv_n_kernel_F_END) -.Lgemv_n_kernel_F10: +L(gemv_n_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lgemv_n_kernel_F10 + bne L(gemv_n_kernel_F10) -.Lgemv_n_kernel_F_END: +L(gemv_n_kernel_F_END): add A, A, LDA subs J, J, #1 - bne .Lgemv_n_kernel_F_LOOP + bne L(gemv_n_kernel_F_LOOP) - b .Lgemv_n_kernel_L999 + b L(gemv_n_kernel_L999) -.Lgemv_n_kernel_S_BEGIN: +L(gemv_n_kernel_S_BEGIN): INIT_S -.Lgemv_n_kernel_S_LOOP: +L(gemv_n_kernel_S_LOOP): ld1 TEMPV, [X], INC_X fmul TEMP, ALPHA, TEMP @@ -288,9 +288,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #2 cmp I, xzr - ble .Lgemv_n_kernel_S1 + ble L(gemv_n_kernel_S1) -.Lgemv_n_kernel_S4: +L(gemv_n_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -298,27 +298,27 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lgemv_n_kernel_S4 + bne L(gemv_n_kernel_S4) -.Lgemv_n_kernel_S1: +L(gemv_n_kernel_S1): ands I, M, #3 - ble .Lgemv_n_kernel_S_END + ble L(gemv_n_kernel_S_END) -.Lgemv_n_kernel_S10: +L(gemv_n_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lgemv_n_kernel_S10 + bne L(gemv_n_kernel_S10) -.Lgemv_n_kernel_S_END: +L(gemv_n_kernel_S_END): add A, A, LDA subs J, J, #1 - bne .Lgemv_n_kernel_S_LOOP + bne L(gemv_n_kernel_S_LOOP) -.Lgemv_n_kernel_L999: +L(gemv_n_kernel_L999): mov w0, wzr diff --git a/kernel/arm64/gemv_t.S b/kernel/arm64/gemv_t.S index a98eef49b0..18559a1f93 100644 --- a/kernel/arm64/gemv_t.S +++ b/kernel/arm64/gemv_t.S @@ -242,18 +242,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SAVE_REGS cmp N, xzr - ble .Lgemv_t_kernel_L999 + ble L(gemv_t_kernel_L999) cmp M, xzr - ble .Lgemv_t_kernel_L999 + ble L(gemv_t_kernel_L999) lsl LDA, LDA, #SHZ lsl INC_Y, INC_Y, #SHZ mov J, N cmp INC_X, #1 - bne .Lgemv_t_kernel_S_BEGIN + bne L(gemv_t_kernel_S_BEGIN) -.Lgemv_t_kernel_F_LOOP: +L(gemv_t_kernel_F_LOOP): fmov TEMP, REG0 fmov TEMP1, REG0 @@ -263,64 +263,64 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov A_PTR, A mov X_PTR, X -.Lgemv_t_kernel_F32: +L(gemv_t_kernel_F32): asr I, M, #5 cmp I, xzr - beq .Lgemv_t_kernel_F8 + beq L(gemv_t_kernel_F8) -.Lgemv_t_kernel_F320: +L(gemv_t_kernel_F320): KERNEL_F32 subs I, I, #1 - bne .Lgemv_t_kernel_F320 + bne L(gemv_t_kernel_F320) KERNEL_F32_FINALIZE -.Lgemv_t_kernel_F8: +L(gemv_t_kernel_F8): ands I, M, #31 asr I, I, #3 cmp I, xzr - beq .Lgemv_t_kernel_F1 + beq L(gemv_t_kernel_F1) -.Lgemv_t_kernel_F80: +L(gemv_t_kernel_F80): KERNEL_F8 subs I, I, #1 - bne .Lgemv_t_kernel_F80 + bne L(gemv_t_kernel_F80) -.Lgemv_t_kernel_F1: +L(gemv_t_kernel_F1): KERNEL_F8_FINALIZE ands I, M, #7 - ble .Lgemv_t_kernel_F_END + ble L(gemv_t_kernel_F_END) -.Lgemv_t_kernel_F10: +L(gemv_t_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lgemv_t_kernel_F10 + bne L(gemv_t_kernel_F10) -.Lgemv_t_kernel_F_END: +L(gemv_t_kernel_F_END): ld1 TMPV1, [Y] add A, A, LDA subs J, J, #1 fmadd TMP1, ALPHA, TEMP, TMP1 st1 TMPV1, [Y], INC_Y - bne .Lgemv_t_kernel_F_LOOP + bne L(gemv_t_kernel_F_LOOP) - b .Lgemv_t_kernel_L999 + b L(gemv_t_kernel_L999) -.Lgemv_t_kernel_S_BEGIN: +L(gemv_t_kernel_S_BEGIN): INIT_S -.Lgemv_t_kernel_S_LOOP: +L(gemv_t_kernel_S_LOOP): fmov TEMP, REG0 mov A_PTR, A @@ -328,9 +328,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #2 cmp I, xzr - ble .Lgemv_t_kernel_S1 + ble L(gemv_t_kernel_S1) -.Lgemv_t_kernel_S4: +L(gemv_t_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -338,30 +338,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lgemv_t_kernel_S4 + bne L(gemv_t_kernel_S4) -.Lgemv_t_kernel_S1: +L(gemv_t_kernel_S1): ands I, M, #3 - ble .Lgemv_t_kernel_S_END + ble L(gemv_t_kernel_S_END) -.Lgemv_t_kernel_S10: +L(gemv_t_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lgemv_t_kernel_S10 + bne L(gemv_t_kernel_S10) -.Lgemv_t_kernel_S_END: +L(gemv_t_kernel_S_END): ld1 TMPV1, [Y] add A, A, LDA subs J, J, #1 fmadd TMP1, ALPHA, TEMP, TMP1 st1 TMPV1, [Y], INC_Y - bne .Lgemv_t_kernel_S_LOOP + bne L(gemv_t_kernel_S_LOOP) -.Lgemv_t_kernel_L999: +L(gemv_t_kernel_L999): RESTORE_REGS diff --git a/kernel/arm64/iamax.S b/kernel/arm64/iamax.S index 31d0cd6466..b089505b12 100644 --- a/kernel/arm64/iamax.S +++ b/kernel/arm64/iamax.S @@ -230,62 +230,62 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Liamax_kernel_zero + ble L(iamax_kernel_zero) cmp INC_X, xzr - ble .Liamax_kernel_zero + ble L(iamax_kernel_zero) cmp INC_X, #1 - bne .Liamax_kernel_S_BEGIN + bne L(iamax_kernel_S_BEGIN) mov x7, X -.Liamax_kernel_F_BEGIN: +L(iamax_kernel_F_BEGIN): INIT_S subs N, N, #1 - ble .Liamax_kernel_L999 + ble L(iamax_kernel_L999) asr I, N, #3 cmp I, xzr - beq .Liamax_kernel_F1 + beq L(iamax_kernel_F1) add Z, Z, #1 -.Liamax_kernel_F8: +L(iamax_kernel_F8): KERNEL_F8 subs I, I, #1 - bne .Liamax_kernel_F8 + bne L(iamax_kernel_F8) KERNEL_F8_FINALIZE sub Z, Z, #1 -.Liamax_kernel_F1: +L(iamax_kernel_F1): ands I, N, #7 - ble .Liamax_kernel_L999 + ble L(iamax_kernel_L999) -.Liamax_kernel_F10: +L(iamax_kernel_F10): KERNEL_S1 subs I, I, #1 - bne .Liamax_kernel_F10 + bne L(iamax_kernel_F10) - b .Liamax_kernel_L999 + b L(iamax_kernel_L999) -.Liamax_kernel_S_BEGIN: +L(iamax_kernel_S_BEGIN): INIT_S subs N, N, #1 - ble .Liamax_kernel_L999 + ble L(iamax_kernel_L999) asr I, N, #2 cmp I, xzr - ble .Liamax_kernel_S1 + ble L(iamax_kernel_S1) -.Liamax_kernel_S4: +L(iamax_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -293,25 +293,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Liamax_kernel_S4 + bne L(iamax_kernel_S4) -.Liamax_kernel_S1: +L(iamax_kernel_S1): ands I, N, #3 - ble .Liamax_kernel_L999 + ble L(iamax_kernel_L999) -.Liamax_kernel_S10: +L(iamax_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Liamax_kernel_S10 + bne L(iamax_kernel_S10) -.Liamax_kernel_L999: +L(iamax_kernel_L999): mov x0, INDEX ret -.Liamax_kernel_zero: +L(iamax_kernel_zero): mov x0, xzr ret diff --git a/kernel/arm64/izamax.S b/kernel/arm64/izamax.S index 42fa4e7111..904d6fc701 100644 --- a/kernel/arm64/izamax.S +++ b/kernel/arm64/izamax.S @@ -276,64 +276,64 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Lizamax_kernel_zero + ble L(izamax_kernel_zero) cmp INC_X, xzr - ble .Lizamax_kernel_zero + ble L(izamax_kernel_zero) cmp INC_X, #1 - bne .Lizamax_kernel_S_BEGIN + bne L(izamax_kernel_S_BEGIN) mov x7, X -.Lizamax_kernel_F_BEGIN: +L(izamax_kernel_F_BEGIN): INIT_S subs N, N, #1 - ble .Lizamax_kernel_L999 + ble L(izamax_kernel_L999) asr I, N, #3 cmp I, xzr - ble .Lizamax_kernel_F1 + ble L(izamax_kernel_F1) add Z, Z, #1 -.Lizamax_kernel_F8: +L(izamax_kernel_F8): KERNEL_F8 subs I, I, #1 - bne .Lizamax_kernel_F8 + bne L(izamax_kernel_F8) KERNEL_F8_FINALIZE sub Z, Z, #1 -.Lizamax_kernel_F1: +L(izamax_kernel_F1): ands I, N, #7 - ble .Lizamax_kernel_L999 + ble L(izamax_kernel_L999) -.Lizamax_kernel_F10: +L(izamax_kernel_F10): KERNEL_S1 subs I, I, #1 - bne .Lizamax_kernel_F10 + bne L(izamax_kernel_F10) - b .Lizamax_kernel_L999 + b L(izamax_kernel_L999) -.Lizamax_kernel_S_BEGIN: +L(izamax_kernel_S_BEGIN): INIT_S subs N, N, #1 - ble .Lizamax_kernel_L999 + ble L(izamax_kernel_L999) asr I, N, #2 cmp I, xzr - ble .Lizamax_kernel_S1 + ble L(izamax_kernel_S1) -.Lizamax_kernel_S4: +L(izamax_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -341,26 +341,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lizamax_kernel_S4 + bne L(izamax_kernel_S4) -.Lizamax_kernel_S1: +L(izamax_kernel_S1): ands I, N, #3 - ble .Lizamax_kernel_L999 + ble L(izamax_kernel_L999) -.Lizamax_kernel_S10: +L(izamax_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lizamax_kernel_S10 + bne L(izamax_kernel_S10) -.Lizamax_kernel_L999: +L(izamax_kernel_L999): mov x0, INDEX ret -.Lizamax_kernel_zero: +L(izamax_kernel_zero): mov x0, xzr ret diff --git a/kernel/arm64/nrm2.S b/kernel/arm64/nrm2.S index 93218ff6e0..c73307df74 100644 --- a/kernel/arm64/nrm2.S +++ b/kernel/arm64/nrm2.S @@ -79,20 +79,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ldr d4, [X] #endif fcmp d4, REGZERO - beq KERNEL_S1_NEXT + beq L(KERNEL_S1_NEXT) fabs d4, d4 fcmp SCALE, d4 - bge KERNEL_S1_SCALE_GE_X + bge L(KERNEL_S1_SCALE_GE_X) fdiv d2, SCALE, d4 fmul d2, d2, d2 fmul d3, SSQ, d2 fadd SSQ, REGONE, d3 fmov SCALE, d4 - b KERNEL_S1_NEXT -KERNEL_S1_SCALE_GE_X: + b L(KERNEL_S1_NEXT) +L(KERNEL_S1_SCALE_GE_X): fdiv d2, d4, SCALE fmla SSQ, d2, v2.d[0] -KERNEL_S1_NEXT: +L(KERNEL_S1_NEXT): add X, X, INC_X .endm @@ -133,44 +133,44 @@ KERNEL_S1_NEXT: INIT cmp N, #0 - ble .Lnrm2_kernel_L999 + ble L(nrm2_kernel_L999) cmp INC_X, #0 - beq .Lnrm2_kernel_L999 + beq L(nrm2_kernel_L999) cmp INC_X, #1 - bne .Lnrm2_kernel_S_BEGIN + bne L(nrm2_kernel_S_BEGIN) -.Lnrm2_kernel_F_BEGIN: +L(nrm2_kernel_F_BEGIN): asr I, N, #3 // I = N / 8 cmp I, xzr - ble .Lnrm2_kernel_F1 + ble L(nrm2_kernel_F1) -.Lnrm2_kernel_F8: +L(nrm2_kernel_F8): KERNEL_F8 subs I, I, #1 - bne .Lnrm2_kernel_F8 + bne L(nrm2_kernel_F8) -.Lnrm2_kernel_F1: +L(nrm2_kernel_F1): ands I, N, #7 - ble .Lnrm2_kernel_L999 + ble L(nrm2_kernel_L999) -.Lnrm2_kernel_F10: +L(nrm2_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lnrm2_kernel_F10 + bne L(nrm2_kernel_F10) - b .Lnrm2_kernel_L999 + b L(nrm2_kernel_L999) -.Lnrm2_kernel_S_BEGIN: +L(nrm2_kernel_S_BEGIN): INIT_S @@ -178,15 +178,15 @@ KERNEL_S1_NEXT: .align 5 -.Lnrm2_kernel_S10: +L(nrm2_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lnrm2_kernel_S10 + bne L(nrm2_kernel_S10) -.Lnrm2_kernel_L999: +L(nrm2_kernel_L999): fsqrt SSQ, SSQ fmul SSQ, SCALE, SSQ #if !defined(DOUBLE) diff --git a/kernel/arm64/rot.S b/kernel/arm64/rot.S index 00c3085fa3..3f37e9b1e9 100644 --- a/kernel/arm64/rot.S +++ b/kernel/arm64/rot.S @@ -165,48 +165,48 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Lrot_kernel_L999 + ble L(rot_kernel_L999) INIT cmp INC_X, #1 - bne .Lrot_kernel_S_BEGIN + bne L(rot_kernel_S_BEGIN) cmp INC_Y, #1 - bne .Lrot_kernel_S_BEGIN + bne L(rot_kernel_S_BEGIN) -.Lrot_kernel_F_BEGIN: +L(rot_kernel_F_BEGIN): asr I, N, #2 cmp I, xzr - beq .Lrot_kernel_F1 + beq L(rot_kernel_F1) KERNEL_INIT_F4 -.Lrot_kernel_F4: +L(rot_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lrot_kernel_F4 + bne L(rot_kernel_F4) -.Lrot_kernel_F1: +L(rot_kernel_F1): ands I, N, #3 - ble .Lrot_kernel_L999 + ble L(rot_kernel_L999) INIT_F1 -.Lrot_kernel_F10: +L(rot_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lrot_kernel_F10 + bne L(rot_kernel_F10) mov w0, wzr ret -.Lrot_kernel_S_BEGIN: +L(rot_kernel_S_BEGIN): INIT_S INIT_F1 @@ -214,9 +214,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, N, #2 cmp I, xzr - ble .Lrot_kernel_S1 + ble L(rot_kernel_S1) -.Lrot_kernel_S4: +L(rot_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -224,22 +224,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lrot_kernel_S4 + bne L(rot_kernel_S4) -.Lrot_kernel_S1: +L(rot_kernel_S1): ands I, N, #3 - ble .Lrot_kernel_L999 + ble L(rot_kernel_L999) -.Lrot_kernel_S10: +L(rot_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lrot_kernel_S10 + bne L(rot_kernel_S10) -.Lrot_kernel_L999: +L(rot_kernel_L999): mov w0, wzr ret + + EPILOGUE diff --git a/kernel/arm64/scal.S b/kernel/arm64/scal.S index 33400b6309..757667e5bb 100644 --- a/kernel/arm64/scal.S +++ b/kernel/arm64/scal.S @@ -166,91 +166,91 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Lscal_kernel_L999 + ble L(scal_kernel_L999) ldr FLAG, [sp] cmp FLAG, #1 - beq .Lscal_kernel_nansafe + beq L(scal_kernel_nansafe) fcmp DA, #0.0 - beq .Lscal_kernel_zero + beq L(scal_kernel_zero) -.Lscal_kernel_nansafe: +L(scal_kernel_nansafe): cmp INC_X, #1 - bne .Lscal_kernel_S_BEGIN + bne L(scal_kernel_S_BEGIN) -.Lscal_kernel_F_BEGIN: +L(scal_kernel_F_BEGIN): asr I, N, #3 cmp I, xzr - beq .Lscal_kernel_F1 + beq L(scal_kernel_F1) KERNEL_INIT_F8 -.Lscal_kernel_F8: +L(scal_kernel_F8): KERNEL_F8 subs I, I, #1 - bne .Lscal_kernel_F8 + bne L(scal_kernel_F8) -.Lscal_kernel_F1: +L(scal_kernel_F1): ands I, N, #7 - ble .Lscal_kernel_L999 + ble L(scal_kernel_L999) -.Lscal_kernel_F10: +L(scal_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lscal_kernel_F10 + bne L(scal_kernel_F10) mov w0, wzr ret -.Lscal_kernel_S_BEGIN: +L(scal_kernel_S_BEGIN): INIT_S mov X_COPY, X asr I, N, #2 cmp I, xzr - ble .Lscal_kernel_S1 + ble L(scal_kernel_S1) -.Lscal_kernel_S4: +L(scal_kernel_S4): KERNEL_S4 subs I, I, #1 - bne .Lscal_kernel_S4 + bne L(scal_kernel_S4) -.Lscal_kernel_S1: +L(scal_kernel_S1): ands I, N, #3 - ble .Lscal_kernel_L999 + ble L(scal_kernel_L999) -.Lscal_kernel_S10: +L(scal_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lscal_kernel_S10 + bne L(scal_kernel_S10) -.Lscal_kernel_L999: +L(scal_kernel_L999): mov w0, wzr ret -.Lscal_kernel_zero: +L(scal_kernel_zero): INIT_S -.Lscal_kernel_Z1: +L(scal_kernel_Z1): st1 DAV, [X], INC_X subs N, N, #1 - bne .Lscal_kernel_Z1 + bne L(scal_kernel_Z1) mov w0, wzr ret diff --git a/kernel/arm64/sgemm_beta.S b/kernel/arm64/sgemm_beta.S index 574485bc44..d892d97ab0 100644 --- a/kernel/arm64/sgemm_beta.S +++ b/kernel/arm64/sgemm_beta.S @@ -102,33 +102,33 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ldr LDC, [sp] SAVE_REGS -.Lgemm_beta_BEGIN: +L(gemm_beta_BEGIN): fmov beta0, BETA cmp N, #0 - ble .Lgemm_beta_L999 + ble L(gemm_beta_L999) fcmp BETA, #0.0 - beq .Lgemm_beta_zero_01 + beq L(gemm_beta_zero_01) -.Lgemm_beta_01: +L(gemm_beta_01): lsl LDC, LDC, #2 .align 5 -.Lgemm_beta_02: +L(gemm_beta_02): mov A01, C00 add C00, C00, LDC asr I, M, #5 cmp I, #0 - ble .Lgemm_beta_04 + ble L(gemm_beta_04) add A02, A01, #32 add A03, A02, #32 add A04, A03, #32 .align 5 -.Lgemm_beta_03: +L(gemm_beta_03): prfm PLDL1KEEP, [A01, prfm_size] @@ -161,17 +161,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add A04, A04, calc_size subs I , I , #1 - bne .Lgemm_beta_03 + bne L(gemm_beta_03) .align 5 -.Lgemm_beta_04: +L(gemm_beta_04): and I, M , #31 cmp I, #0 - ble .Lgemm_beta_06 + ble L(gemm_beta_06) .align 5 -.Lgemm_beta_05: +L(gemm_beta_05): ldr s12, [A01] fmul s12, s12, beta0 @@ -179,42 +179,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add A01, A01, #4 subs I , I , #1 - bne .Lgemm_beta_05 + bne L(gemm_beta_05) .align 5 -.Lgemm_beta_06: +L(gemm_beta_06): subs N , N, #1 // N-- - bne .Lgemm_beta_02 + bne L(gemm_beta_02) .align 5 -.Lgemm_beta_L999: +L(gemm_beta_L999): mov x0, #0 RESTORE_REGS ret .align 5 -.Lgemm_beta_zero_01: +L(gemm_beta_zero_01): INIT_ZERO lsl LDC, LDC, #2 .align 5 -.Lgemm_beta_zero_02: +L(gemm_beta_zero_02): mov A01, C00 add C00, C00, LDC asr I, M, #5 cmp I, #0 - ble .Lgemm_beta_zero_04 + ble L(gemm_beta_zero_04) add A02, A01, #32 add A03, A02, #32 add A04, A03, #32 .align 5 -.Lgemm_beta_zero_03: +L(gemm_beta_zero_03): st1 {v0.4s, v1.4s}, [A01] add A01, A01, calc_size @@ -226,32 +226,32 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add A04, A04, calc_size subs I, I, #1 - bne .Lgemm_beta_zero_03 + bne L(gemm_beta_zero_03) .align 5 -.Lgemm_beta_zero_04: +L(gemm_beta_zero_04): and I, M, #31 cmp I, #0 - ble .Lgemm_beta_zero_06 + ble L(gemm_beta_zero_06) .align 5 -.Lgemm_beta_zero_05: +L(gemm_beta_zero_05): str beta0, [A01] add A01, A01, #4 subs I, I, #1 - bne .Lgemm_beta_zero_05 + bne L(gemm_beta_zero_05) .align 5 -.Lgemm_beta_zero_06: +L(gemm_beta_zero_06): subs N, N, #1 - bne .Lgemm_beta_zero_02 + bne L(gemm_beta_zero_02) .align 5 -.Lgemm_beta_zero_L999: +L(gemm_beta_zero_L999): mov x0, #0 RESTORE_REGS ret diff --git a/kernel/arm64/sgemm_direct_sme1_2VLx2VL.S b/kernel/arm64/sgemm_direct_sme1_2VLx2VL.S index afb662c1fb..c6533e657c 100644 --- a/kernel/arm64/sgemm_direct_sme1_2VLx2VL.S +++ b/kernel/arm64/sgemm_direct_sme1_2VLx2VL.S @@ -3,6 +3,9 @@ SPDX-License-Identifier: BSD-3-Clause-Clear */ +#define ASSEMBLER +#include "common.h" + /*-------------------------------------------------------------------------- * SME1 based Matrix multiplication code for FP32 input matrices to FP32 * output matrix @@ -64,14 +67,14 @@ whilelt p2.s, M_cntr, M //Tile 0,1 predicate (M dimension) sub w21, w21, #2 //SVLs-2 -.M_Loop: +L(M_Loop): incw M_cntr whilelt p3.s, M_cntr, M //Tile 2,3 predicate (M dimension) mov Bptr, B_base //B_base mov Cptr, C_base //C_base whilelt p0.b, Bptr, N_exit //Tile 0/2 predicate (N dimension) -.N_Loop: +L(N_Loop): mov Aptr, A_base //Aptr = A_base mov Bptr0, Bptr //Bptr = B_base mov Cptr0, Cptr //Cptr0 = C_base @@ -92,7 +95,7 @@ // Aptr += SVLb addvl Aptr, Aptr, #1 -.K_Loop: +L(K_Loop): // ZA2 += 2nd Aptr vector OP 1st Bptr vector fmopa za2.s, p3/m, p0/m, z5.s, z2.s // Load 2nd vector from Bptr @@ -104,7 +107,7 @@ // ZA3 += 2nd Aptr vector OP 2nd Bptr vector fmopa za3.s, p3/m, p1/m, z5.s, z3.s cmp K, #2 - b.le process_K_less_than_equal_2 + b.le L(process_K_less_than_equal_2) // Load next 1st vector from Bptr ld1w {z6.s}, p0/z, [Bptr0, N, lsl #2] // ZA0 += 1st Aptr vector OP 1st Bptr vector @@ -132,7 +135,7 @@ // Aptr += 2*SVLb [Bytes] addvl Aptr, Aptr, #2 cmp Aptr, K_exit - b.mi .K_Loop + b.mi L(K_Loop) // ZA2 += 2nd Aptr vector OP 1st Bptr vector fmopa za2.s, p3/m, p0/m, z5.s, z2.s // Load next 2nd vector from Bptr @@ -142,13 +145,13 @@ // ZA3 += 2nd Aptr vector OP 2nd Bptr vector fmopa za3.s, p3/m, p1/m, z5.s, z3.s -process_K_less_than_equal_2: +L(process_K_less_than_equal_2): // Bptr += 2*ldb FP32 elements add Bptr0, Bptr0, N, lsl #2 cmp Aptr, Aptr_end - b.pl .Ktail_end + b.pl L(Ktail_end) -.Ktail_start: +L(Ktail_start): ld1w {z1.s}, p2/z, [Aptr] ld1w {z2.s}, p0/z, [Bptr0] ld1w {z3.s}, p1/z, [Bptr0, #1, MUL VL] @@ -158,7 +161,7 @@ process_K_less_than_equal_2: fmopa za1.s, p2/m, p1/m, z1.s, z3.s fmopa za3.s, p3/m, p1/m, z5.s, z3.s -.Ktail_end: +L(Ktail_end): mov w13, #0 psel p4, p0, p2.s[w13, 0] psel p5, p1, p2.s[w13, 0] @@ -173,7 +176,7 @@ process_K_less_than_equal_2: // Store to Cptr1 + N*SVLs st1w {za3h.s[w13, #0]}, p7, [Cptr1, C6, lsl #2] -.Loop_store_ZA: +L(Loop_store_ZA): psel p4, p0, p2.s[w13, 1] psel p5, p1, p2.s[w13, 1] psel p6, p0, p3.s[w13, 1] @@ -200,7 +203,7 @@ process_K_less_than_equal_2: st1w {za2h.s[w13, #0]}, p6, [Cptr0, C6, lsl #2] st1w {za3h.s[w13, #0]}, p7, [Cptr1, C6, lsl #2] cmp w13, w21 - b.mi .Loop_store_ZA + b.mi L(Loop_store_ZA) psel p4, p0, p2.s[w13, 1] psel p5, p1, p2.s[w13, 1] psel p6, p0, p3.s[w13, 1] @@ -212,12 +215,12 @@ process_K_less_than_equal_2: addvl Cptr, Cptr, #2 addvl Bptr, Bptr, #1 whilelt p0.b, Bptr, N_exit //1st Tile predicate (N dimension) - b.mi .N_Loop + b.mi L(N_Loop) add A_base, A_base, C5, lsl #3 //A_base += 2*K*SVLs FP32 elements add C_base, C_base, C6, lsl #3 //C_base += 2*N*SVLs FP32 elements incw M_cntr whilelt p2.s, M_cntr, M //1st Tile predicate (M dimension) - b.mi .M_Loop + b.mi L(M_Loop) smstop @@ -227,3 +230,4 @@ process_K_less_than_equal_2: ret + EPILOGUE diff --git a/kernel/arm64/sgemm_direct_sme1_preprocess.S b/kernel/arm64/sgemm_direct_sme1_preprocess.S index 6c51b0bf63..b7536b5a46 100644 --- a/kernel/arm64/sgemm_direct_sme1_preprocess.S +++ b/kernel/arm64/sgemm_direct_sme1_preprocess.S @@ -3,6 +3,9 @@ SPDX-License-Identifier: BSD-3-Clause-Clear */ +#define ASSEMBLER +#include "common.h" + /*---------------------------------------------------------------------------- * This function is used to re-arrange the elements of input matrix to * make it suitable for matrix outer product computation using SME for matrix @@ -60,18 +63,18 @@ //Predicate for stores ptrue p9.s -.M_Loop: +L(M_Loop): mov mat_ptr0, mat //Load base address of mat mov mat_mod_ptr, mat_mod //a_mod store base address add inner_loop_exit, mat, ncol, lsl #2 //Exit condition for inner loop whilelt p8.b, mat_ptr0, inner_loop_exit //Tile predicate (K dimension) -.Loop_process: +L(Loop_process): mov mat_ptr1, mat_ptr0 //Load_to_tile loop counter mov w12, #0 -.Load_to_tile: +L(Load_to_tile): psel p2, p8, p0.s[w12, 0] psel p3, p8, p0.s[w12, 1] psel p4, p8, p0.s[w12, 2] @@ -89,11 +92,11 @@ //Increment counter add w12, w12, #4 cmp w12, w9 - b.mi .Load_to_tile + b.mi L(Load_to_tile) // Store_from_tile loop counter mov w12, #0 -.Store_from_tile: +L(Store_from_tile): psel p2, p9, p8.s[w12, 0] psel p3, p9, p8.s[w12, 1] psel p4, p9, p8.s[w12, 2] @@ -110,18 +113,18 @@ addvl mat_mod_ptr, mat_mod_ptr, #4 //mat_mod_ptr += 4*SVLb add w12, w12, #4 //Increment counter cmp w12, w9 - b.mi .Store_from_tile + b.mi L(Store_from_tile) addvl mat_ptr0, mat_ptr0, #1 //mat_ptr0 += SVLb whilelt p8.b, mat_ptr0, inner_loop_exit - b.mi .Loop_process + b.mi L(Loop_process) add mat_mod, mat_mod, C3, lsl #2 //mat_mod+=SVLs*nbc FP32 elements add mat, mat, C3, lsl #2 //mat+=SVLs*nbc FP32 elements incw outer_loop_cntr whilelt p0.s, outer_loop_cntr, nrow - b.mi .M_Loop + b.mi L(M_Loop) smstop @@ -131,3 +134,4 @@ ret + EPILOGUE diff --git a/kernel/arm64/sgemm_kernel_16x4.S b/kernel/arm64/sgemm_kernel_16x4.S index 99099ea6fd..3ad1f008a0 100644 --- a/kernel/arm64/sgemm_kernel_16x4.S +++ b/kernel/arm64/sgemm_kernel_16x4.S @@ -1070,7 +1070,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE -.Lsgemm_kernel_begin: +L(sgemm_kernel_begin): .align 5 add sp, sp, #-(11 * 16) @@ -1098,11 +1098,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lsgemm_kernel_L2_BEGIN + ble L(sgemm_kernel_L2_BEGIN) /******************************************************************************/ -.Lsgemm_kernel_L4_BEGIN: +L(sgemm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -1112,21 +1112,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lsgemm_kernel_L4_M16_BEGIN: +L(sgemm_kernel_L4_M16_BEGIN): mov counterI, origM asr counterI, counterI, #4 // counterI = counterI / 16 cmp counterI, #0 - ble .Lsgemm_kernel_L4_M8_BEGIN + ble L(sgemm_kernel_L4_M8_BEGIN) .align 5 -.Lsgemm_kernel_L4_M16_20: +L(sgemm_kernel_L4_M16_20): mov pB, origPB asr counterL , origK, #3 cmp counterL , #2 - blt .Lsgemm_kernel_L4_M16_32 + blt L(sgemm_kernel_L4_M16_32) KERNEL16x4_I KERNEL16x4_M2 @@ -1138,10 +1138,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x4_M2 subs counterL, counterL, #2 - ble .Lsgemm_kernel_L4_M16_22a + ble L(sgemm_kernel_L4_M16_22a) .align 5 -.Lsgemm_kernel_L4_M16_22: +L(sgemm_kernel_L4_M16_22): KERNEL16x4_M1 KERNEL16x4_M2 @@ -1153,10 +1153,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x4_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M16_22 + bgt L(sgemm_kernel_L4_M16_22) .align 5 -.Lsgemm_kernel_L4_M16_22a: +L(sgemm_kernel_L4_M16_22a): KERNEL16x4_M1 KERNEL16x4_M2 @@ -1167,13 +1167,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x4_M1 KERNEL16x4_E - b .Lsgemm_kernel_L4_M16_44 + b L(sgemm_kernel_L4_M16_44) .align 5 -.Lsgemm_kernel_L4_M16_32: +L(sgemm_kernel_L4_M16_32): tst counterL, #1 - ble .Lsgemm_kernel_L4_M16_40 + ble L(sgemm_kernel_L4_M16_40) KERNEL16x4_I KERNEL16x4_M2 @@ -1184,187 +1184,187 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x4_M1 KERNEL16x4_E - b .Lsgemm_kernel_L4_M16_44 + b L(sgemm_kernel_L4_M16_44) -.Lsgemm_kernel_L4_M16_40: +L(sgemm_kernel_L4_M16_40): INIT16x4 -.Lsgemm_kernel_L4_M16_44: +L(sgemm_kernel_L4_M16_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L4_M16_100 + ble L(sgemm_kernel_L4_M16_100) .align 5 -.Lsgemm_kernel_L4_M16_46: +L(sgemm_kernel_L4_M16_46): KERNEL16x4_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L4_M16_46 + bne L(sgemm_kernel_L4_M16_46) -.Lsgemm_kernel_L4_M16_100: +L(sgemm_kernel_L4_M16_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVE16x4 -.Lsgemm_kernel_L4_M16_END: +L(sgemm_kernel_L4_M16_END): subs counterI, counterI, #1 - bne .Lsgemm_kernel_L4_M16_20 + bne L(sgemm_kernel_L4_M16_20) //------------------------------------------------------------------------------ -.Lsgemm_kernel_L4_M8_BEGIN: +L(sgemm_kernel_L4_M8_BEGIN): mov counterI, origM tst counterI , #15 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #8 - ble .Lsgemm_kernel_L4_M4_BEGIN + ble L(sgemm_kernel_L4_M4_BEGIN) -.Lsgemm_kernel_L4_M8_20: +L(sgemm_kernel_L4_M8_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L4_M8_32 + blt L(sgemm_kernel_L4_M8_32) KERNEL8x4_I // do one in the K KERNEL8x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L4_M8_22a + ble L(sgemm_kernel_L4_M8_22a) .align 5 -.Lsgemm_kernel_L4_M8_22: +L(sgemm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M8_22 + bgt L(sgemm_kernel_L4_M8_22) -.Lsgemm_kernel_L4_M8_22a: +L(sgemm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_E - b .Lsgemm_kernel_L4_M8_44 + b L(sgemm_kernel_L4_M8_44) -.Lsgemm_kernel_L4_M8_32: +L(sgemm_kernel_L4_M8_32): tst counterL, #1 - ble .Lsgemm_kernel_L4_M8_40 + ble L(sgemm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_E - b .Lsgemm_kernel_L4_M8_44 + b L(sgemm_kernel_L4_M8_44) -.Lsgemm_kernel_L4_M8_40: +L(sgemm_kernel_L4_M8_40): INIT8x4 -.Lsgemm_kernel_L4_M8_44: +L(sgemm_kernel_L4_M8_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L4_M8_100 + ble L(sgemm_kernel_L4_M8_100) -.Lsgemm_kernel_L4_M8_46: +L(sgemm_kernel_L4_M8_46): KERNEL8x4_SUB -.Lsgemm_kernel_L4_M8_100: +L(sgemm_kernel_L4_M8_100): SAVE8x4 -.Lsgemm_kernel_L4_M8_END: +L(sgemm_kernel_L4_M8_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L4_M4_BEGIN: +L(sgemm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #4 - ble .Lsgemm_kernel_L4_M2_BEGIN + ble L(sgemm_kernel_L4_M2_BEGIN) -.Lsgemm_kernel_L4_M4_20: +L(sgemm_kernel_L4_M4_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L4_M4_32 + blt L(sgemm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L4_M4_22a + ble L(sgemm_kernel_L4_M4_22a) .align 5 -.Lsgemm_kernel_L4_M4_22: +L(sgemm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M4_22 + bgt L(sgemm_kernel_L4_M4_22) -.Lsgemm_kernel_L4_M4_22a: +L(sgemm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lsgemm_kernel_L4_M4_44 + b L(sgemm_kernel_L4_M4_44) -.Lsgemm_kernel_L4_M4_32: +L(sgemm_kernel_L4_M4_32): tst counterL, #1 - ble .Lsgemm_kernel_L4_M4_40 + ble L(sgemm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lsgemm_kernel_L4_M4_44 + b L(sgemm_kernel_L4_M4_44) -.Lsgemm_kernel_L4_M4_40: +L(sgemm_kernel_L4_M4_40): INIT4x4 -.Lsgemm_kernel_L4_M4_44: +L(sgemm_kernel_L4_M4_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L4_M4_100 + ble L(sgemm_kernel_L4_M4_100) -.Lsgemm_kernel_L4_M4_46: +L(sgemm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lsgemm_kernel_L4_M4_100: +L(sgemm_kernel_L4_M4_100): SAVE4x4 -.Lsgemm_kernel_L4_M4_END: +L(sgemm_kernel_L4_M4_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L4_M2_BEGIN: +L(sgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L4_M1_BEGIN + ble L(sgemm_kernel_L4_M1_BEGIN) -.Lsgemm_kernel_L4_M2_20: +L(sgemm_kernel_L4_M2_20): INIT2x4 @@ -1372,9 +1372,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L4_M2_40 + ble L(sgemm_kernel_L4_M2_40) -.Lsgemm_kernel_L4_M2_22: +L(sgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1387,34 +1387,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M2_22 + bgt L(sgemm_kernel_L4_M2_22) -.Lsgemm_kernel_L4_M2_40: +L(sgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M2_100 + ble L(sgemm_kernel_L4_M2_100) -.Lsgemm_kernel_L4_M2_42: +L(sgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M2_42 + bgt L(sgemm_kernel_L4_M2_42) -.Lsgemm_kernel_L4_M2_100: +L(sgemm_kernel_L4_M2_100): SAVE2x4 -.Lsgemm_kernel_L4_M2_END: +L(sgemm_kernel_L4_M2_END): -.Lsgemm_kernel_L4_M1_BEGIN: +L(sgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) -.Lsgemm_kernel_L4_M1_20: +L(sgemm_kernel_L4_M1_20): INIT1x4 @@ -1422,9 +1422,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L4_M1_40 + ble L(sgemm_kernel_L4_M1_40) -.Lsgemm_kernel_L4_M1_22: +L(sgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1436,42 +1436,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M1_22 + bgt L(sgemm_kernel_L4_M1_22) -.Lsgemm_kernel_L4_M1_40: +L(sgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M1_100 + ble L(sgemm_kernel_L4_M1_100) -.Lsgemm_kernel_L4_M1_42: +L(sgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M1_42 + bgt L(sgemm_kernel_L4_M1_42) -.Lsgemm_kernel_L4_M1_100: +L(sgemm_kernel_L4_M1_100): SAVE1x4 -.Lsgemm_kernel_L4_END: +L(sgemm_kernel_L4_END): add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4 subs counterJ, counterJ , #1 // j-- - bgt .Lsgemm_kernel_L4_BEGIN + bgt L(sgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Lsgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(sgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lsgemm_kernel_L999 + ble L(sgemm_kernel_L999) tst counterJ , #2 - ble .Lsgemm_kernel_L1_BEGIN + ble L(sgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1479,14 +1479,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lsgemm_kernel_L2_M16_BEGIN: +L(sgemm_kernel_L2_M16_BEGIN): mov counterI, origM asr counterI, counterI, #4 // counterI = counterI / 16 cmp counterI,#0 - ble .Lsgemm_kernel_L2_M8_BEGIN + ble L(sgemm_kernel_L2_M8_BEGIN) -.Lsgemm_kernel_L2_M16_20: +L(sgemm_kernel_L2_M16_20): INIT16x2 @@ -1494,10 +1494,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M16_40 + ble L(sgemm_kernel_L2_M16_40) .align 5 -.Lsgemm_kernel_L2_M16_22: +L(sgemm_kernel_L2_M16_22): KERNEL16x2_SUB KERNEL16x2_SUB KERNEL16x2_SUB @@ -1509,41 +1509,41 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M16_22 + bgt L(sgemm_kernel_L2_M16_22) -.Lsgemm_kernel_L2_M16_40: +L(sgemm_kernel_L2_M16_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M16_100 + ble L(sgemm_kernel_L2_M16_100) -.Lsgemm_kernel_L2_M16_42: +L(sgemm_kernel_L2_M16_42): KERNEL16x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M16_42 + bgt L(sgemm_kernel_L2_M16_42) -.Lsgemm_kernel_L2_M16_100: +L(sgemm_kernel_L2_M16_100): SAVE16x2 -.Lsgemm_kernel_L2_M16_END: +L(sgemm_kernel_L2_M16_END): subs counterI, counterI, #1 - bgt .Lsgemm_kernel_L2_M16_20 + bgt L(sgemm_kernel_L2_M16_20) //------------------------------------------------------------------------------ -.Lsgemm_kernel_L2_M8_BEGIN: +L(sgemm_kernel_L2_M8_BEGIN): mov counterI, origM tst counterI , #15 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) tst counterI, #8 - ble .Lsgemm_kernel_L2_M4_BEGIN + ble L(sgemm_kernel_L2_M4_BEGIN) -.Lsgemm_kernel_L2_M8_20: +L(sgemm_kernel_L2_M8_20): INIT8x2 @@ -1551,10 +1551,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M8_40 + ble L(sgemm_kernel_L2_M8_40) .align 5 -.Lsgemm_kernel_L2_M8_22: +L(sgemm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB KERNEL8x2_SUB @@ -1566,38 +1566,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M8_22 + bgt L(sgemm_kernel_L2_M8_22) -.Lsgemm_kernel_L2_M8_40: +L(sgemm_kernel_L2_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M8_100 + ble L(sgemm_kernel_L2_M8_100) -.Lsgemm_kernel_L2_M8_42: +L(sgemm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M8_42 + bgt L(sgemm_kernel_L2_M8_42) -.Lsgemm_kernel_L2_M8_100: +L(sgemm_kernel_L2_M8_100): SAVE8x2 -.Lsgemm_kernel_L2_M8_END: +L(sgemm_kernel_L2_M8_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L2_M4_BEGIN: +L(sgemm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) tst counterI, #4 - ble .Lsgemm_kernel_L2_M2_BEGIN + ble L(sgemm_kernel_L2_M2_BEGIN) -.Lsgemm_kernel_L2_M4_20: +L(sgemm_kernel_L2_M4_20): INIT4x2 @@ -1605,10 +1605,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M4_40 + ble L(sgemm_kernel_L2_M4_40) .align 5 -.Lsgemm_kernel_L2_M4_22: +L(sgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1620,40 +1620,40 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M4_22 + bgt L(sgemm_kernel_L2_M4_22) -.Lsgemm_kernel_L2_M4_40: +L(sgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M4_100 + ble L(sgemm_kernel_L2_M4_100) -.Lsgemm_kernel_L2_M4_42: +L(sgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M4_42 + bgt L(sgemm_kernel_L2_M4_42) -.Lsgemm_kernel_L2_M4_100: +L(sgemm_kernel_L2_M4_100): SAVE4x2 -.Lsgemm_kernel_L2_M4_END: +L(sgemm_kernel_L2_M4_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L2_M2_BEGIN: +L(sgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L2_M1_BEGIN + ble L(sgemm_kernel_L2_M1_BEGIN) -.Lsgemm_kernel_L2_M2_20: +L(sgemm_kernel_L2_M2_20): INIT2x2 @@ -1661,9 +1661,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M2_40 + ble L(sgemm_kernel_L2_M2_40) -.Lsgemm_kernel_L2_M2_22: +L(sgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1676,34 +1676,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M2_22 + bgt L(sgemm_kernel_L2_M2_22) -.Lsgemm_kernel_L2_M2_40: +L(sgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M2_100 + ble L(sgemm_kernel_L2_M2_100) -.Lsgemm_kernel_L2_M2_42: +L(sgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M2_42 + bgt L(sgemm_kernel_L2_M2_42) -.Lsgemm_kernel_L2_M2_100: +L(sgemm_kernel_L2_M2_100): SAVE2x2 -.Lsgemm_kernel_L2_M2_END: +L(sgemm_kernel_L2_M2_END): -.Lsgemm_kernel_L2_M1_BEGIN: +L(sgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) -.Lsgemm_kernel_L2_M1_20: +L(sgemm_kernel_L2_M1_20): INIT1x2 @@ -1711,9 +1711,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lsgemm_kernel_L2_M1_40 + ble L(sgemm_kernel_L2_M1_40) -.Lsgemm_kernel_L2_M1_22: +L(sgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1725,36 +1725,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M1_22 + bgt L(sgemm_kernel_L2_M1_22) -.Lsgemm_kernel_L2_M1_40: +L(sgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M1_100 + ble L(sgemm_kernel_L2_M1_100) -.Lsgemm_kernel_L2_M1_42: +L(sgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M1_42 + bgt L(sgemm_kernel_L2_M1_42) -.Lsgemm_kernel_L2_M1_100: +L(sgemm_kernel_L2_M1_100): SAVE1x2 -.Lsgemm_kernel_L2_END: +L(sgemm_kernel_L2_END): add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 /******************************************************************************/ -.Lsgemm_kernel_L1_BEGIN: +L(sgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lsgemm_kernel_L999 // done + ble L(sgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1762,14 +1762,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lsgemm_kernel_L1_M16_BEGIN: +L(sgemm_kernel_L1_M16_BEGIN): mov counterI, origM asr counterI, counterI, #4 // counterI = counterI / 16 cmp counterI, #0 - ble .Lsgemm_kernel_L1_M8_BEGIN + ble L(sgemm_kernel_L1_M8_BEGIN) -.Lsgemm_kernel_L1_M16_20: +L(sgemm_kernel_L1_M16_20): INIT16x1 @@ -1777,10 +1777,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M16_40 + ble L(sgemm_kernel_L1_M16_40) .align 5 -.Lsgemm_kernel_L1_M16_22: +L(sgemm_kernel_L1_M16_22): KERNEL16x1_SUB KERNEL16x1_SUB KERNEL16x1_SUB @@ -1792,42 +1792,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M16_22 + bgt L(sgemm_kernel_L1_M16_22) -.Lsgemm_kernel_L1_M16_40: +L(sgemm_kernel_L1_M16_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M16_100 + ble L(sgemm_kernel_L1_M16_100) -.Lsgemm_kernel_L1_M16_42: +L(sgemm_kernel_L1_M16_42): KERNEL16x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M16_42 + bgt L(sgemm_kernel_L1_M16_42) -.Lsgemm_kernel_L1_M16_100: +L(sgemm_kernel_L1_M16_100): SAVE16x1 -.Lsgemm_kernel_L1_M16_END: +L(sgemm_kernel_L1_M16_END): subs counterI, counterI, #1 - bgt .Lsgemm_kernel_L1_M16_20 + bgt L(sgemm_kernel_L1_M16_20) //------------------------------------------------------------------------------ -.Lsgemm_kernel_L1_M8_BEGIN: +L(sgemm_kernel_L1_M8_BEGIN): mov counterI, origM tst counterI , #15 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) tst counterI, #8 - ble .Lsgemm_kernel_L1_M4_BEGIN + ble L(sgemm_kernel_L1_M4_BEGIN) -.Lsgemm_kernel_L1_M8_20: +L(sgemm_kernel_L1_M8_20): INIT8x1 @@ -1835,10 +1835,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M8_40 + ble L(sgemm_kernel_L1_M8_40) .align 5 -.Lsgemm_kernel_L1_M8_22: +L(sgemm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -1850,38 +1850,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M8_22 + bgt L(sgemm_kernel_L1_M8_22) -.Lsgemm_kernel_L1_M8_40: +L(sgemm_kernel_L1_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M8_100 + ble L(sgemm_kernel_L1_M8_100) -.Lsgemm_kernel_L1_M8_42: +L(sgemm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M8_42 + bgt L(sgemm_kernel_L1_M8_42) -.Lsgemm_kernel_L1_M8_100: +L(sgemm_kernel_L1_M8_100): SAVE8x1 -.Lsgemm_kernel_L1_M8_END: +L(sgemm_kernel_L1_M8_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L1_M4_BEGIN: +L(sgemm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) tst counterI, #4 - ble .Lsgemm_kernel_L1_M2_BEGIN + ble L(sgemm_kernel_L1_M2_BEGIN) -.Lsgemm_kernel_L1_M4_20: +L(sgemm_kernel_L1_M4_20): INIT4x1 @@ -1889,10 +1889,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M4_40 + ble L(sgemm_kernel_L1_M4_40) .align 5 -.Lsgemm_kernel_L1_M4_22: +L(sgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1904,39 +1904,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M4_22 + bgt L(sgemm_kernel_L1_M4_22) -.Lsgemm_kernel_L1_M4_40: +L(sgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M4_100 + ble L(sgemm_kernel_L1_M4_100) -.Lsgemm_kernel_L1_M4_42: +L(sgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M4_42 + bgt L(sgemm_kernel_L1_M4_42) -.Lsgemm_kernel_L1_M4_100: +L(sgemm_kernel_L1_M4_100): SAVE4x1 -.Lsgemm_kernel_L1_M4_END: +L(sgemm_kernel_L1_M4_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L1_M2_BEGIN: +L(sgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L1_M1_BEGIN + ble L(sgemm_kernel_L1_M1_BEGIN) -.Lsgemm_kernel_L1_M2_20: +L(sgemm_kernel_L1_M2_20): INIT2x1 @@ -1944,9 +1944,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M2_40 + ble L(sgemm_kernel_L1_M2_40) -.Lsgemm_kernel_L1_M2_22: +L(sgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1959,34 +1959,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M2_22 + bgt L(sgemm_kernel_L1_M2_22) -.Lsgemm_kernel_L1_M2_40: +L(sgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M2_100 + ble L(sgemm_kernel_L1_M2_100) -.Lsgemm_kernel_L1_M2_42: +L(sgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M2_42 + bgt L(sgemm_kernel_L1_M2_42) -.Lsgemm_kernel_L1_M2_100: +L(sgemm_kernel_L1_M2_100): SAVE2x1 -.Lsgemm_kernel_L1_M2_END: +L(sgemm_kernel_L1_M2_END): -.Lsgemm_kernel_L1_M1_BEGIN: +L(sgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) -.Lsgemm_kernel_L1_M1_20: +L(sgemm_kernel_L1_M1_20): INIT1x1 @@ -1994,9 +1994,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M1_40 + ble L(sgemm_kernel_L1_M1_40) -.Lsgemm_kernel_L1_M1_22: +L(sgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -2008,28 +2008,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M1_22 + bgt L(sgemm_kernel_L1_M1_22) -.Lsgemm_kernel_L1_M1_40: +L(sgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M1_100 + ble L(sgemm_kernel_L1_M1_100) -.Lsgemm_kernel_L1_M1_42: +L(sgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M1_42 + bgt L(sgemm_kernel_L1_M1_42) -.Lsgemm_kernel_L1_M1_100: +L(sgemm_kernel_L1_M1_100): SAVE1x1 -.Lsgemm_kernel_L1_END: +L(sgemm_kernel_L1_END): -.Lsgemm_kernel_L999: +L(sgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/sgemm_kernel_16x4_thunderx2t99.S b/kernel/arm64/sgemm_kernel_16x4_thunderx2t99.S index 144d4bcd69..e2a225f384 100644 --- a/kernel/arm64/sgemm_kernel_16x4_thunderx2t99.S +++ b/kernel/arm64/sgemm_kernel_16x4_thunderx2t99.S @@ -1117,7 +1117,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE -.Lsgemm_kernel_begin: +L(sgemm_kernel_begin): .align 5 add sp, sp, #-(11 * 16) @@ -1145,11 +1145,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lsgemm_kernel_L2_BEGIN + ble L(sgemm_kernel_L2_BEGIN) /******************************************************************************/ -.Lsgemm_kernel_L4_BEGIN: +L(sgemm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -1159,21 +1159,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lsgemm_kernel_L4_M16_BEGIN: +L(sgemm_kernel_L4_M16_BEGIN): mov counterI, origM asr counterI, counterI, #4 // counterI = counterI / 16 cmp counterI, #0 - ble .Lsgemm_kernel_L4_M8_BEGIN + ble L(sgemm_kernel_L4_M8_BEGIN) .align 5 -.Lsgemm_kernel_L4_M16_20: +L(sgemm_kernel_L4_M16_20): mov pB, origPB asr counterL , origK, #4 // L = K / 16 cmp counterL , #2 - blt .Lsgemm_kernel_L4_M16_32 + blt L(sgemm_kernel_L4_M16_32) KERNEL16x4_I KERNEL16x4_M2 @@ -1182,18 +1182,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x4_M1_M2_x1 subs counterL, counterL, #2 - ble .Lsgemm_kernel_L4_M16_22a + ble L(sgemm_kernel_L4_M16_22a) .align 5 -.Lsgemm_kernel_L4_M16_22: +L(sgemm_kernel_L4_M16_22): KERNEL16x4_M1_M2_x8 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M16_22 + bgt L(sgemm_kernel_L4_M16_22) .align 5 -.Lsgemm_kernel_L4_M16_22a: +L(sgemm_kernel_L4_M16_22a): KERNEL16x4_M1_M2_x4 KERNEL16x4_M1_M2_x2 @@ -1201,13 +1201,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x4_M1 KERNEL16x4_E - b .Lsgemm_kernel_L4_M16_44 + b L(sgemm_kernel_L4_M16_44) .align 5 -.Lsgemm_kernel_L4_M16_32: +L(sgemm_kernel_L4_M16_32): tst counterL, #1 - ble .Lsgemm_kernel_L4_M16_40 + ble L(sgemm_kernel_L4_M16_40) KERNEL16x4_I KERNEL16x4_M2 @@ -1216,187 +1216,187 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x4_M1 KERNEL16x4_E - b .Lsgemm_kernel_L4_M16_44 + b L(sgemm_kernel_L4_M16_44) -.Lsgemm_kernel_L4_M16_40: +L(sgemm_kernel_L4_M16_40): INIT16x4 -.Lsgemm_kernel_L4_M16_44: +L(sgemm_kernel_L4_M16_44): ands counterL , origK, #15 - ble .Lsgemm_kernel_L4_M16_100 + ble L(sgemm_kernel_L4_M16_100) .align 5 -.Lsgemm_kernel_L4_M16_46: +L(sgemm_kernel_L4_M16_46): KERNEL16x4_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L4_M16_46 + bne L(sgemm_kernel_L4_M16_46) -.Lsgemm_kernel_L4_M16_100: +L(sgemm_kernel_L4_M16_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVE16x4 -.Lsgemm_kernel_L4_M16_END: +L(sgemm_kernel_L4_M16_END): subs counterI, counterI, #1 - bne .Lsgemm_kernel_L4_M16_20 + bne L(sgemm_kernel_L4_M16_20) //------------------------------------------------------------------------------ -.Lsgemm_kernel_L4_M8_BEGIN: +L(sgemm_kernel_L4_M8_BEGIN): mov counterI, origM tst counterI , #15 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #8 - ble .Lsgemm_kernel_L4_M4_BEGIN + ble L(sgemm_kernel_L4_M4_BEGIN) -.Lsgemm_kernel_L4_M8_20: +L(sgemm_kernel_L4_M8_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L4_M8_32 + blt L(sgemm_kernel_L4_M8_32) KERNEL8x4_I // do one in the K KERNEL8x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L4_M8_22a + ble L(sgemm_kernel_L4_M8_22a) .align 5 -.Lsgemm_kernel_L4_M8_22: +L(sgemm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M8_22 + bgt L(sgemm_kernel_L4_M8_22) -.Lsgemm_kernel_L4_M8_22a: +L(sgemm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_E - b .Lsgemm_kernel_L4_M8_44 + b L(sgemm_kernel_L4_M8_44) -.Lsgemm_kernel_L4_M8_32: +L(sgemm_kernel_L4_M8_32): tst counterL, #1 - ble .Lsgemm_kernel_L4_M8_40 + ble L(sgemm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_E - b .Lsgemm_kernel_L4_M8_44 + b L(sgemm_kernel_L4_M8_44) -.Lsgemm_kernel_L4_M8_40: +L(sgemm_kernel_L4_M8_40): INIT8x4 -.Lsgemm_kernel_L4_M8_44: +L(sgemm_kernel_L4_M8_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L4_M8_100 + ble L(sgemm_kernel_L4_M8_100) -.Lsgemm_kernel_L4_M8_46: +L(sgemm_kernel_L4_M8_46): KERNEL8x4_SUB -.Lsgemm_kernel_L4_M8_100: +L(sgemm_kernel_L4_M8_100): SAVE8x4 -.Lsgemm_kernel_L4_M8_END: +L(sgemm_kernel_L4_M8_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L4_M4_BEGIN: +L(sgemm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #4 - ble .Lsgemm_kernel_L4_M2_BEGIN + ble L(sgemm_kernel_L4_M2_BEGIN) -.Lsgemm_kernel_L4_M4_20: +L(sgemm_kernel_L4_M4_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L4_M4_32 + blt L(sgemm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L4_M4_22a + ble L(sgemm_kernel_L4_M4_22a) .align 5 -.Lsgemm_kernel_L4_M4_22: +L(sgemm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M4_22 + bgt L(sgemm_kernel_L4_M4_22) -.Lsgemm_kernel_L4_M4_22a: +L(sgemm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lsgemm_kernel_L4_M4_44 + b L(sgemm_kernel_L4_M4_44) -.Lsgemm_kernel_L4_M4_32: +L(sgemm_kernel_L4_M4_32): tst counterL, #1 - ble .Lsgemm_kernel_L4_M4_40 + ble L(sgemm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lsgemm_kernel_L4_M4_44 + b L(sgemm_kernel_L4_M4_44) -.Lsgemm_kernel_L4_M4_40: +L(sgemm_kernel_L4_M4_40): INIT4x4 -.Lsgemm_kernel_L4_M4_44: +L(sgemm_kernel_L4_M4_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L4_M4_100 + ble L(sgemm_kernel_L4_M4_100) -.Lsgemm_kernel_L4_M4_46: +L(sgemm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lsgemm_kernel_L4_M4_100: +L(sgemm_kernel_L4_M4_100): SAVE4x4 -.Lsgemm_kernel_L4_M4_END: +L(sgemm_kernel_L4_M4_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L4_M2_BEGIN: +L(sgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L4_M1_BEGIN + ble L(sgemm_kernel_L4_M1_BEGIN) -.Lsgemm_kernel_L4_M2_20: +L(sgemm_kernel_L4_M2_20): INIT2x4 @@ -1404,9 +1404,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L4_M2_40 + ble L(sgemm_kernel_L4_M2_40) -.Lsgemm_kernel_L4_M2_22: +L(sgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1419,34 +1419,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M2_22 + bgt L(sgemm_kernel_L4_M2_22) -.Lsgemm_kernel_L4_M2_40: +L(sgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M2_100 + ble L(sgemm_kernel_L4_M2_100) -.Lsgemm_kernel_L4_M2_42: +L(sgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M2_42 + bgt L(sgemm_kernel_L4_M2_42) -.Lsgemm_kernel_L4_M2_100: +L(sgemm_kernel_L4_M2_100): SAVE2x4 -.Lsgemm_kernel_L4_M2_END: +L(sgemm_kernel_L4_M2_END): -.Lsgemm_kernel_L4_M1_BEGIN: +L(sgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) -.Lsgemm_kernel_L4_M1_20: +L(sgemm_kernel_L4_M1_20): INIT1x4 @@ -1454,9 +1454,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L4_M1_40 + ble L(sgemm_kernel_L4_M1_40) -.Lsgemm_kernel_L4_M1_22: +L(sgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1468,42 +1468,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M1_22 + bgt L(sgemm_kernel_L4_M1_22) -.Lsgemm_kernel_L4_M1_40: +L(sgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M1_100 + ble L(sgemm_kernel_L4_M1_100) -.Lsgemm_kernel_L4_M1_42: +L(sgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M1_42 + bgt L(sgemm_kernel_L4_M1_42) -.Lsgemm_kernel_L4_M1_100: +L(sgemm_kernel_L4_M1_100): SAVE1x4 -.Lsgemm_kernel_L4_END: +L(sgemm_kernel_L4_END): add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4 subs counterJ, counterJ , #1 // j-- - bgt .Lsgemm_kernel_L4_BEGIN + bgt L(sgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Lsgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(sgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lsgemm_kernel_L999 + ble L(sgemm_kernel_L999) tst counterJ , #2 - ble .Lsgemm_kernel_L1_BEGIN + ble L(sgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1511,14 +1511,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lsgemm_kernel_L2_M16_BEGIN: +L(sgemm_kernel_L2_M16_BEGIN): mov counterI, origM asr counterI, counterI, #4 // counterI = counterI / 16 cmp counterI,#0 - ble .Lsgemm_kernel_L2_M8_BEGIN + ble L(sgemm_kernel_L2_M8_BEGIN) -.Lsgemm_kernel_L2_M16_20: +L(sgemm_kernel_L2_M16_20): INIT16x2 @@ -1526,10 +1526,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M16_40 + ble L(sgemm_kernel_L2_M16_40) .align 5 -.Lsgemm_kernel_L2_M16_22: +L(sgemm_kernel_L2_M16_22): KERNEL16x2_SUB KERNEL16x2_SUB KERNEL16x2_SUB @@ -1541,41 +1541,41 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M16_22 + bgt L(sgemm_kernel_L2_M16_22) -.Lsgemm_kernel_L2_M16_40: +L(sgemm_kernel_L2_M16_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M16_100 + ble L(sgemm_kernel_L2_M16_100) -.Lsgemm_kernel_L2_M16_42: +L(sgemm_kernel_L2_M16_42): KERNEL16x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M16_42 + bgt L(sgemm_kernel_L2_M16_42) -.Lsgemm_kernel_L2_M16_100: +L(sgemm_kernel_L2_M16_100): SAVE16x2 -.Lsgemm_kernel_L2_M16_END: +L(sgemm_kernel_L2_M16_END): subs counterI, counterI, #1 - bgt .Lsgemm_kernel_L2_M16_20 + bgt L(sgemm_kernel_L2_M16_20) //------------------------------------------------------------------------------ -.Lsgemm_kernel_L2_M8_BEGIN: +L(sgemm_kernel_L2_M8_BEGIN): mov counterI, origM tst counterI , #15 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) tst counterI, #8 - ble .Lsgemm_kernel_L2_M4_BEGIN + ble L(sgemm_kernel_L2_M4_BEGIN) -.Lsgemm_kernel_L2_M8_20: +L(sgemm_kernel_L2_M8_20): INIT8x2 @@ -1583,10 +1583,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M8_40 + ble L(sgemm_kernel_L2_M8_40) .align 5 -.Lsgemm_kernel_L2_M8_22: +L(sgemm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB KERNEL8x2_SUB @@ -1598,38 +1598,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M8_22 + bgt L(sgemm_kernel_L2_M8_22) -.Lsgemm_kernel_L2_M8_40: +L(sgemm_kernel_L2_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M8_100 + ble L(sgemm_kernel_L2_M8_100) -.Lsgemm_kernel_L2_M8_42: +L(sgemm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M8_42 + bgt L(sgemm_kernel_L2_M8_42) -.Lsgemm_kernel_L2_M8_100: +L(sgemm_kernel_L2_M8_100): SAVE8x2 -.Lsgemm_kernel_L2_M8_END: +L(sgemm_kernel_L2_M8_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L2_M4_BEGIN: +L(sgemm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) tst counterI, #4 - ble .Lsgemm_kernel_L2_M2_BEGIN + ble L(sgemm_kernel_L2_M2_BEGIN) -.Lsgemm_kernel_L2_M4_20: +L(sgemm_kernel_L2_M4_20): INIT4x2 @@ -1637,10 +1637,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M4_40 + ble L(sgemm_kernel_L2_M4_40) .align 5 -.Lsgemm_kernel_L2_M4_22: +L(sgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1652,40 +1652,40 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M4_22 + bgt L(sgemm_kernel_L2_M4_22) -.Lsgemm_kernel_L2_M4_40: +L(sgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M4_100 + ble L(sgemm_kernel_L2_M4_100) -.Lsgemm_kernel_L2_M4_42: +L(sgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M4_42 + bgt L(sgemm_kernel_L2_M4_42) -.Lsgemm_kernel_L2_M4_100: +L(sgemm_kernel_L2_M4_100): SAVE4x2 -.Lsgemm_kernel_L2_M4_END: +L(sgemm_kernel_L2_M4_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L2_M2_BEGIN: +L(sgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L2_M1_BEGIN + ble L(sgemm_kernel_L2_M1_BEGIN) -.Lsgemm_kernel_L2_M2_20: +L(sgemm_kernel_L2_M2_20): INIT2x2 @@ -1693,9 +1693,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M2_40 + ble L(sgemm_kernel_L2_M2_40) -.Lsgemm_kernel_L2_M2_22: +L(sgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1708,34 +1708,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M2_22 + bgt L(sgemm_kernel_L2_M2_22) -.Lsgemm_kernel_L2_M2_40: +L(sgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M2_100 + ble L(sgemm_kernel_L2_M2_100) -.Lsgemm_kernel_L2_M2_42: +L(sgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M2_42 + bgt L(sgemm_kernel_L2_M2_42) -.Lsgemm_kernel_L2_M2_100: +L(sgemm_kernel_L2_M2_100): SAVE2x2 -.Lsgemm_kernel_L2_M2_END: +L(sgemm_kernel_L2_M2_END): -.Lsgemm_kernel_L2_M1_BEGIN: +L(sgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) -.Lsgemm_kernel_L2_M1_20: +L(sgemm_kernel_L2_M1_20): INIT1x2 @@ -1743,9 +1743,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lsgemm_kernel_L2_M1_40 + ble L(sgemm_kernel_L2_M1_40) -.Lsgemm_kernel_L2_M1_22: +L(sgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1757,36 +1757,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M1_22 + bgt L(sgemm_kernel_L2_M1_22) -.Lsgemm_kernel_L2_M1_40: +L(sgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M1_100 + ble L(sgemm_kernel_L2_M1_100) -.Lsgemm_kernel_L2_M1_42: +L(sgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M1_42 + bgt L(sgemm_kernel_L2_M1_42) -.Lsgemm_kernel_L2_M1_100: +L(sgemm_kernel_L2_M1_100): SAVE1x2 -.Lsgemm_kernel_L2_END: +L(sgemm_kernel_L2_END): add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 /******************************************************************************/ -.Lsgemm_kernel_L1_BEGIN: +L(sgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lsgemm_kernel_L999 // done + ble L(sgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1794,14 +1794,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lsgemm_kernel_L1_M16_BEGIN: +L(sgemm_kernel_L1_M16_BEGIN): mov counterI, origM asr counterI, counterI, #4 // counterI = counterI / 16 cmp counterI, #0 - ble .Lsgemm_kernel_L1_M8_BEGIN + ble L(sgemm_kernel_L1_M8_BEGIN) -.Lsgemm_kernel_L1_M16_20: +L(sgemm_kernel_L1_M16_20): INIT16x1 @@ -1809,10 +1809,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M16_40 + ble L(sgemm_kernel_L1_M16_40) .align 5 -.Lsgemm_kernel_L1_M16_22: +L(sgemm_kernel_L1_M16_22): KERNEL16x1_SUB KERNEL16x1_SUB KERNEL16x1_SUB @@ -1824,42 +1824,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M16_22 + bgt L(sgemm_kernel_L1_M16_22) -.Lsgemm_kernel_L1_M16_40: +L(sgemm_kernel_L1_M16_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M16_100 + ble L(sgemm_kernel_L1_M16_100) -.Lsgemm_kernel_L1_M16_42: +L(sgemm_kernel_L1_M16_42): KERNEL16x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M16_42 + bgt L(sgemm_kernel_L1_M16_42) -.Lsgemm_kernel_L1_M16_100: +L(sgemm_kernel_L1_M16_100): SAVE16x1 -.Lsgemm_kernel_L1_M16_END: +L(sgemm_kernel_L1_M16_END): subs counterI, counterI, #1 - bgt .Lsgemm_kernel_L1_M16_20 + bgt L(sgemm_kernel_L1_M16_20) //------------------------------------------------------------------------------ -.Lsgemm_kernel_L1_M8_BEGIN: +L(sgemm_kernel_L1_M8_BEGIN): mov counterI, origM tst counterI , #15 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) tst counterI, #8 - ble .Lsgemm_kernel_L1_M4_BEGIN + ble L(sgemm_kernel_L1_M4_BEGIN) -.Lsgemm_kernel_L1_M8_20: +L(sgemm_kernel_L1_M8_20): INIT8x1 @@ -1867,10 +1867,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M8_40 + ble L(sgemm_kernel_L1_M8_40) .align 5 -.Lsgemm_kernel_L1_M8_22: +L(sgemm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -1882,38 +1882,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M8_22 + bgt L(sgemm_kernel_L1_M8_22) -.Lsgemm_kernel_L1_M8_40: +L(sgemm_kernel_L1_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M8_100 + ble L(sgemm_kernel_L1_M8_100) -.Lsgemm_kernel_L1_M8_42: +L(sgemm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M8_42 + bgt L(sgemm_kernel_L1_M8_42) -.Lsgemm_kernel_L1_M8_100: +L(sgemm_kernel_L1_M8_100): SAVE8x1 -.Lsgemm_kernel_L1_M8_END: +L(sgemm_kernel_L1_M8_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L1_M4_BEGIN: +L(sgemm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) tst counterI, #4 - ble .Lsgemm_kernel_L1_M2_BEGIN + ble L(sgemm_kernel_L1_M2_BEGIN) -.Lsgemm_kernel_L1_M4_20: +L(sgemm_kernel_L1_M4_20): INIT4x1 @@ -1921,10 +1921,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M4_40 + ble L(sgemm_kernel_L1_M4_40) .align 5 -.Lsgemm_kernel_L1_M4_22: +L(sgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1936,39 +1936,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M4_22 + bgt L(sgemm_kernel_L1_M4_22) -.Lsgemm_kernel_L1_M4_40: +L(sgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M4_100 + ble L(sgemm_kernel_L1_M4_100) -.Lsgemm_kernel_L1_M4_42: +L(sgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M4_42 + bgt L(sgemm_kernel_L1_M4_42) -.Lsgemm_kernel_L1_M4_100: +L(sgemm_kernel_L1_M4_100): SAVE4x1 -.Lsgemm_kernel_L1_M4_END: +L(sgemm_kernel_L1_M4_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L1_M2_BEGIN: +L(sgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L1_M1_BEGIN + ble L(sgemm_kernel_L1_M1_BEGIN) -.Lsgemm_kernel_L1_M2_20: +L(sgemm_kernel_L1_M2_20): INIT2x1 @@ -1976,9 +1976,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M2_40 + ble L(sgemm_kernel_L1_M2_40) -.Lsgemm_kernel_L1_M2_22: +L(sgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1991,34 +1991,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M2_22 + bgt L(sgemm_kernel_L1_M2_22) -.Lsgemm_kernel_L1_M2_40: +L(sgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M2_100 + ble L(sgemm_kernel_L1_M2_100) -.Lsgemm_kernel_L1_M2_42: +L(sgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M2_42 + bgt L(sgemm_kernel_L1_M2_42) -.Lsgemm_kernel_L1_M2_100: +L(sgemm_kernel_L1_M2_100): SAVE2x1 -.Lsgemm_kernel_L1_M2_END: +L(sgemm_kernel_L1_M2_END): -.Lsgemm_kernel_L1_M1_BEGIN: +L(sgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) -.Lsgemm_kernel_L1_M1_20: +L(sgemm_kernel_L1_M1_20): INIT1x1 @@ -2026,9 +2026,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M1_40 + ble L(sgemm_kernel_L1_M1_40) -.Lsgemm_kernel_L1_M1_22: +L(sgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -2040,28 +2040,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M1_22 + bgt L(sgemm_kernel_L1_M1_22) -.Lsgemm_kernel_L1_M1_40: +L(sgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M1_100 + ble L(sgemm_kernel_L1_M1_100) -.Lsgemm_kernel_L1_M1_42: +L(sgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M1_42 + bgt L(sgemm_kernel_L1_M1_42) -.Lsgemm_kernel_L1_M1_100: +L(sgemm_kernel_L1_M1_100): SAVE1x1 -.Lsgemm_kernel_L1_END: +L(sgemm_kernel_L1_END): -.Lsgemm_kernel_L999: +L(sgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/sgemm_kernel_4x4.S b/kernel/arm64/sgemm_kernel_4x4.S index 76c11f1e14..354aaa2994 100644 --- a/kernel/arm64/sgemm_kernel_4x4.S +++ b/kernel/arm64/sgemm_kernel_4x4.S @@ -892,11 +892,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lsgemm_kernel_L2_BEGIN + ble L(sgemm_kernel_L2_BEGIN) /******************************************************************************/ -.Lsgemm_kernel_L4_BEGIN: +L(sgemm_kernel_L4_BEGIN): mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #2 @@ -906,73 +906,73 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add pA_2, temp, pA_1 add pA_3, temp, pA_2 -.Lsgemm_kernel_L4_M16_BEGIN: +L(sgemm_kernel_L4_M16_BEGIN): mov counterI, origM asr counterI, counterI, #4 // counterI = counterI / 16 cmp counterI, #0 - ble .Lsgemm_kernel_L4_M8_BEGIN + ble L(sgemm_kernel_L4_M8_BEGIN) -.Lsgemm_kernel_L4_M16_20: +L(sgemm_kernel_L4_M16_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L4_M16_32 + blt L(sgemm_kernel_L4_M16_32) KERNEL16x4_I // do one in the K KERNEL16x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L4_M16_22a + ble L(sgemm_kernel_L4_M16_22a) .align 5 -.Lsgemm_kernel_L4_M16_22: +L(sgemm_kernel_L4_M16_22): KERNEL16x4_M1 KERNEL16x4_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M16_22 + bgt L(sgemm_kernel_L4_M16_22) -.Lsgemm_kernel_L4_M16_22a: +L(sgemm_kernel_L4_M16_22a): KERNEL16x4_M1 KERNEL16x4_E - b .Lsgemm_kernel_L4_M16_44 + b L(sgemm_kernel_L4_M16_44) -.Lsgemm_kernel_L4_M16_32: +L(sgemm_kernel_L4_M16_32): tst counterL, #1 - ble .Lsgemm_kernel_L4_M16_40 + ble L(sgemm_kernel_L4_M16_40) KERNEL16x4_I KERNEL16x4_E - b .Lsgemm_kernel_L4_M16_44 + b L(sgemm_kernel_L4_M16_44) -.Lsgemm_kernel_L4_M16_40: +L(sgemm_kernel_L4_M16_40): INIT16x4 -.Lsgemm_kernel_L4_M16_44: +L(sgemm_kernel_L4_M16_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L4_M16_100 + ble L(sgemm_kernel_L4_M16_100) -.Lsgemm_kernel_L4_M16_46: +L(sgemm_kernel_L4_M16_46): KERNEL16x4_SUB -.Lsgemm_kernel_L4_M16_100: +L(sgemm_kernel_L4_M16_100): SAVE16x4 -.Lsgemm_kernel_L4_M16_END: +L(sgemm_kernel_L4_M16_END): lsl temp, origK, #4 // k * 4 * 4 = Four rows of A add pA_0, pA_0, temp add pA_0, pA_0, temp @@ -981,26 +981,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add pA_2, pA_1, temp add pA_3, pA_2, temp subs counterI, counterI, #1 - bne .Lsgemm_kernel_L4_M16_20 + bne L(sgemm_kernel_L4_M16_20) -.Lsgemm_kernel_L4_M8_BEGIN: +L(sgemm_kernel_L4_M8_BEGIN): mov counterI, origM tst counterI , #15 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #8 - ble .Lsgemm_kernel_L4_M4_BEGIN + ble L(sgemm_kernel_L4_M4_BEGIN) -.Lsgemm_kernel_L4_M8_20: +L(sgemm_kernel_L4_M8_20): INIT8x4 mov pB, origPB asr counterL, origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lsgemm_kernel_L4_M8_40 + ble L(sgemm_kernel_L4_M8_40) -.Lsgemm_kernel_L4_M8_22: +L(sgemm_kernel_L4_M8_22): KERNEL8x4_SUB KERNEL8x4_SUB @@ -1013,47 +1013,47 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M8_22 + bgt L(sgemm_kernel_L4_M8_22) -.Lsgemm_kernel_L4_M8_40: +L(sgemm_kernel_L4_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M8_100 + ble L(sgemm_kernel_L4_M8_100) -.Lsgemm_kernel_L4_M8_42: +L(sgemm_kernel_L4_M8_42): KERNEL8x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M8_42 + bgt L(sgemm_kernel_L4_M8_42) -.Lsgemm_kernel_L4_M8_100: +L(sgemm_kernel_L4_M8_100): SAVE8x4 -.Lsgemm_kernel_L4_M8_END: +L(sgemm_kernel_L4_M8_END): lsl temp, origK, #4 // k * 4 * 4 add pA_0, pA_0, temp -.Lsgemm_kernel_L4_M4_BEGIN: +L(sgemm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #4 - ble .Lsgemm_kernel_L4_M2_BEGIN + ble L(sgemm_kernel_L4_M2_BEGIN) -.Lsgemm_kernel_L4_M4_20: +L(sgemm_kernel_L4_M4_20): INIT4x4 mov pB, origPB asr counterL, origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lsgemm_kernel_L4_M4_40 + ble L(sgemm_kernel_L4_M4_40) -.Lsgemm_kernel_L4_M4_22: +L(sgemm_kernel_L4_M4_22): KERNEL4x4_SUB KERNEL4x4_SUB @@ -1066,47 +1066,47 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M4_22 + bgt L(sgemm_kernel_L4_M4_22) -.Lsgemm_kernel_L4_M4_40: +L(sgemm_kernel_L4_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M4_100 + ble L(sgemm_kernel_L4_M4_100) -.Lsgemm_kernel_L4_M4_42: +L(sgemm_kernel_L4_M4_42): KERNEL4x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M4_42 + bgt L(sgemm_kernel_L4_M4_42) -.Lsgemm_kernel_L4_M4_100: +L(sgemm_kernel_L4_M4_100): SAVE4x4 -.Lsgemm_kernel_L4_M4_END: +L(sgemm_kernel_L4_M4_END): -.Lsgemm_kernel_L4_M2_BEGIN: +L(sgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L4_M1_BEGIN + ble L(sgemm_kernel_L4_M1_BEGIN) -.Lsgemm_kernel_L4_M2_20: +L(sgemm_kernel_L4_M2_20): INIT2x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L4_M2_40 + ble L(sgemm_kernel_L4_M2_40) -.Lsgemm_kernel_L4_M2_22: +L(sgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1119,43 +1119,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M2_22 + bgt L(sgemm_kernel_L4_M2_22) -.Lsgemm_kernel_L4_M2_40: +L(sgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M2_100 + ble L(sgemm_kernel_L4_M2_100) -.Lsgemm_kernel_L4_M2_42: +L(sgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M2_42 + bgt L(sgemm_kernel_L4_M2_42) -.Lsgemm_kernel_L4_M2_100: +L(sgemm_kernel_L4_M2_100): SAVE2x4 -.Lsgemm_kernel_L4_M2_END: +L(sgemm_kernel_L4_M2_END): -.Lsgemm_kernel_L4_M1_BEGIN: +L(sgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) -.Lsgemm_kernel_L4_M1_20: +L(sgemm_kernel_L4_M1_20): INIT1x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L4_M1_40 + ble L(sgemm_kernel_L4_M1_40) -.Lsgemm_kernel_L4_M1_22: +L(sgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1167,45 +1167,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M1_22 + bgt L(sgemm_kernel_L4_M1_22) -.Lsgemm_kernel_L4_M1_40: +L(sgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M1_100 + ble L(sgemm_kernel_L4_M1_100) -.Lsgemm_kernel_L4_M1_42: +L(sgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M1_42 + bgt L(sgemm_kernel_L4_M1_42) -.Lsgemm_kernel_L4_M1_100: +L(sgemm_kernel_L4_M1_100): SAVE1x4 -.Lsgemm_kernel_L4_END: +L(sgemm_kernel_L4_END): lsl temp, origK, #4 add origPB, origPB, temp // B = B + K * 4 * 4 subs counterJ, counterJ , #1 // j-- - bgt .Lsgemm_kernel_L4_BEGIN + bgt L(sgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Lsgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(sgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lsgemm_kernel_L999 + ble L(sgemm_kernel_L999) tst counterJ , #2 - ble .Lsgemm_kernel_L1_BEGIN + ble L(sgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1215,24 +1215,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lsgemm_kernel_L2_M4_BEGIN: +L(sgemm_kernel_L2_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI,#0 - ble .Lsgemm_kernel_L2_M2_BEGIN + ble L(sgemm_kernel_L2_M2_BEGIN) -.Lsgemm_kernel_L2_M4_20: +L(sgemm_kernel_L2_M4_20): INIT4x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M4_40 + ble L(sgemm_kernel_L2_M4_40) .align 5 -.Lsgemm_kernel_L2_M4_22: +L(sgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1244,50 +1244,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M4_22 + bgt L(sgemm_kernel_L2_M4_22) -.Lsgemm_kernel_L2_M4_40: +L(sgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M4_100 + ble L(sgemm_kernel_L2_M4_100) -.Lsgemm_kernel_L2_M4_42: +L(sgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M4_42 + bgt L(sgemm_kernel_L2_M4_42) -.Lsgemm_kernel_L2_M4_100: +L(sgemm_kernel_L2_M4_100): SAVE4x2 -.Lsgemm_kernel_L2_M4_END: +L(sgemm_kernel_L2_M4_END): subs counterI, counterI, #1 - bgt .Lsgemm_kernel_L2_M4_20 + bgt L(sgemm_kernel_L2_M4_20) -.Lsgemm_kernel_L2_M2_BEGIN: +L(sgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L2_M1_BEGIN + ble L(sgemm_kernel_L2_M1_BEGIN) -.Lsgemm_kernel_L2_M2_20: +L(sgemm_kernel_L2_M2_20): INIT2x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M2_40 + ble L(sgemm_kernel_L2_M2_40) -.Lsgemm_kernel_L2_M2_22: +L(sgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1300,43 +1300,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M2_22 + bgt L(sgemm_kernel_L2_M2_22) -.Lsgemm_kernel_L2_M2_40: +L(sgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M2_100 + ble L(sgemm_kernel_L2_M2_100) -.Lsgemm_kernel_L2_M2_42: +L(sgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M2_42 + bgt L(sgemm_kernel_L2_M2_42) -.Lsgemm_kernel_L2_M2_100: +L(sgemm_kernel_L2_M2_100): SAVE2x2 -.Lsgemm_kernel_L2_M2_END: +L(sgemm_kernel_L2_M2_END): -.Lsgemm_kernel_L2_M1_BEGIN: +L(sgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) -.Lsgemm_kernel_L2_M1_20: +L(sgemm_kernel_L2_M1_20): INIT1x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lsgemm_kernel_L2_M1_40 + ble L(sgemm_kernel_L2_M1_40) -.Lsgemm_kernel_L2_M1_22: +L(sgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1348,36 +1348,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M1_22 + bgt L(sgemm_kernel_L2_M1_22) -.Lsgemm_kernel_L2_M1_40: +L(sgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M1_100 + ble L(sgemm_kernel_L2_M1_100) -.Lsgemm_kernel_L2_M1_42: +L(sgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M1_42 + bgt L(sgemm_kernel_L2_M1_42) -.Lsgemm_kernel_L2_M1_100: +L(sgemm_kernel_L2_M1_100): SAVE1x2 -.Lsgemm_kernel_L2_END: +L(sgemm_kernel_L2_END): add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 /******************************************************************************/ -.Lsgemm_kernel_L1_BEGIN: +L(sgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lsgemm_kernel_L999 // done + ble L(sgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1387,24 +1387,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lsgemm_kernel_L1_M4_BEGIN: +L(sgemm_kernel_L1_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lsgemm_kernel_L1_M2_BEGIN + ble L(sgemm_kernel_L1_M2_BEGIN) -.Lsgemm_kernel_L1_M4_20: +L(sgemm_kernel_L1_M4_20): INIT4x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M4_40 + ble L(sgemm_kernel_L1_M4_40) .align 5 -.Lsgemm_kernel_L1_M4_22: +L(sgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1416,50 +1416,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M4_22 + bgt L(sgemm_kernel_L1_M4_22) -.Lsgemm_kernel_L1_M4_40: +L(sgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M4_100 + ble L(sgemm_kernel_L1_M4_100) -.Lsgemm_kernel_L1_M4_42: +L(sgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M4_42 + bgt L(sgemm_kernel_L1_M4_42) -.Lsgemm_kernel_L1_M4_100: +L(sgemm_kernel_L1_M4_100): SAVE4x1 -.Lsgemm_kernel_L1_M4_END: +L(sgemm_kernel_L1_M4_END): subs counterI, counterI, #1 - bgt .Lsgemm_kernel_L1_M4_20 + bgt L(sgemm_kernel_L1_M4_20) -.Lsgemm_kernel_L1_M2_BEGIN: +L(sgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L1_M1_BEGIN + ble L(sgemm_kernel_L1_M1_BEGIN) -.Lsgemm_kernel_L1_M2_20: +L(sgemm_kernel_L1_M2_20): INIT2x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M2_40 + ble L(sgemm_kernel_L1_M2_40) -.Lsgemm_kernel_L1_M2_22: +L(sgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1472,43 +1472,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M2_22 + bgt L(sgemm_kernel_L1_M2_22) -.Lsgemm_kernel_L1_M2_40: +L(sgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M2_100 + ble L(sgemm_kernel_L1_M2_100) -.Lsgemm_kernel_L1_M2_42: +L(sgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M2_42 + bgt L(sgemm_kernel_L1_M2_42) -.Lsgemm_kernel_L1_M2_100: +L(sgemm_kernel_L1_M2_100): SAVE2x1 -.Lsgemm_kernel_L1_M2_END: +L(sgemm_kernel_L1_M2_END): -.Lsgemm_kernel_L1_M1_BEGIN: +L(sgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) -.Lsgemm_kernel_L1_M1_20: +L(sgemm_kernel_L1_M1_20): INIT1x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M1_40 + ble L(sgemm_kernel_L1_M1_40) -.Lsgemm_kernel_L1_M1_22: +L(sgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1520,30 +1520,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M1_22 + bgt L(sgemm_kernel_L1_M1_22) -.Lsgemm_kernel_L1_M1_40: +L(sgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M1_100 + ble L(sgemm_kernel_L1_M1_100) -.Lsgemm_kernel_L1_M1_42: +L(sgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M1_42 + bgt L(sgemm_kernel_L1_M1_42) -.Lsgemm_kernel_L1_M1_100: +L(sgemm_kernel_L1_M1_100): SAVE1x1 -.Lsgemm_kernel_L1_END: +L(sgemm_kernel_L1_END): -.Lsgemm_kernel_L999: +L(sgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/sgemm_kernel_8x8.S b/kernel/arm64/sgemm_kernel_8x8.S index 6ba64dd355..4d73fb1e81 100644 --- a/kernel/arm64/sgemm_kernel_8x8.S +++ b/kernel/arm64/sgemm_kernel_8x8.S @@ -1263,7 +1263,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE -.Lsgemm_kernel_begin: +L(sgemm_kernel_begin): .align 5 add sp, sp, #-(11 * 16) @@ -1291,12 +1291,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Lsgemm_kernel_L4_BEGIN + ble L(sgemm_kernel_L4_BEGIN) /******************************************************************************/ /******************************************************************************/ -.Lsgemm_kernel_L8_BEGIN: +L(sgemm_kernel_L8_BEGIN): mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #3 @@ -1304,156 +1304,156 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lsgemm_kernel_L8_M8_BEGIN: +L(sgemm_kernel_L8_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lsgemm_kernel_L8_M4_BEGIN + ble L(sgemm_kernel_L8_M4_BEGIN) -.Lsgemm_kernel_L8_M8_20: +L(sgemm_kernel_L8_M8_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L8_M8_32 + blt L(sgemm_kernel_L8_M8_32) KERNEL8x8_I // do one in the K KERNEL8x8_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L8_M8_22a + ble L(sgemm_kernel_L8_M8_22a) .align 5 -.Lsgemm_kernel_L8_M8_22: +L(sgemm_kernel_L8_M8_22): KERNEL8x8_M1 KERNEL8x8_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M8_22 + bgt L(sgemm_kernel_L8_M8_22) -.Lsgemm_kernel_L8_M8_22a: +L(sgemm_kernel_L8_M8_22a): KERNEL8x8_M1 KERNEL8x8_E - b .Lsgemm_kernel_L8_M8_44 + b L(sgemm_kernel_L8_M8_44) -.Lsgemm_kernel_L8_M8_32: +L(sgemm_kernel_L8_M8_32): tst counterL, #1 - ble .Lsgemm_kernel_L8_M8_40 + ble L(sgemm_kernel_L8_M8_40) KERNEL8x8_I KERNEL8x8_E - b .Lsgemm_kernel_L8_M8_44 + b L(sgemm_kernel_L8_M8_44) -.Lsgemm_kernel_L8_M8_40: +L(sgemm_kernel_L8_M8_40): INIT8x8 -.Lsgemm_kernel_L8_M8_44: +L(sgemm_kernel_L8_M8_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L8_M8_100 + ble L(sgemm_kernel_L8_M8_100) -.Lsgemm_kernel_L8_M8_46: +L(sgemm_kernel_L8_M8_46): KERNEL8x8_SUB -.Lsgemm_kernel_L8_M8_100: +L(sgemm_kernel_L8_M8_100): SAVE8x8 -.Lsgemm_kernel_L8_M8_END: +L(sgemm_kernel_L8_M8_END): subs counterI, counterI, #1 - bne .Lsgemm_kernel_L8_M8_20 + bne L(sgemm_kernel_L8_M8_20) /******************************************************************************/ -.Lsgemm_kernel_L8_M4_BEGIN: +L(sgemm_kernel_L8_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L8_END + ble L(sgemm_kernel_L8_END) tst counterI, #4 - ble .Lsgemm_kernel_L8_M2_BEGIN + ble L(sgemm_kernel_L8_M2_BEGIN) -.Lsgemm_kernel_L8_M4_20: +L(sgemm_kernel_L8_M4_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L8_M4_32 + blt L(sgemm_kernel_L8_M4_32) KERNEL4x8_I // do one in the K KERNEL4x8_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L8_M4_22a + ble L(sgemm_kernel_L8_M4_22a) .align 5 -.Lsgemm_kernel_L8_M4_22: +L(sgemm_kernel_L8_M4_22): KERNEL4x8_M1 KERNEL4x8_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M4_22 + bgt L(sgemm_kernel_L8_M4_22) -.Lsgemm_kernel_L8_M4_22a: +L(sgemm_kernel_L8_M4_22a): KERNEL4x8_M1 KERNEL4x8_E - b .Lsgemm_kernel_L8_M4_44 + b L(sgemm_kernel_L8_M4_44) -.Lsgemm_kernel_L8_M4_32: +L(sgemm_kernel_L8_M4_32): tst counterL, #1 - ble .Lsgemm_kernel_L8_M4_40 + ble L(sgemm_kernel_L8_M4_40) KERNEL4x8_I KERNEL4x8_E - b .Lsgemm_kernel_L8_M4_44 + b L(sgemm_kernel_L8_M4_44) -.Lsgemm_kernel_L8_M4_40: +L(sgemm_kernel_L8_M4_40): INIT4x8 -.Lsgemm_kernel_L8_M4_44: +L(sgemm_kernel_L8_M4_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L8_M4_100 + ble L(sgemm_kernel_L8_M4_100) -.Lsgemm_kernel_L8_M4_46: +L(sgemm_kernel_L8_M4_46): KERNEL4x8_SUB -.Lsgemm_kernel_L8_M4_100: +L(sgemm_kernel_L8_M4_100): SAVE4x8 -.Lsgemm_kernel_L8_M4_END: +L(sgemm_kernel_L8_M4_END): /******************************************************************************/ -.Lsgemm_kernel_L8_M2_BEGIN: +L(sgemm_kernel_L8_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L8_END + ble L(sgemm_kernel_L8_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L8_M1_BEGIN + ble L(sgemm_kernel_L8_M1_BEGIN) -.Lsgemm_kernel_L8_M2_20: +L(sgemm_kernel_L8_M2_20): INIT2x8 @@ -1461,9 +1461,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L8_M2_40 + ble L(sgemm_kernel_L8_M2_40) -.Lsgemm_kernel_L8_M2_22: +L(sgemm_kernel_L8_M2_22): KERNEL2x8_SUB KERNEL2x8_SUB @@ -1476,35 +1476,35 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M2_22 + bgt L(sgemm_kernel_L8_M2_22) -.Lsgemm_kernel_L8_M2_40: +L(sgemm_kernel_L8_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L8_M2_100 + ble L(sgemm_kernel_L8_M2_100) -.Lsgemm_kernel_L8_M2_42: +L(sgemm_kernel_L8_M2_42): KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M2_42 + bgt L(sgemm_kernel_L8_M2_42) -.Lsgemm_kernel_L8_M2_100: +L(sgemm_kernel_L8_M2_100): SAVE2x8 -.Lsgemm_kernel_L8_M2_END: +L(sgemm_kernel_L8_M2_END): /******************************************************************************/ -.Lsgemm_kernel_L8_M1_BEGIN: +L(sgemm_kernel_L8_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L8_END + ble L(sgemm_kernel_L8_END) -.Lsgemm_kernel_L8_M1_20: +L(sgemm_kernel_L8_M1_20): INIT1x8 @@ -1512,9 +1512,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L8_M1_40 + ble L(sgemm_kernel_L8_M1_40) -.Lsgemm_kernel_L8_M1_22: +L(sgemm_kernel_L8_M1_22): KERNEL1x8_SUB KERNEL1x8_SUB KERNEL1x8_SUB @@ -1526,43 +1526,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M1_22 + bgt L(sgemm_kernel_L8_M1_22) -.Lsgemm_kernel_L8_M1_40: +L(sgemm_kernel_L8_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L8_M1_100 + ble L(sgemm_kernel_L8_M1_100) -.Lsgemm_kernel_L8_M1_42: +L(sgemm_kernel_L8_M1_42): KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M1_42 + bgt L(sgemm_kernel_L8_M1_42) -.Lsgemm_kernel_L8_M1_100: +L(sgemm_kernel_L8_M1_100): SAVE1x8 -.Lsgemm_kernel_L8_END: +L(sgemm_kernel_L8_END): lsl temp, origK, #5 // B = B + K * 4 * 8 add origPB, origPB, temp subs counterJ, counterJ , #1 // j-- - bgt .Lsgemm_kernel_L8_BEGIN + bgt L(sgemm_kernel_L8_BEGIN) /******************************************************************************/ /******************************************************************************/ -.Lsgemm_kernel_L4_BEGIN: +L(sgemm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #7 - ble .Lsgemm_kernel_L999 + ble L(sgemm_kernel_L999) tst counterJ , #4 - ble .Lsgemm_kernel_L2_BEGIN + ble L(sgemm_kernel_L2_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1572,156 +1572,156 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lsgemm_kernel_L4_M8_BEGIN: +L(sgemm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lsgemm_kernel_L4_M4_BEGIN + ble L(sgemm_kernel_L4_M4_BEGIN) -.Lsgemm_kernel_L4_M8_20: +L(sgemm_kernel_L4_M8_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L4_M8_32 + blt L(sgemm_kernel_L4_M8_32) KERNEL8x4_I // do one in the K KERNEL8x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L4_M8_22a + ble L(sgemm_kernel_L4_M8_22a) .align 5 -.Lsgemm_kernel_L4_M8_22: +L(sgemm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M8_22 + bgt L(sgemm_kernel_L4_M8_22) -.Lsgemm_kernel_L4_M8_22a: +L(sgemm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_E - b .Lsgemm_kernel_L4_M8_44 + b L(sgemm_kernel_L4_M8_44) -.Lsgemm_kernel_L4_M8_32: +L(sgemm_kernel_L4_M8_32): tst counterL, #1 - ble .Lsgemm_kernel_L4_M8_40 + ble L(sgemm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_E - b .Lsgemm_kernel_L4_M8_44 + b L(sgemm_kernel_L4_M8_44) -.Lsgemm_kernel_L4_M8_40: +L(sgemm_kernel_L4_M8_40): INIT8x4 -.Lsgemm_kernel_L4_M8_44: +L(sgemm_kernel_L4_M8_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L4_M8_100 + ble L(sgemm_kernel_L4_M8_100) -.Lsgemm_kernel_L4_M8_46: +L(sgemm_kernel_L4_M8_46): KERNEL8x4_SUB -.Lsgemm_kernel_L4_M8_100: +L(sgemm_kernel_L4_M8_100): SAVE8x4 -.Lsgemm_kernel_L4_M8_END: +L(sgemm_kernel_L4_M8_END): subs counterI, counterI, #1 - bne .Lsgemm_kernel_L4_M8_20 + bne L(sgemm_kernel_L4_M8_20) /******************************************************************************/ -.Lsgemm_kernel_L4_M4_BEGIN: +L(sgemm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #4 - ble .Lsgemm_kernel_L4_M2_BEGIN + ble L(sgemm_kernel_L4_M2_BEGIN) -.Lsgemm_kernel_L4_M4_20: +L(sgemm_kernel_L4_M4_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L4_M4_32 + blt L(sgemm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L4_M4_22a + ble L(sgemm_kernel_L4_M4_22a) .align 5 -.Lsgemm_kernel_L4_M4_22: +L(sgemm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M4_22 + bgt L(sgemm_kernel_L4_M4_22) -.Lsgemm_kernel_L4_M4_22a: +L(sgemm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lsgemm_kernel_L4_M4_44 + b L(sgemm_kernel_L4_M4_44) -.Lsgemm_kernel_L4_M4_32: +L(sgemm_kernel_L4_M4_32): tst counterL, #1 - ble .Lsgemm_kernel_L4_M4_40 + ble L(sgemm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lsgemm_kernel_L4_M4_44 + b L(sgemm_kernel_L4_M4_44) -.Lsgemm_kernel_L4_M4_40: +L(sgemm_kernel_L4_M4_40): INIT4x4 -.Lsgemm_kernel_L4_M4_44: +L(sgemm_kernel_L4_M4_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L4_M4_100 + ble L(sgemm_kernel_L4_M4_100) -.Lsgemm_kernel_L4_M4_46: +L(sgemm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lsgemm_kernel_L4_M4_100: +L(sgemm_kernel_L4_M4_100): SAVE4x4 -.Lsgemm_kernel_L4_M4_END: +L(sgemm_kernel_L4_M4_END): /******************************************************************************/ -.Lsgemm_kernel_L4_M2_BEGIN: +L(sgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L4_M1_BEGIN + ble L(sgemm_kernel_L4_M1_BEGIN) -.Lsgemm_kernel_L4_M2_20: +L(sgemm_kernel_L4_M2_20): INIT2x4 @@ -1729,9 +1729,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L4_M2_40 + ble L(sgemm_kernel_L4_M2_40) -.Lsgemm_kernel_L4_M2_22: +L(sgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1744,35 +1744,35 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M2_22 + bgt L(sgemm_kernel_L4_M2_22) -.Lsgemm_kernel_L4_M2_40: +L(sgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M2_100 + ble L(sgemm_kernel_L4_M2_100) -.Lsgemm_kernel_L4_M2_42: +L(sgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M2_42 + bgt L(sgemm_kernel_L4_M2_42) -.Lsgemm_kernel_L4_M2_100: +L(sgemm_kernel_L4_M2_100): SAVE2x4 -.Lsgemm_kernel_L4_M2_END: +L(sgemm_kernel_L4_M2_END): /******************************************************************************/ -.Lsgemm_kernel_L4_M1_BEGIN: +L(sgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) -.Lsgemm_kernel_L4_M1_20: +L(sgemm_kernel_L4_M1_20): INIT1x4 @@ -1780,9 +1780,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L4_M1_40 + ble L(sgemm_kernel_L4_M1_40) -.Lsgemm_kernel_L4_M1_22: +L(sgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1794,39 +1794,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M1_22 + bgt L(sgemm_kernel_L4_M1_22) -.Lsgemm_kernel_L4_M1_40: +L(sgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M1_100 + ble L(sgemm_kernel_L4_M1_100) -.Lsgemm_kernel_L4_M1_42: +L(sgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M1_42 + bgt L(sgemm_kernel_L4_M1_42) -.Lsgemm_kernel_L4_M1_100: +L(sgemm_kernel_L4_M1_100): SAVE1x4 -.Lsgemm_kernel_L4_END: +L(sgemm_kernel_L4_END): add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4 /******************************************************************************/ /******************************************************************************/ -.Lsgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(sgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lsgemm_kernel_L999 + ble L(sgemm_kernel_L999) tst counterJ , #2 - ble .Lsgemm_kernel_L1_BEGIN + ble L(sgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1836,14 +1836,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lsgemm_kernel_L2_M8_BEGIN: +L(sgemm_kernel_L2_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI,#0 - ble .Lsgemm_kernel_L2_M4_BEGIN + ble L(sgemm_kernel_L2_M4_BEGIN) -.Lsgemm_kernel_L2_M8_20: +L(sgemm_kernel_L2_M8_20): INIT8x2 @@ -1851,10 +1851,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M8_40 + ble L(sgemm_kernel_L2_M8_40) .align 5 -.Lsgemm_kernel_L2_M8_22: +L(sgemm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB KERNEL8x2_SUB @@ -1866,42 +1866,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M8_22 + bgt L(sgemm_kernel_L2_M8_22) -.Lsgemm_kernel_L2_M8_40: +L(sgemm_kernel_L2_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M8_100 + ble L(sgemm_kernel_L2_M8_100) -.Lsgemm_kernel_L2_M8_42: +L(sgemm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M8_42 + bgt L(sgemm_kernel_L2_M8_42) -.Lsgemm_kernel_L2_M8_100: +L(sgemm_kernel_L2_M8_100): SAVE8x2 -.Lsgemm_kernel_L2_M8_END: +L(sgemm_kernel_L2_M8_END): subs counterI, counterI, #1 - bgt .Lsgemm_kernel_L2_M8_20 + bgt L(sgemm_kernel_L2_M8_20) /******************************************************************************/ -.Lsgemm_kernel_L2_M4_BEGIN: +L(sgemm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) tst counterI, #4 - ble .Lsgemm_kernel_L2_M2_BEGIN + ble L(sgemm_kernel_L2_M2_BEGIN) -.Lsgemm_kernel_L2_M4_20: +L(sgemm_kernel_L2_M4_20): INIT4x2 @@ -1909,10 +1909,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M4_40 + ble L(sgemm_kernel_L2_M4_40) .align 5 -.Lsgemm_kernel_L2_M4_22: +L(sgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1924,39 +1924,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M4_22 + bgt L(sgemm_kernel_L2_M4_22) -.Lsgemm_kernel_L2_M4_40: +L(sgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M4_100 + ble L(sgemm_kernel_L2_M4_100) -.Lsgemm_kernel_L2_M4_42: +L(sgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M4_42 + bgt L(sgemm_kernel_L2_M4_42) -.Lsgemm_kernel_L2_M4_100: +L(sgemm_kernel_L2_M4_100): SAVE4x2 -.Lsgemm_kernel_L2_M4_END: +L(sgemm_kernel_L2_M4_END): /******************************************************************************/ -.Lsgemm_kernel_L2_M2_BEGIN: +L(sgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L2_M1_BEGIN + ble L(sgemm_kernel_L2_M1_BEGIN) -.Lsgemm_kernel_L2_M2_20: +L(sgemm_kernel_L2_M2_20): INIT2x2 @@ -1964,9 +1964,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M2_40 + ble L(sgemm_kernel_L2_M2_40) -.Lsgemm_kernel_L2_M2_22: +L(sgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1979,35 +1979,35 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M2_22 + bgt L(sgemm_kernel_L2_M2_22) -.Lsgemm_kernel_L2_M2_40: +L(sgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M2_100 + ble L(sgemm_kernel_L2_M2_100) -.Lsgemm_kernel_L2_M2_42: +L(sgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M2_42 + bgt L(sgemm_kernel_L2_M2_42) -.Lsgemm_kernel_L2_M2_100: +L(sgemm_kernel_L2_M2_100): SAVE2x2 -.Lsgemm_kernel_L2_M2_END: +L(sgemm_kernel_L2_M2_END): /******************************************************************************/ -.Lsgemm_kernel_L2_M1_BEGIN: +L(sgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) -.Lsgemm_kernel_L2_M1_20: +L(sgemm_kernel_L2_M1_20): INIT1x2 @@ -2015,9 +2015,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lsgemm_kernel_L2_M1_40 + ble L(sgemm_kernel_L2_M1_40) -.Lsgemm_kernel_L2_M1_22: +L(sgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -2029,37 +2029,37 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M1_22 + bgt L(sgemm_kernel_L2_M1_22) -.Lsgemm_kernel_L2_M1_40: +L(sgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M1_100 + ble L(sgemm_kernel_L2_M1_100) -.Lsgemm_kernel_L2_M1_42: +L(sgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M1_42 + bgt L(sgemm_kernel_L2_M1_42) -.Lsgemm_kernel_L2_M1_100: +L(sgemm_kernel_L2_M1_100): SAVE1x2 -.Lsgemm_kernel_L2_END: +L(sgemm_kernel_L2_END): add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 /******************************************************************************/ /******************************************************************************/ -.Lsgemm_kernel_L1_BEGIN: +L(sgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lsgemm_kernel_L999 // done + ble L(sgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -2069,14 +2069,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lsgemm_kernel_L1_M8_BEGIN: +L(sgemm_kernel_L1_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 cmp counterI, #0 - ble .Lsgemm_kernel_L1_M4_BEGIN + ble L(sgemm_kernel_L1_M4_BEGIN) -.Lsgemm_kernel_L1_M8_20: +L(sgemm_kernel_L1_M8_20): INIT8x1 @@ -2084,10 +2084,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M8_40 + ble L(sgemm_kernel_L1_M8_40) .align 5 -.Lsgemm_kernel_L1_M8_22: +L(sgemm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -2099,42 +2099,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M8_22 + bgt L(sgemm_kernel_L1_M8_22) -.Lsgemm_kernel_L1_M8_40: +L(sgemm_kernel_L1_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M8_100 + ble L(sgemm_kernel_L1_M8_100) -.Lsgemm_kernel_L1_M8_42: +L(sgemm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M8_42 + bgt L(sgemm_kernel_L1_M8_42) -.Lsgemm_kernel_L1_M8_100: +L(sgemm_kernel_L1_M8_100): SAVE8x1 -.Lsgemm_kernel_L1_M8_END: +L(sgemm_kernel_L1_M8_END): subs counterI, counterI, #1 - bgt .Lsgemm_kernel_L1_M8_20 + bgt L(sgemm_kernel_L1_M8_20) /******************************************************************************/ -.Lsgemm_kernel_L1_M4_BEGIN: +L(sgemm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) tst counterI, #4 - ble .Lsgemm_kernel_L1_M2_BEGIN + ble L(sgemm_kernel_L1_M2_BEGIN) -.Lsgemm_kernel_L1_M4_20: +L(sgemm_kernel_L1_M4_20): INIT4x1 @@ -2142,10 +2142,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M4_40 + ble L(sgemm_kernel_L1_M4_40) .align 5 -.Lsgemm_kernel_L1_M4_22: +L(sgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -2157,39 +2157,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M4_22 + bgt L(sgemm_kernel_L1_M4_22) -.Lsgemm_kernel_L1_M4_40: +L(sgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M4_100 + ble L(sgemm_kernel_L1_M4_100) -.Lsgemm_kernel_L1_M4_42: +L(sgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M4_42 + bgt L(sgemm_kernel_L1_M4_42) -.Lsgemm_kernel_L1_M4_100: +L(sgemm_kernel_L1_M4_100): SAVE4x1 -.Lsgemm_kernel_L1_M4_END: +L(sgemm_kernel_L1_M4_END): /******************************************************************************/ -.Lsgemm_kernel_L1_M2_BEGIN: +L(sgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L1_M1_BEGIN + ble L(sgemm_kernel_L1_M1_BEGIN) -.Lsgemm_kernel_L1_M2_20: +L(sgemm_kernel_L1_M2_20): INIT2x1 @@ -2197,9 +2197,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M2_40 + ble L(sgemm_kernel_L1_M2_40) -.Lsgemm_kernel_L1_M2_22: +L(sgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -2212,35 +2212,35 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M2_22 + bgt L(sgemm_kernel_L1_M2_22) -.Lsgemm_kernel_L1_M2_40: +L(sgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M2_100 + ble L(sgemm_kernel_L1_M2_100) -.Lsgemm_kernel_L1_M2_42: +L(sgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M2_42 + bgt L(sgemm_kernel_L1_M2_42) -.Lsgemm_kernel_L1_M2_100: +L(sgemm_kernel_L1_M2_100): SAVE2x1 -.Lsgemm_kernel_L1_M2_END: +L(sgemm_kernel_L1_M2_END): /******************************************************************************/ -.Lsgemm_kernel_L1_M1_BEGIN: +L(sgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) -.Lsgemm_kernel_L1_M1_20: +L(sgemm_kernel_L1_M1_20): INIT1x1 @@ -2248,9 +2248,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M1_40 + ble L(sgemm_kernel_L1_M1_40) -.Lsgemm_kernel_L1_M1_22: +L(sgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -2262,30 +2262,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M1_22 + bgt L(sgemm_kernel_L1_M1_22) -.Lsgemm_kernel_L1_M1_40: +L(sgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M1_100 + ble L(sgemm_kernel_L1_M1_100) -.Lsgemm_kernel_L1_M1_42: +L(sgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M1_42 + bgt L(sgemm_kernel_L1_M1_42) -.Lsgemm_kernel_L1_M1_100: +L(sgemm_kernel_L1_M1_100): SAVE1x1 -.Lsgemm_kernel_L1_END: +L(sgemm_kernel_L1_END): /******************************************************************************/ -.Lsgemm_kernel_L999: +L(sgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/sgemm_kernel_8x8_cortexa53.S b/kernel/arm64/sgemm_kernel_8x8_cortexa53.S index 628a928ca7..68d933be64 100644 --- a/kernel/arm64/sgemm_kernel_8x8_cortexa53.S +++ b/kernel/arm64/sgemm_kernel_8x8_cortexa53.S @@ -1230,7 +1230,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE -.Lsgemm_kernel_begin: +L(sgemm_kernel_begin): .align 5 add sp, sp, #-(11 * 16) @@ -1258,12 +1258,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Lsgemm_kernel_L4_BEGIN + ble L(sgemm_kernel_L4_BEGIN) /******************************************************************************/ /******************************************************************************/ -.Lsgemm_kernel_L8_BEGIN: +L(sgemm_kernel_L8_BEGIN): mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #3 @@ -1271,20 +1271,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lsgemm_kernel_L8_M8_BEGIN: +L(sgemm_kernel_L8_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lsgemm_kernel_L8_M4_BEGIN + ble L(sgemm_kernel_L8_M4_BEGIN) -.Lsgemm_kernel_L8_M8_20: +L(sgemm_kernel_L8_M8_20): mov pB, origPB asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 16 to do? - blt .Lsgemm_kernel_L8_M8_32 + blt L(sgemm_kernel_L8_M8_32) KERNEL8x8_I // do one in the K KERNEL8x8_M2 // do another in the K @@ -1296,10 +1296,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x8_M2 subs counterL, counterL, #2 - ble .Lsgemm_kernel_L8_M8_22a + ble L(sgemm_kernel_L8_M8_22a) .align 5 -.Lsgemm_kernel_L8_M8_22: +L(sgemm_kernel_L8_M8_22): KERNEL8x8_M1 KERNEL8x8_M2 @@ -1311,9 +1311,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x8_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M8_22 + bgt L(sgemm_kernel_L8_M8_22) -.Lsgemm_kernel_L8_M8_22a: +L(sgemm_kernel_L8_M8_22a): KERNEL8x8_M1 KERNEL8x8_M2 @@ -1324,12 +1324,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x8_M1 KERNEL8x8_E - b .Lsgemm_kernel_L8_M8_44 + b L(sgemm_kernel_L8_M8_44) -.Lsgemm_kernel_L8_M8_32: +L(sgemm_kernel_L8_M8_32): tst counterL, #1 - ble .Lsgemm_kernel_L8_M8_40 + ble L(sgemm_kernel_L8_M8_40) KERNEL8x8_I KERNEL8x8_M2 @@ -1340,114 +1340,114 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x8_M1 KERNEL8x8_E - b .Lsgemm_kernel_L8_M8_44 + b L(sgemm_kernel_L8_M8_44) -.Lsgemm_kernel_L8_M8_40: +L(sgemm_kernel_L8_M8_40): INIT8x8 -.Lsgemm_kernel_L8_M8_44: +L(sgemm_kernel_L8_M8_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L8_M8_100 + ble L(sgemm_kernel_L8_M8_100) -.Lsgemm_kernel_L8_M8_46: +L(sgemm_kernel_L8_M8_46): KERNEL8x8_SUB subs counterL, counterL, 1 - bgt .Lsgemm_kernel_L8_M8_46 + bgt L(sgemm_kernel_L8_M8_46) -.Lsgemm_kernel_L8_M8_100: +L(sgemm_kernel_L8_M8_100): SAVE8x8 -.Lsgemm_kernel_L8_M8_END: +L(sgemm_kernel_L8_M8_END): subs counterI, counterI, #1 - bne .Lsgemm_kernel_L8_M8_20 + bne L(sgemm_kernel_L8_M8_20) /******************************************************************************/ -.Lsgemm_kernel_L8_M4_BEGIN: +L(sgemm_kernel_L8_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L8_END + ble L(sgemm_kernel_L8_END) tst counterI, #4 - ble .Lsgemm_kernel_L8_M2_BEGIN + ble L(sgemm_kernel_L8_M2_BEGIN) -.Lsgemm_kernel_L8_M4_20: +L(sgemm_kernel_L8_M4_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L8_M4_32 + blt L(sgemm_kernel_L8_M4_32) KERNEL4x8_I // do one in the K KERNEL4x8_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L8_M4_22a + ble L(sgemm_kernel_L8_M4_22a) .align 5 -.Lsgemm_kernel_L8_M4_22: +L(sgemm_kernel_L8_M4_22): KERNEL4x8_M1 KERNEL4x8_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M4_22 + bgt L(sgemm_kernel_L8_M4_22) -.Lsgemm_kernel_L8_M4_22a: +L(sgemm_kernel_L8_M4_22a): KERNEL4x8_M1 KERNEL4x8_E - b .Lsgemm_kernel_L8_M4_44 + b L(sgemm_kernel_L8_M4_44) -.Lsgemm_kernel_L8_M4_32: +L(sgemm_kernel_L8_M4_32): tst counterL, #1 - ble .Lsgemm_kernel_L8_M4_40 + ble L(sgemm_kernel_L8_M4_40) KERNEL4x8_I KERNEL4x8_E - b .Lsgemm_kernel_L8_M4_44 + b L(sgemm_kernel_L8_M4_44) -.Lsgemm_kernel_L8_M4_40: +L(sgemm_kernel_L8_M4_40): INIT4x8 -.Lsgemm_kernel_L8_M4_44: +L(sgemm_kernel_L8_M4_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L8_M4_100 + ble L(sgemm_kernel_L8_M4_100) -.Lsgemm_kernel_L8_M4_46: +L(sgemm_kernel_L8_M4_46): KERNEL4x8_SUB -.Lsgemm_kernel_L8_M4_100: +L(sgemm_kernel_L8_M4_100): SAVE4x8 -.Lsgemm_kernel_L8_M4_END: +L(sgemm_kernel_L8_M4_END): /******************************************************************************/ -.Lsgemm_kernel_L8_M2_BEGIN: +L(sgemm_kernel_L8_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L8_END + ble L(sgemm_kernel_L8_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L8_M1_BEGIN + ble L(sgemm_kernel_L8_M1_BEGIN) -.Lsgemm_kernel_L8_M2_20: +L(sgemm_kernel_L8_M2_20): INIT2x8 @@ -1455,9 +1455,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L8_M2_40 + ble L(sgemm_kernel_L8_M2_40) -.Lsgemm_kernel_L8_M2_22: +L(sgemm_kernel_L8_M2_22): KERNEL2x8_SUB KERNEL2x8_SUB @@ -1470,35 +1470,35 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M2_22 + bgt L(sgemm_kernel_L8_M2_22) -.Lsgemm_kernel_L8_M2_40: +L(sgemm_kernel_L8_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L8_M2_100 + ble L(sgemm_kernel_L8_M2_100) -.Lsgemm_kernel_L8_M2_42: +L(sgemm_kernel_L8_M2_42): KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M2_42 + bgt L(sgemm_kernel_L8_M2_42) -.Lsgemm_kernel_L8_M2_100: +L(sgemm_kernel_L8_M2_100): SAVE2x8 -.Lsgemm_kernel_L8_M2_END: +L(sgemm_kernel_L8_M2_END): /******************************************************************************/ -.Lsgemm_kernel_L8_M1_BEGIN: +L(sgemm_kernel_L8_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L8_END + ble L(sgemm_kernel_L8_END) -.Lsgemm_kernel_L8_M1_20: +L(sgemm_kernel_L8_M1_20): INIT1x8 @@ -1506,9 +1506,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L8_M1_40 + ble L(sgemm_kernel_L8_M1_40) -.Lsgemm_kernel_L8_M1_22: +L(sgemm_kernel_L8_M1_22): KERNEL1x8_SUB KERNEL1x8_SUB KERNEL1x8_SUB @@ -1520,43 +1520,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M1_22 + bgt L(sgemm_kernel_L8_M1_22) -.Lsgemm_kernel_L8_M1_40: +L(sgemm_kernel_L8_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L8_M1_100 + ble L(sgemm_kernel_L8_M1_100) -.Lsgemm_kernel_L8_M1_42: +L(sgemm_kernel_L8_M1_42): KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M1_42 + bgt L(sgemm_kernel_L8_M1_42) -.Lsgemm_kernel_L8_M1_100: +L(sgemm_kernel_L8_M1_100): SAVE1x8 -.Lsgemm_kernel_L8_END: +L(sgemm_kernel_L8_END): lsl temp, origK, #5 // B = B + K * 4 * 8 add origPB, origPB, temp subs counterJ, counterJ , #1 // j-- - bgt .Lsgemm_kernel_L8_BEGIN + bgt L(sgemm_kernel_L8_BEGIN) /******************************************************************************/ /******************************************************************************/ -.Lsgemm_kernel_L4_BEGIN: +L(sgemm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #7 - ble .Lsgemm_kernel_L999 + ble L(sgemm_kernel_L999) tst counterJ , #4 - ble .Lsgemm_kernel_L2_BEGIN + ble L(sgemm_kernel_L2_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1566,156 +1566,156 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lsgemm_kernel_L4_M8_BEGIN: +L(sgemm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lsgemm_kernel_L4_M4_BEGIN + ble L(sgemm_kernel_L4_M4_BEGIN) -.Lsgemm_kernel_L4_M8_20: +L(sgemm_kernel_L4_M8_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L4_M8_32 + blt L(sgemm_kernel_L4_M8_32) KERNEL8x4_I // do one in the K KERNEL8x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L4_M8_22a + ble L(sgemm_kernel_L4_M8_22a) .align 5 -.Lsgemm_kernel_L4_M8_22: +L(sgemm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M8_22 + bgt L(sgemm_kernel_L4_M8_22) -.Lsgemm_kernel_L4_M8_22a: +L(sgemm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_E - b .Lsgemm_kernel_L4_M8_44 + b L(sgemm_kernel_L4_M8_44) -.Lsgemm_kernel_L4_M8_32: +L(sgemm_kernel_L4_M8_32): tst counterL, #1 - ble .Lsgemm_kernel_L4_M8_40 + ble L(sgemm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_E - b .Lsgemm_kernel_L4_M8_44 + b L(sgemm_kernel_L4_M8_44) -.Lsgemm_kernel_L4_M8_40: +L(sgemm_kernel_L4_M8_40): INIT8x4 -.Lsgemm_kernel_L4_M8_44: +L(sgemm_kernel_L4_M8_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L4_M8_100 + ble L(sgemm_kernel_L4_M8_100) -.Lsgemm_kernel_L4_M8_46: +L(sgemm_kernel_L4_M8_46): KERNEL8x4_SUB -.Lsgemm_kernel_L4_M8_100: +L(sgemm_kernel_L4_M8_100): SAVE8x4 -.Lsgemm_kernel_L4_M8_END: +L(sgemm_kernel_L4_M8_END): subs counterI, counterI, #1 - bne .Lsgemm_kernel_L4_M8_20 + bne L(sgemm_kernel_L4_M8_20) /******************************************************************************/ -.Lsgemm_kernel_L4_M4_BEGIN: +L(sgemm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #4 - ble .Lsgemm_kernel_L4_M2_BEGIN + ble L(sgemm_kernel_L4_M2_BEGIN) -.Lsgemm_kernel_L4_M4_20: +L(sgemm_kernel_L4_M4_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L4_M4_32 + blt L(sgemm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L4_M4_22a + ble L(sgemm_kernel_L4_M4_22a) .align 5 -.Lsgemm_kernel_L4_M4_22: +L(sgemm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M4_22 + bgt L(sgemm_kernel_L4_M4_22) -.Lsgemm_kernel_L4_M4_22a: +L(sgemm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lsgemm_kernel_L4_M4_44 + b L(sgemm_kernel_L4_M4_44) -.Lsgemm_kernel_L4_M4_32: +L(sgemm_kernel_L4_M4_32): tst counterL, #1 - ble .Lsgemm_kernel_L4_M4_40 + ble L(sgemm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lsgemm_kernel_L4_M4_44 + b L(sgemm_kernel_L4_M4_44) -.Lsgemm_kernel_L4_M4_40: +L(sgemm_kernel_L4_M4_40): INIT4x4 -.Lsgemm_kernel_L4_M4_44: +L(sgemm_kernel_L4_M4_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L4_M4_100 + ble L(sgemm_kernel_L4_M4_100) -.Lsgemm_kernel_L4_M4_46: +L(sgemm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lsgemm_kernel_L4_M4_100: +L(sgemm_kernel_L4_M4_100): SAVE4x4 -.Lsgemm_kernel_L4_M4_END: +L(sgemm_kernel_L4_M4_END): /******************************************************************************/ -.Lsgemm_kernel_L4_M2_BEGIN: +L(sgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L4_M1_BEGIN + ble L(sgemm_kernel_L4_M1_BEGIN) -.Lsgemm_kernel_L4_M2_20: +L(sgemm_kernel_L4_M2_20): INIT2x4 @@ -1723,9 +1723,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L4_M2_40 + ble L(sgemm_kernel_L4_M2_40) -.Lsgemm_kernel_L4_M2_22: +L(sgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1738,35 +1738,35 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M2_22 + bgt L(sgemm_kernel_L4_M2_22) -.Lsgemm_kernel_L4_M2_40: +L(sgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M2_100 + ble L(sgemm_kernel_L4_M2_100) -.Lsgemm_kernel_L4_M2_42: +L(sgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M2_42 + bgt L(sgemm_kernel_L4_M2_42) -.Lsgemm_kernel_L4_M2_100: +L(sgemm_kernel_L4_M2_100): SAVE2x4 -.Lsgemm_kernel_L4_M2_END: +L(sgemm_kernel_L4_M2_END): /******************************************************************************/ -.Lsgemm_kernel_L4_M1_BEGIN: +L(sgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) -.Lsgemm_kernel_L4_M1_20: +L(sgemm_kernel_L4_M1_20): INIT1x4 @@ -1774,9 +1774,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L4_M1_40 + ble L(sgemm_kernel_L4_M1_40) -.Lsgemm_kernel_L4_M1_22: +L(sgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1788,39 +1788,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M1_22 + bgt L(sgemm_kernel_L4_M1_22) -.Lsgemm_kernel_L4_M1_40: +L(sgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M1_100 + ble L(sgemm_kernel_L4_M1_100) -.Lsgemm_kernel_L4_M1_42: +L(sgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M1_42 + bgt L(sgemm_kernel_L4_M1_42) -.Lsgemm_kernel_L4_M1_100: +L(sgemm_kernel_L4_M1_100): SAVE1x4 -.Lsgemm_kernel_L4_END: +L(sgemm_kernel_L4_END): add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4 /******************************************************************************/ /******************************************************************************/ -.Lsgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(sgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lsgemm_kernel_L999 + ble L(sgemm_kernel_L999) tst counterJ , #2 - ble .Lsgemm_kernel_L1_BEGIN + ble L(sgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1830,14 +1830,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lsgemm_kernel_L2_M8_BEGIN: +L(sgemm_kernel_L2_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI,#0 - ble .Lsgemm_kernel_L2_M4_BEGIN + ble L(sgemm_kernel_L2_M4_BEGIN) -.Lsgemm_kernel_L2_M8_20: +L(sgemm_kernel_L2_M8_20): INIT8x2 @@ -1845,10 +1845,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M8_40 + ble L(sgemm_kernel_L2_M8_40) .align 5 -.Lsgemm_kernel_L2_M8_22: +L(sgemm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB KERNEL8x2_SUB @@ -1860,42 +1860,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M8_22 + bgt L(sgemm_kernel_L2_M8_22) -.Lsgemm_kernel_L2_M8_40: +L(sgemm_kernel_L2_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M8_100 + ble L(sgemm_kernel_L2_M8_100) -.Lsgemm_kernel_L2_M8_42: +L(sgemm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M8_42 + bgt L(sgemm_kernel_L2_M8_42) -.Lsgemm_kernel_L2_M8_100: +L(sgemm_kernel_L2_M8_100): SAVE8x2 -.Lsgemm_kernel_L2_M8_END: +L(sgemm_kernel_L2_M8_END): subs counterI, counterI, #1 - bgt .Lsgemm_kernel_L2_M8_20 + bgt L(sgemm_kernel_L2_M8_20) /******************************************************************************/ -.Lsgemm_kernel_L2_M4_BEGIN: +L(sgemm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) tst counterI, #4 - ble .Lsgemm_kernel_L2_M2_BEGIN + ble L(sgemm_kernel_L2_M2_BEGIN) -.Lsgemm_kernel_L2_M4_20: +L(sgemm_kernel_L2_M4_20): INIT4x2 @@ -1903,10 +1903,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M4_40 + ble L(sgemm_kernel_L2_M4_40) .align 5 -.Lsgemm_kernel_L2_M4_22: +L(sgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1918,39 +1918,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M4_22 + bgt L(sgemm_kernel_L2_M4_22) -.Lsgemm_kernel_L2_M4_40: +L(sgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M4_100 + ble L(sgemm_kernel_L2_M4_100) -.Lsgemm_kernel_L2_M4_42: +L(sgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M4_42 + bgt L(sgemm_kernel_L2_M4_42) -.Lsgemm_kernel_L2_M4_100: +L(sgemm_kernel_L2_M4_100): SAVE4x2 -.Lsgemm_kernel_L2_M4_END: +L(sgemm_kernel_L2_M4_END): /******************************************************************************/ -.Lsgemm_kernel_L2_M2_BEGIN: +L(sgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L2_M1_BEGIN + ble L(sgemm_kernel_L2_M1_BEGIN) -.Lsgemm_kernel_L2_M2_20: +L(sgemm_kernel_L2_M2_20): INIT2x2 @@ -1958,9 +1958,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M2_40 + ble L(sgemm_kernel_L2_M2_40) -.Lsgemm_kernel_L2_M2_22: +L(sgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1973,35 +1973,35 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M2_22 + bgt L(sgemm_kernel_L2_M2_22) -.Lsgemm_kernel_L2_M2_40: +L(sgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M2_100 + ble L(sgemm_kernel_L2_M2_100) -.Lsgemm_kernel_L2_M2_42: +L(sgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M2_42 + bgt L(sgemm_kernel_L2_M2_42) -.Lsgemm_kernel_L2_M2_100: +L(sgemm_kernel_L2_M2_100): SAVE2x2 -.Lsgemm_kernel_L2_M2_END: +L(sgemm_kernel_L2_M2_END): /******************************************************************************/ -.Lsgemm_kernel_L2_M1_BEGIN: +L(sgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) -.Lsgemm_kernel_L2_M1_20: +L(sgemm_kernel_L2_M1_20): INIT1x2 @@ -2009,9 +2009,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lsgemm_kernel_L2_M1_40 + ble L(sgemm_kernel_L2_M1_40) -.Lsgemm_kernel_L2_M1_22: +L(sgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -2023,37 +2023,37 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M1_22 + bgt L(sgemm_kernel_L2_M1_22) -.Lsgemm_kernel_L2_M1_40: +L(sgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M1_100 + ble L(sgemm_kernel_L2_M1_100) -.Lsgemm_kernel_L2_M1_42: +L(sgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M1_42 + bgt L(sgemm_kernel_L2_M1_42) -.Lsgemm_kernel_L2_M1_100: +L(sgemm_kernel_L2_M1_100): SAVE1x2 -.Lsgemm_kernel_L2_END: +L(sgemm_kernel_L2_END): add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 /******************************************************************************/ /******************************************************************************/ -.Lsgemm_kernel_L1_BEGIN: +L(sgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lsgemm_kernel_L999 // done + ble L(sgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -2063,14 +2063,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lsgemm_kernel_L1_M8_BEGIN: +L(sgemm_kernel_L1_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 cmp counterI, #0 - ble .Lsgemm_kernel_L1_M4_BEGIN + ble L(sgemm_kernel_L1_M4_BEGIN) -.Lsgemm_kernel_L1_M8_20: +L(sgemm_kernel_L1_M8_20): INIT8x1 @@ -2078,10 +2078,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M8_40 + ble L(sgemm_kernel_L1_M8_40) .align 5 -.Lsgemm_kernel_L1_M8_22: +L(sgemm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -2093,42 +2093,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M8_22 + bgt L(sgemm_kernel_L1_M8_22) -.Lsgemm_kernel_L1_M8_40: +L(sgemm_kernel_L1_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M8_100 + ble L(sgemm_kernel_L1_M8_100) -.Lsgemm_kernel_L1_M8_42: +L(sgemm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M8_42 + bgt L(sgemm_kernel_L1_M8_42) -.Lsgemm_kernel_L1_M8_100: +L(sgemm_kernel_L1_M8_100): SAVE8x1 -.Lsgemm_kernel_L1_M8_END: +L(sgemm_kernel_L1_M8_END): subs counterI, counterI, #1 - bgt .Lsgemm_kernel_L1_M8_20 + bgt L(sgemm_kernel_L1_M8_20) /******************************************************************************/ -.Lsgemm_kernel_L1_M4_BEGIN: +L(sgemm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) tst counterI, #4 - ble .Lsgemm_kernel_L1_M2_BEGIN + ble L(sgemm_kernel_L1_M2_BEGIN) -.Lsgemm_kernel_L1_M4_20: +L(sgemm_kernel_L1_M4_20): INIT4x1 @@ -2136,10 +2136,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M4_40 + ble L(sgemm_kernel_L1_M4_40) .align 5 -.Lsgemm_kernel_L1_M4_22: +L(sgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -2151,39 +2151,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M4_22 + bgt L(sgemm_kernel_L1_M4_22) -.Lsgemm_kernel_L1_M4_40: +L(sgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M4_100 + ble L(sgemm_kernel_L1_M4_100) -.Lsgemm_kernel_L1_M4_42: +L(sgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M4_42 + bgt L(sgemm_kernel_L1_M4_42) -.Lsgemm_kernel_L1_M4_100: +L(sgemm_kernel_L1_M4_100): SAVE4x1 -.Lsgemm_kernel_L1_M4_END: +L(sgemm_kernel_L1_M4_END): /******************************************************************************/ -.Lsgemm_kernel_L1_M2_BEGIN: +L(sgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L1_M1_BEGIN + ble L(sgemm_kernel_L1_M1_BEGIN) -.Lsgemm_kernel_L1_M2_20: +L(sgemm_kernel_L1_M2_20): INIT2x1 @@ -2191,9 +2191,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M2_40 + ble L(sgemm_kernel_L1_M2_40) -.Lsgemm_kernel_L1_M2_22: +L(sgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -2206,35 +2206,35 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M2_22 + bgt L(sgemm_kernel_L1_M2_22) -.Lsgemm_kernel_L1_M2_40: +L(sgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M2_100 + ble L(sgemm_kernel_L1_M2_100) -.Lsgemm_kernel_L1_M2_42: +L(sgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M2_42 + bgt L(sgemm_kernel_L1_M2_42) -.Lsgemm_kernel_L1_M2_100: +L(sgemm_kernel_L1_M2_100): SAVE2x1 -.Lsgemm_kernel_L1_M2_END: +L(sgemm_kernel_L1_M2_END): /******************************************************************************/ -.Lsgemm_kernel_L1_M1_BEGIN: +L(sgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) -.Lsgemm_kernel_L1_M1_20: +L(sgemm_kernel_L1_M1_20): INIT1x1 @@ -2242,9 +2242,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M1_40 + ble L(sgemm_kernel_L1_M1_40) -.Lsgemm_kernel_L1_M1_22: +L(sgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -2256,30 +2256,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M1_22 + bgt L(sgemm_kernel_L1_M1_22) -.Lsgemm_kernel_L1_M1_40: +L(sgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M1_100 + ble L(sgemm_kernel_L1_M1_100) -.Lsgemm_kernel_L1_M1_42: +L(sgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M1_42 + bgt L(sgemm_kernel_L1_M1_42) -.Lsgemm_kernel_L1_M1_100: +L(sgemm_kernel_L1_M1_100): SAVE1x1 -.Lsgemm_kernel_L1_END: +L(sgemm_kernel_L1_END): /******************************************************************************/ -.Lsgemm_kernel_L999: +L(sgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/sgemm_kernel_inter_sve_v2x8.S b/kernel/arm64/sgemm_kernel_inter_sve_v2x8.S index f8c4aa61c1..247581c218 100644 --- a/kernel/arm64/sgemm_kernel_inter_sve_v2x8.S +++ b/kernel/arm64/sgemm_kernel_inter_sve_v2x8.S @@ -855,38 +855,38 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Lsgemm_kernel_L4_BEGIN + ble L(sgemm_kernel_L4_BEGIN) /******************************************************************************/ /* Repeat this as long as there are 8 left in N */ .align 5 -.Lsgemm_kernel_L8_BEGIN: +L(sgemm_kernel_L8_BEGIN): mov pCRow0, pC add pC, pC, LDC, lsl #3 // add 8 x LDC mov pA1, origPA // pA1 = start of A array -.Lsgemm_kernel_L8_Mv2_BEGIN: +L(sgemm_kernel_L8_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 // Check if M < 2*SVE_LEN - blt .Lsgemm_kernel_L8_Mv1_BEGIN + blt L(sgemm_kernel_L8_Mv1_BEGIN) mov counterI, origM /* Until we have at least 2*SVE_LEN iters left in M, we do them with V2*8 kernel */ .align 5 -.Lsgemm_kernel_L8_Mv2_20: +L(sgemm_kernel_L8_Mv2_20): mov pB, origPB INITv2x8 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L8_Mv2_32 + blt L(sgemm_kernel_L8_Mv2_32) KERNELv2x8_I KERNELv2x8_M2 @@ -898,10 +898,10 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lsgemm_kernel_L8_Mv2_22a + ble L(sgemm_kernel_L8_Mv2_22a) .align 5 -.Lsgemm_kernel_L8_Mv2_22: +L(sgemm_kernel_L8_Mv2_22): KERNELv2x8_M1 KERNELv2x8_M2 @@ -913,10 +913,10 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_Mv2_22 + bgt L(sgemm_kernel_L8_Mv2_22) .align 5 -.Lsgemm_kernel_L8_Mv2_22a: +L(sgemm_kernel_L8_Mv2_22a): KERNELv2x8_M1 KERNELv2x8_M2 @@ -927,13 +927,13 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M1 KERNELv2x8_E - b .Lsgemm_kernel_L8_Mv2_44 + b L(sgemm_kernel_L8_Mv2_44) .align 5 -.Lsgemm_kernel_L8_Mv2_32: +L(sgemm_kernel_L8_Mv2_32): tst counterL, #1 - ble .Lsgemm_kernel_L8_Mv2_40 + ble L(sgemm_kernel_L8_Mv2_40) KERNELv2x8_I KERNELv2x8_M2 @@ -945,54 +945,54 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_E - b .Lsgemm_kernel_L8_Mv2_44 + b L(sgemm_kernel_L8_Mv2_44) -.Lsgemm_kernel_L8_Mv2_40: +L(sgemm_kernel_L8_Mv2_40): INITv2x8 -.Lsgemm_kernel_L8_Mv2_44: +L(sgemm_kernel_L8_Mv2_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L8_Mv2_100 + ble L(sgemm_kernel_L8_Mv2_100) .align 5 -.Lsgemm_kernel_L8_Mv2_46: +L(sgemm_kernel_L8_Mv2_46): KERNELv2x8_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L8_Mv2_46 + bne L(sgemm_kernel_L8_Mv2_46) -.Lsgemm_kernel_L8_Mv2_100: +L(sgemm_kernel_L8_Mv2_100): SAVEv2x8 -.Lsgemm_kernel_L8_Mv2_END: +L(sgemm_kernel_L8_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Lsgemm_kernel_L8_Mv2_20 + bge L(sgemm_kernel_L8_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Lsgemm_kernel_L8_END + beq L(sgemm_kernel_L8_END) ////////////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x8 kernel. -.Lsgemm_kernel_L8_Mv1_BEGIN: +L(sgemm_kernel_L8_Mv1_BEGIN): whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension .align 5 -.Lsgemm_kernel_L8_Mv1_20: +L(sgemm_kernel_L8_Mv1_20): mov pB, origPB INITv1x8 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L8_Mv1_32 + blt L(sgemm_kernel_L8_Mv1_32) KERNELv1x8_I KERNELv1x8_M2 @@ -1004,10 +1004,10 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lsgemm_kernel_L8_Mv1_22a + ble L(sgemm_kernel_L8_Mv1_22a) .align 5 -.Lsgemm_kernel_L8_Mv1_22: +L(sgemm_kernel_L8_Mv1_22): KERNELv1x8_M1 KERNELv1x8_M2 @@ -1019,10 +1019,10 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_Mv1_22 + bgt L(sgemm_kernel_L8_Mv1_22) .align 5 -.Lsgemm_kernel_L8_Mv1_22a: +L(sgemm_kernel_L8_Mv1_22a): KERNELv1x8_M1 KERNELv1x8_M2 @@ -1033,13 +1033,13 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M1 KERNELv1x8_E - b .Lsgemm_kernel_L8_Mv1_44 + b L(sgemm_kernel_L8_Mv1_44) .align 5 -.Lsgemm_kernel_L8_Mv1_32: +L(sgemm_kernel_L8_Mv1_32): tst counterL, #1 - ble .Lsgemm_kernel_L8_Mv1_40 + ble L(sgemm_kernel_L8_Mv1_40) KERNELv1x8_I KERNELv1x8_M2 @@ -1051,53 +1051,53 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_E - b .Lsgemm_kernel_L8_Mv1_44 + b L(sgemm_kernel_L8_Mv1_44) -.Lsgemm_kernel_L8_Mv1_40: +L(sgemm_kernel_L8_Mv1_40): INITv1x8 -.Lsgemm_kernel_L8_Mv1_44: +L(sgemm_kernel_L8_Mv1_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L8_Mv1_100 + ble L(sgemm_kernel_L8_Mv1_100) .align 5 -.Lsgemm_kernel_L8_Mv1_46: +L(sgemm_kernel_L8_Mv1_46): KERNELv1x8_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L8_Mv1_46 + bne L(sgemm_kernel_L8_Mv1_46) -.Lsgemm_kernel_L8_Mv1_100: +L(sgemm_kernel_L8_Mv1_100): SAVEv1x8 -.Lsgemm_kernel_L8_Mv1_END: +L(sgemm_kernel_L8_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension - b.any .Lsgemm_kernel_L8_Mv1_20 + b.any L(sgemm_kernel_L8_Mv1_20) -.Lsgemm_kernel_L8_END: +L(sgemm_kernel_L8_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 8 * 4 subs counterJ, counterJ , #1 // j-- - bgt .Lsgemm_kernel_L8_BEGIN + bgt L(sgemm_kernel_L8_BEGIN) /******************************************************************************/ /* Repeat the same thing if 4 left in N */ .align 5 -.Lsgemm_kernel_L4_BEGIN: +L(sgemm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #4 - ble .Lsgemm_kernel_L2_BEGIN + ble L(sgemm_kernel_L2_BEGIN) mov pCRow0, pC @@ -1106,26 +1106,26 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Lsgemm_kernel_L4_Mv2_BEGIN: +L(sgemm_kernel_L4_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Lsgemm_kernel_L4_Mv1_BEGIN + blt L(sgemm_kernel_L4_Mv1_BEGIN) mov counterI, origM .align 5 -.Lsgemm_kernel_L4_Mv2_20: +L(sgemm_kernel_L4_Mv2_20): mov pB, origPB INITv2x4 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Lsgemm_kernel_L4_Mv2_44 + ble L(sgemm_kernel_L4_Mv2_44) .align 5 -.Lsgemm_kernel_L4_Mv2_22: +L(sgemm_kernel_L4_Mv2_22): KERNELv2x4_SUB KERNELv2x4_SUB @@ -1137,53 +1137,53 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_Mv2_22 + bgt L(sgemm_kernel_L4_Mv2_22) -.Lsgemm_kernel_L4_Mv2_44: +L(sgemm_kernel_L4_Mv2_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L4_Mv2_100 + ble L(sgemm_kernel_L4_Mv2_100) .align 5 -.Lsgemm_kernel_L4_Mv2_46: +L(sgemm_kernel_L4_Mv2_46): KERNELv2x4_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L4_Mv2_46 + bne L(sgemm_kernel_L4_Mv2_46) -.Lsgemm_kernel_L4_Mv2_100: +L(sgemm_kernel_L4_Mv2_100): SAVEv2x4 -.Lsgemm_kernel_L4_Mv2_END: +L(sgemm_kernel_L4_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Lsgemm_kernel_L4_Mv2_20 + bge L(sgemm_kernel_L4_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Lsgemm_kernel_L4_END + beq L(sgemm_kernel_L4_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x4 kernel. -.Lsgemm_kernel_L4_Mv1_BEGIN: +L(sgemm_kernel_L4_Mv1_BEGIN): whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension .align 5 -.Lsgemm_kernel_L4_Mv1_20: +L(sgemm_kernel_L4_Mv1_20): mov pB, origPB INITv1x4 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Lsgemm_kernel_L4_Mv1_44 + ble L(sgemm_kernel_L4_Mv1_44) .align 5 -.Lsgemm_kernel_L4_Mv1_22: +L(sgemm_kernel_L4_Mv1_22): KERNELv1x4_SUB KERNELv1x4_SUB @@ -1195,34 +1195,34 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_Mv1_22 + bgt L(sgemm_kernel_L4_Mv1_22) -.Lsgemm_kernel_L4_Mv1_44: +L(sgemm_kernel_L4_Mv1_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L4_Mv1_100 + ble L(sgemm_kernel_L4_Mv1_100) .align 5 -.Lsgemm_kernel_L4_Mv1_46: +L(sgemm_kernel_L4_Mv1_46): KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L4_Mv1_46 + bne L(sgemm_kernel_L4_Mv1_46) -.Lsgemm_kernel_L4_Mv1_100: +L(sgemm_kernel_L4_Mv1_100): SAVEv1x4 -.Lsgemm_kernel_L4_Mv1_END: +L(sgemm_kernel_L4_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lsgemm_kernel_L4_Mv1_20 + b.any L(sgemm_kernel_L4_Mv1_20) -.Lsgemm_kernel_L4_END: +L(sgemm_kernel_L4_END): lsl temp, origK, #4 add origPB, origPB, temp // B = B + K * 4 * 4 @@ -1230,11 +1230,11 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ /* Repeat the same thing if 2 left in N */ .align 5 -.Lsgemm_kernel_L2_BEGIN: +L(sgemm_kernel_L2_BEGIN): mov counterJ , origN tst counterJ , #2 - ble .Lsgemm_kernel_L1_BEGIN + ble L(sgemm_kernel_L1_BEGIN) mov pCRow0, pC @@ -1242,26 +1242,26 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Lsgemm_kernel_L2_Mv2_BEGIN: +L(sgemm_kernel_L2_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Lsgemm_kernel_L2_Mv1_BEGIN + blt L(sgemm_kernel_L2_Mv1_BEGIN) mov counterI, origM .align 5 -.Lsgemm_kernel_L2_Mv2_20: +L(sgemm_kernel_L2_Mv2_20): mov pB, origPB INITv2x2 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Lsgemm_kernel_L2_Mv2_44 + ble L(sgemm_kernel_L2_Mv2_44) .align 5 -.Lsgemm_kernel_L2_Mv2_22: +L(sgemm_kernel_L2_Mv2_22): KERNELv2x2_SUB KERNELv2x2_SUB @@ -1273,54 +1273,54 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_Mv2_22 + bgt L(sgemm_kernel_L2_Mv2_22) -.Lsgemm_kernel_L2_Mv2_44: +L(sgemm_kernel_L2_Mv2_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L2_Mv2_100 + ble L(sgemm_kernel_L2_Mv2_100) .align 5 -.Lsgemm_kernel_L2_Mv2_46: +L(sgemm_kernel_L2_Mv2_46): KERNELv2x2_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L2_Mv2_46 + bne L(sgemm_kernel_L2_Mv2_46) -.Lsgemm_kernel_L2_Mv2_100: +L(sgemm_kernel_L2_Mv2_100): SAVEv2x2 -.Lsgemm_kernel_L2_Mv2_END: +L(sgemm_kernel_L2_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Lsgemm_kernel_L2_Mv2_20 + bge L(sgemm_kernel_L2_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Lsgemm_kernel_L2_END + beq L(sgemm_kernel_L2_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x2 kernel. -.Lsgemm_kernel_L2_Mv1_BEGIN: +L(sgemm_kernel_L2_Mv1_BEGIN): whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s .align 5 -.Lsgemm_kernel_L2_Mv1_20: +L(sgemm_kernel_L2_Mv1_20): mov pB, origPB INITv1x2 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Lsgemm_kernel_L2_Mv1_44 + ble L(sgemm_kernel_L2_Mv1_44) .align 5 -.Lsgemm_kernel_L2_Mv1_22: +L(sgemm_kernel_L2_Mv1_22): KERNELv1x2_SUB KERNELv1x2_SUB @@ -1332,45 +1332,45 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_Mv1_22 + bgt L(sgemm_kernel_L2_Mv1_22) -.Lsgemm_kernel_L2_Mv1_44: +L(sgemm_kernel_L2_Mv1_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L2_Mv1_100 + ble L(sgemm_kernel_L2_Mv1_100) .align 5 -.Lsgemm_kernel_L2_Mv1_46: +L(sgemm_kernel_L2_Mv1_46): KERNELv1x2_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L2_Mv1_46 + bne L(sgemm_kernel_L2_Mv1_46) -.Lsgemm_kernel_L2_Mv1_100: +L(sgemm_kernel_L2_Mv1_100): SAVEv1x2 -.Lsgemm_kernel_L2_Mv1_END: +L(sgemm_kernel_L2_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lsgemm_kernel_L2_Mv1_20 + b.any L(sgemm_kernel_L2_Mv1_20) -.Lsgemm_kernel_L2_END: +L(sgemm_kernel_L2_END): add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 /******************************************************************************/ /* Repeat the same thing if 1 left in N */ .align 5 -.Lsgemm_kernel_L1_BEGIN: +L(sgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lsgemm_kernel_L999 // done + ble L(sgemm_kernel_L999) // done mov pCRow0, pC @@ -1378,26 +1378,26 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Lsgemm_kernel_L1_Mv2_BEGIN: +L(sgemm_kernel_L1_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Lsgemm_kernel_L1_Mv1_BEGIN + blt L(sgemm_kernel_L1_Mv1_BEGIN) mov counterI, origM .align 5 -.Lsgemm_kernel_L1_Mv2_20: +L(sgemm_kernel_L1_Mv2_20): mov pB, origPB INITv2x1 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Lsgemm_kernel_L1_Mv2_44 + ble L(sgemm_kernel_L1_Mv2_44) .align 5 -.Lsgemm_kernel_L1_Mv2_22: +L(sgemm_kernel_L1_Mv2_22): KERNELv2x1_SUB KERNELv2x1_SUB @@ -1409,54 +1409,54 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_Mv2_22 + bgt L(sgemm_kernel_L1_Mv2_22) -.Lsgemm_kernel_L1_Mv2_44: +L(sgemm_kernel_L1_Mv2_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L1_Mv2_100 + ble L(sgemm_kernel_L1_Mv2_100) .align 5 -.Lsgemm_kernel_L1_Mv2_46: +L(sgemm_kernel_L1_Mv2_46): KERNELv2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_Mv2_46 + bgt L(sgemm_kernel_L1_Mv2_46) -.Lsgemm_kernel_L1_Mv2_100: +L(sgemm_kernel_L1_Mv2_100): SAVEv2x1 -.Lsgemm_kernel_L1_Mv2_END: +L(sgemm_kernel_L1_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Lsgemm_kernel_L1_Mv2_20 + bge L(sgemm_kernel_L1_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Lsgemm_kernel_L1_END + beq L(sgemm_kernel_L1_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x1 kernel. -.Lsgemm_kernel_L1_Mv1_BEGIN: +L(sgemm_kernel_L1_Mv1_BEGIN): whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s .align 5 -.Lsgemm_kernel_L1_Mv1_20: +L(sgemm_kernel_L1_Mv1_20): mov pB, origPB INITv1x1 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Lsgemm_kernel_L1_Mv1_44 + ble L(sgemm_kernel_L1_Mv1_44) .align 5 -.Lsgemm_kernel_L1_Mv1_22: +L(sgemm_kernel_L1_Mv1_22): KERNELv1x1_SUB KERNELv1x1_SUB @@ -1468,38 +1468,38 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_Mv1_22 + bgt L(sgemm_kernel_L1_Mv1_22) -.Lsgemm_kernel_L1_Mv1_44: +L(sgemm_kernel_L1_Mv1_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L1_Mv1_100 + ble L(sgemm_kernel_L1_Mv1_100) .align 5 -.Lsgemm_kernel_L1_Mv1_46: +L(sgemm_kernel_L1_Mv1_46): KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_Mv1_46 + bgt L(sgemm_kernel_L1_Mv1_46) -.Lsgemm_kernel_L1_Mv1_100: +L(sgemm_kernel_L1_Mv1_100): SAVEv1x1 -.Lsgemm_kernel_L1_Mv1_END: +L(sgemm_kernel_L1_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lsgemm_kernel_L1_Mv1_20 + b.any L(sgemm_kernel_L1_Mv1_20) -.Lsgemm_kernel_L1_END: +L(sgemm_kernel_L1_END): /******************************************************************************/ -.Lsgemm_kernel_L999: +L(sgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/sgemm_kernel_sve_v1x8.S b/kernel/arm64/sgemm_kernel_sve_v1x8.S index 88c74bc0f5..b3d5e21560 100644 --- a/kernel/arm64/sgemm_kernel_sve_v1x8.S +++ b/kernel/arm64/sgemm_kernel_sve_v1x8.S @@ -479,20 +479,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Ldgemm_kernel_L4_BEGIN + ble L(dgemm_kernel_L4_BEGIN) /******************************************************************************/ /* Repeat this as long as there are 8 left in N */ .align 5 -.Ldgemm_kernel_L8_BEGIN: +L(dgemm_kernel_L8_BEGIN): mov pCRow0, pC add pC, pC, LDC, lsl #3 // add 8 x LDC mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L8_Mv1_BEGIN: +L(dgemm_kernel_L8_Mv1_BEGIN): /* Loop over M is done in an SVE fashion. This has the benefit of the last M%SVE_LEN iterations being done in a single sweep */ mov counterI, #0 @@ -500,14 +500,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension .align 5 -.Ldgemm_kernel_L8_Mv1_20: +L(dgemm_kernel_L8_Mv1_20): mov pB, origPB INITv1x8 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Ldgemm_kernel_L8_Mv1_32 + blt L(dgemm_kernel_L8_Mv1_32) KERNELv1x8_I KERNELv1x8_M2 @@ -519,10 +519,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Ldgemm_kernel_L8_Mv1_22a + ble L(dgemm_kernel_L8_Mv1_22a) .align 5 -.Ldgemm_kernel_L8_Mv1_22: +L(dgemm_kernel_L8_Mv1_22): KERNELv1x8_M1 KERNELv1x8_M2 @@ -534,10 +534,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M2 subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L8_Mv1_22 + bgt L(dgemm_kernel_L8_Mv1_22) .align 5 -.Ldgemm_kernel_L8_Mv1_22a: +L(dgemm_kernel_L8_Mv1_22a): KERNELv1x8_M1 KERNELv1x8_M2 @@ -548,13 +548,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M1 KERNELv1x8_E - b .Ldgemm_kernel_L8_Mv1_44 + b L(dgemm_kernel_L8_Mv1_44) .align 5 -.Ldgemm_kernel_L8_Mv1_32: +L(dgemm_kernel_L8_Mv1_32): tst counterL, #1 - ble .Ldgemm_kernel_L8_Mv1_40 + ble L(dgemm_kernel_L8_Mv1_40) KERNELv1x8_I KERNELv1x8_M2 @@ -566,56 +566,56 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_E - b .Ldgemm_kernel_L8_Mv1_44 + b L(dgemm_kernel_L8_Mv1_44) -.Ldgemm_kernel_L8_Mv1_40: +L(dgemm_kernel_L8_Mv1_40): INITv1x8 -.Ldgemm_kernel_L8_Mv1_44: +L(dgemm_kernel_L8_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L8_Mv1_100 + ble L(dgemm_kernel_L8_Mv1_100) .align 5 -.Ldgemm_kernel_L8_Mv1_46: +L(dgemm_kernel_L8_Mv1_46): KERNELv1x8_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L8_Mv1_46 + bne L(dgemm_kernel_L8_Mv1_46) -.Ldgemm_kernel_L8_Mv1_100: +L(dgemm_kernel_L8_Mv1_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVEv1x8 -.Ldgemm_kernel_L8_Mv1_END: +L(dgemm_kernel_L8_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension - b.any .Ldgemm_kernel_L8_Mv1_20 + b.any L(dgemm_kernel_L8_Mv1_20) -.Ldgemm_kernel_L8_END: +L(dgemm_kernel_L8_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 8 * 4 subs counterJ, counterJ , #1 // j-- - bgt .Ldgemm_kernel_L8_BEGIN + bgt L(dgemm_kernel_L8_BEGIN) /******************************************************************************/ /* Repeat the same thing if 4 left in N */ .align 5 -.Ldgemm_kernel_L4_BEGIN: +L(dgemm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #4 - ble .Ldgemm_kernel_L2_BEGIN + ble L(dgemm_kernel_L2_BEGIN) mov pCRow0, pC @@ -624,24 +624,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L4_Mv1_BEGIN: +L(dgemm_kernel_L4_Mv1_BEGIN): mov counterI, #0 whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s .align 5 -.Ldgemm_kernel_L4_Mv1_20: +L(dgemm_kernel_L4_Mv1_20): mov pB, origPB INITv1x4 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L4_Mv1_44 + ble L(dgemm_kernel_L4_Mv1_44) .align 5 -.Ldgemm_kernel_L4_Mv1_22: +L(dgemm_kernel_L4_Mv1_22): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x4_SUB @@ -657,38 +657,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_Mv1_22 + bgt L(dgemm_kernel_L4_Mv1_22) -.Ldgemm_kernel_L4_Mv1_44: +L(dgemm_kernel_L4_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L4_Mv1_100 + ble L(dgemm_kernel_L4_Mv1_100) .align 5 -.Ldgemm_kernel_L4_Mv1_46: +L(dgemm_kernel_L4_Mv1_46): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L4_Mv1_46 + bne L(dgemm_kernel_L4_Mv1_46) -.Ldgemm_kernel_L4_Mv1_100: +L(dgemm_kernel_L4_Mv1_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVEv1x4 -.Ldgemm_kernel_L4_Mv1_END: +L(dgemm_kernel_L4_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Ldgemm_kernel_L4_Mv1_20 + b.any L(dgemm_kernel_L4_Mv1_20) -.Ldgemm_kernel_L4_END: +L(dgemm_kernel_L4_END): lsl temp, origK, #4 add origPB, origPB, temp // B = B + K * 4 * 4 @@ -696,11 +696,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /* Repeat the same thing if 2 left in N */ .align 5 -.Ldgemm_kernel_L2_BEGIN: +L(dgemm_kernel_L2_BEGIN): mov counterJ , origN tst counterJ , #2 - ble .Ldgemm_kernel_L1_BEGIN + ble L(dgemm_kernel_L1_BEGIN) mov pCRow0, pC @@ -708,24 +708,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L2_Mv1_BEGIN: +L(dgemm_kernel_L2_Mv1_BEGIN): mov counterI, #0 whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s .align 5 -.Ldgemm_kernel_L2_Mv1_20: +L(dgemm_kernel_L2_Mv1_20): mov pB, origPB INITv1x2 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L2_Mv1_44 + ble L(dgemm_kernel_L2_Mv1_44) .align 5 -.Ldgemm_kernel_L2_Mv1_22: +L(dgemm_kernel_L2_Mv1_22): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x2_SUB @@ -739,49 +739,49 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_Mv1_22 + bgt L(dgemm_kernel_L2_Mv1_22) -.Ldgemm_kernel_L2_Mv1_44: +L(dgemm_kernel_L2_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L2_Mv1_100 + ble L(dgemm_kernel_L2_Mv1_100) .align 5 -.Ldgemm_kernel_L2_Mv1_46: +L(dgemm_kernel_L2_Mv1_46): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x2_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L2_Mv1_46 + bne L(dgemm_kernel_L2_Mv1_46) -.Ldgemm_kernel_L2_Mv1_100: +L(dgemm_kernel_L2_Mv1_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVEv1x2 -.Ldgemm_kernel_L2_Mv1_END: +L(dgemm_kernel_L2_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Ldgemm_kernel_L2_Mv1_20 + b.any L(dgemm_kernel_L2_Mv1_20) -.Ldgemm_kernel_L2_END: +L(dgemm_kernel_L2_END): add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 /******************************************************************************/ /* Repeat the same thing if 1 left in N */ .align 5 -.Ldgemm_kernel_L1_BEGIN: +L(dgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldgemm_kernel_L999 // done + ble L(dgemm_kernel_L999) // done mov pCRow0, pC @@ -789,24 +789,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L1_Mv1_BEGIN: +L(dgemm_kernel_L1_Mv1_BEGIN): mov counterI, #0 whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s .align 5 -.Ldgemm_kernel_L1_Mv1_20: +L(dgemm_kernel_L1_Mv1_20): mov pB, origPB INITv1x1 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Ldgemm_kernel_L1_Mv1_44 + ble L(dgemm_kernel_L1_Mv1_44) .align 5 -.Ldgemm_kernel_L1_Mv1_22: +L(dgemm_kernel_L1_Mv1_22): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x1_SUB @@ -819,42 +819,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv1_22 + bgt L(dgemm_kernel_L1_Mv1_22) -.Ldgemm_kernel_L1_Mv1_44: +L(dgemm_kernel_L1_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L1_Mv1_100 + ble L(dgemm_kernel_L1_Mv1_100) .align 5 -.Ldgemm_kernel_L1_Mv1_46: +L(dgemm_kernel_L1_Mv1_46): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv1_46 + bgt L(dgemm_kernel_L1_Mv1_46) -.Ldgemm_kernel_L1_Mv1_100: +L(dgemm_kernel_L1_Mv1_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVEv1x1 -.Ldgemm_kernel_L1_Mv1_END: +L(dgemm_kernel_L1_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Ldgemm_kernel_L1_Mv1_20 + b.any L(dgemm_kernel_L1_Mv1_20) -.Ldgemm_kernel_L1_END: +L(dgemm_kernel_L1_END): /******************************************************************************/ -.Ldgemm_kernel_L999: +L(dgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/sgemm_kernel_sve_v2x8.S b/kernel/arm64/sgemm_kernel_sve_v2x8.S index c969ed4db4..a073987db0 100644 --- a/kernel/arm64/sgemm_kernel_sve_v2x8.S +++ b/kernel/arm64/sgemm_kernel_sve_v2x8.S @@ -882,24 +882,24 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Lsgemm_kernel_L4_BEGIN + ble L(sgemm_kernel_L4_BEGIN) /******************************************************************************/ /* Repeat this as long as there are 8 left in N */ .align 5 -.Lsgemm_kernel_L8_BEGIN: +L(sgemm_kernel_L8_BEGIN): mov pCRow0, pC add pC, pC, LDC, lsl #3 // add 8 x LDC mov pA1, origPA // pA1 = start of A array -.Lsgemm_kernel_L8_Mv2_BEGIN: +L(sgemm_kernel_L8_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 // Check if M < 2*SVE_LEN - blt .Lsgemm_kernel_L8_Mv1_BEGIN + blt L(sgemm_kernel_L8_Mv1_BEGIN) mov counterI, origM @@ -908,14 +908,14 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ add pA2, pA1, temp, lsl #2 // pA1 = start of A array .align 5 -.Lsgemm_kernel_L8_Mv2_20: +L(sgemm_kernel_L8_Mv2_20): mov pB, origPB INITv2x8 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L8_Mv2_32 + blt L(sgemm_kernel_L8_Mv2_32) KERNELv2x8_I KERNELv2x8_M2 @@ -927,10 +927,10 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lsgemm_kernel_L8_Mv2_22a + ble L(sgemm_kernel_L8_Mv2_22a) .align 5 -.Lsgemm_kernel_L8_Mv2_22: +L(sgemm_kernel_L8_Mv2_22): KERNELv2x8_M1 KERNELv2x8_M2 @@ -942,10 +942,10 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_Mv2_22 + bgt L(sgemm_kernel_L8_Mv2_22) .align 5 -.Lsgemm_kernel_L8_Mv2_22a: +L(sgemm_kernel_L8_Mv2_22a): KERNELv2x8_M1 KERNELv2x8_M2 @@ -956,13 +956,13 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M1 KERNELv2x8_E - b .Lsgemm_kernel_L8_Mv2_44 + b L(sgemm_kernel_L8_Mv2_44) .align 5 -.Lsgemm_kernel_L8_Mv2_32: +L(sgemm_kernel_L8_Mv2_32): tst counterL, #1 - ble .Lsgemm_kernel_L8_Mv2_40 + ble L(sgemm_kernel_L8_Mv2_40) KERNELv2x8_I KERNELv2x8_M2 @@ -974,57 +974,57 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_E - b .Lsgemm_kernel_L8_Mv2_44 + b L(sgemm_kernel_L8_Mv2_44) -.Lsgemm_kernel_L8_Mv2_40: +L(sgemm_kernel_L8_Mv2_40): INITv2x8 -.Lsgemm_kernel_L8_Mv2_44: +L(sgemm_kernel_L8_Mv2_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L8_Mv2_100 + ble L(sgemm_kernel_L8_Mv2_100) .align 5 -.Lsgemm_kernel_L8_Mv2_46: +L(sgemm_kernel_L8_Mv2_46): KERNELv2x8_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L8_Mv2_46 + bne L(sgemm_kernel_L8_Mv2_46) -.Lsgemm_kernel_L8_Mv2_100: +L(sgemm_kernel_L8_Mv2_100): SAVEv2x8 mov pA1, pA2 // pA1 = pA2 mul temp, vec_len, origK // generate address of pA2 add pA2, pA1, temp, lsl #2 // -.Lsgemm_kernel_L8_Mv2_END: +L(sgemm_kernel_L8_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Lsgemm_kernel_L8_Mv2_20 + bge L(sgemm_kernel_L8_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Lsgemm_kernel_L8_END + beq L(sgemm_kernel_L8_END) ////////////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x8 kernel. -.Lsgemm_kernel_L8_Mv1_BEGIN: +L(sgemm_kernel_L8_Mv1_BEGIN): whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension .align 5 -.Lsgemm_kernel_L8_Mv1_20: +L(sgemm_kernel_L8_Mv1_20): mov pB, origPB INITv1x8 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L8_Mv1_32 + blt L(sgemm_kernel_L8_Mv1_32) KERNELv1x8_I KERNELv1x8_M2 @@ -1036,10 +1036,10 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lsgemm_kernel_L8_Mv1_22a + ble L(sgemm_kernel_L8_Mv1_22a) .align 5 -.Lsgemm_kernel_L8_Mv1_22: +L(sgemm_kernel_L8_Mv1_22): KERNELv1x8_M1 KERNELv1x8_M2 @@ -1051,10 +1051,10 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_Mv1_22 + bgt L(sgemm_kernel_L8_Mv1_22) .align 5 -.Lsgemm_kernel_L8_Mv1_22a: +L(sgemm_kernel_L8_Mv1_22a): KERNELv1x8_M1 KERNELv1x8_M2 @@ -1065,13 +1065,13 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M1 KERNELv1x8_E - b .Lsgemm_kernel_L8_Mv1_44 + b L(sgemm_kernel_L8_Mv1_44) .align 5 -.Lsgemm_kernel_L8_Mv1_32: +L(sgemm_kernel_L8_Mv1_32): tst counterL, #1 - ble .Lsgemm_kernel_L8_Mv1_40 + ble L(sgemm_kernel_L8_Mv1_40) KERNELv1x8_I KERNELv1x8_M2 @@ -1083,53 +1083,53 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_E - b .Lsgemm_kernel_L8_Mv1_44 + b L(sgemm_kernel_L8_Mv1_44) -.Lsgemm_kernel_L8_Mv1_40: +L(sgemm_kernel_L8_Mv1_40): INITv1x8 -.Lsgemm_kernel_L8_Mv1_44: +L(sgemm_kernel_L8_Mv1_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L8_Mv1_100 + ble L(sgemm_kernel_L8_Mv1_100) .align 5 -.Lsgemm_kernel_L8_Mv1_46: +L(sgemm_kernel_L8_Mv1_46): KERNELv1x8_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L8_Mv1_46 + bne L(sgemm_kernel_L8_Mv1_46) -.Lsgemm_kernel_L8_Mv1_100: +L(sgemm_kernel_L8_Mv1_100): SAVEv1x8 -.Lsgemm_kernel_L8_Mv1_END: +L(sgemm_kernel_L8_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension - b.any .Lsgemm_kernel_L8_Mv1_20 + b.any L(sgemm_kernel_L8_Mv1_20) -.Lsgemm_kernel_L8_END: +L(sgemm_kernel_L8_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 8 * 4 subs counterJ, counterJ , #1 // j-- - bgt .Lsgemm_kernel_L8_BEGIN + bgt L(sgemm_kernel_L8_BEGIN) /******************************************************************************/ /* Repeat the same thing if 4 left in N */ .align 5 -.Lsgemm_kernel_L4_BEGIN: +L(sgemm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #4 - ble .Lsgemm_kernel_L2_BEGIN + ble L(sgemm_kernel_L2_BEGIN) mov pCRow0, pC @@ -1138,11 +1138,11 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Lsgemm_kernel_L4_Mv2_BEGIN: +L(sgemm_kernel_L4_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Lsgemm_kernel_L4_Mv1_BEGIN + blt L(sgemm_kernel_L4_Mv1_BEGIN) mov counterI, origM @@ -1150,17 +1150,17 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ add pA2, pA1, temp, lsl #2 // pA1 = start of A array .align 5 -.Lsgemm_kernel_L4_Mv2_20: +L(sgemm_kernel_L4_Mv2_20): mov pB, origPB INITv2x4 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Lsgemm_kernel_L4_Mv2_44 + ble L(sgemm_kernel_L4_Mv2_44) .align 5 -.Lsgemm_kernel_L4_Mv2_22: +L(sgemm_kernel_L4_Mv2_22): KERNELv2x4_SUB KERNELv2x4_SUB @@ -1172,56 +1172,56 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_Mv2_22 + bgt L(sgemm_kernel_L4_Mv2_22) -.Lsgemm_kernel_L4_Mv2_44: +L(sgemm_kernel_L4_Mv2_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L4_Mv2_100 + ble L(sgemm_kernel_L4_Mv2_100) .align 5 -.Lsgemm_kernel_L4_Mv2_46: +L(sgemm_kernel_L4_Mv2_46): KERNELv2x4_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L4_Mv2_46 + bne L(sgemm_kernel_L4_Mv2_46) -.Lsgemm_kernel_L4_Mv2_100: +L(sgemm_kernel_L4_Mv2_100): SAVEv2x4 mov pA1, pA2 // pA1 = pA2 mul temp, vec_len, origK // generate address of pA2 add pA2, pA1, temp, lsl #2 // -.Lsgemm_kernel_L4_Mv2_END: +L(sgemm_kernel_L4_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Lsgemm_kernel_L4_Mv2_20 + bge L(sgemm_kernel_L4_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Lsgemm_kernel_L4_END + beq L(sgemm_kernel_L4_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x4 kernel. -.Lsgemm_kernel_L4_Mv1_BEGIN: +L(sgemm_kernel_L4_Mv1_BEGIN): whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension .align 5 -.Lsgemm_kernel_L4_Mv1_20: +L(sgemm_kernel_L4_Mv1_20): mov pB, origPB INITv1x4 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Lsgemm_kernel_L4_Mv1_44 + ble L(sgemm_kernel_L4_Mv1_44) .align 5 -.Lsgemm_kernel_L4_Mv1_22: +L(sgemm_kernel_L4_Mv1_22): KERNELv1x4_SUB KERNELv1x4_SUB @@ -1233,34 +1233,34 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_Mv1_22 + bgt L(sgemm_kernel_L4_Mv1_22) -.Lsgemm_kernel_L4_Mv1_44: +L(sgemm_kernel_L4_Mv1_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L4_Mv1_100 + ble L(sgemm_kernel_L4_Mv1_100) .align 5 -.Lsgemm_kernel_L4_Mv1_46: +L(sgemm_kernel_L4_Mv1_46): KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L4_Mv1_46 + bne L(sgemm_kernel_L4_Mv1_46) -.Lsgemm_kernel_L4_Mv1_100: +L(sgemm_kernel_L4_Mv1_100): SAVEv1x4 -.Lsgemm_kernel_L4_Mv1_END: +L(sgemm_kernel_L4_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lsgemm_kernel_L4_Mv1_20 + b.any L(sgemm_kernel_L4_Mv1_20) -.Lsgemm_kernel_L4_END: +L(sgemm_kernel_L4_END): lsl temp, origK, #4 add origPB, origPB, temp // B = B + K * 4 * 4 @@ -1268,11 +1268,11 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ /* Repeat the same thing if 2 left in N */ .align 5 -.Lsgemm_kernel_L2_BEGIN: +L(sgemm_kernel_L2_BEGIN): mov counterJ , origN tst counterJ , #2 - ble .Lsgemm_kernel_L1_BEGIN + ble L(sgemm_kernel_L1_BEGIN) mov pCRow0, pC @@ -1280,11 +1280,11 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Lsgemm_kernel_L2_Mv2_BEGIN: +L(sgemm_kernel_L2_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Lsgemm_kernel_L2_Mv1_BEGIN + blt L(sgemm_kernel_L2_Mv1_BEGIN) mov counterI, origM @@ -1292,17 +1292,17 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ add pA2, pA1, temp, lsl #2 // pA1 = start of A array .align 5 -.Lsgemm_kernel_L2_Mv2_20: +L(sgemm_kernel_L2_Mv2_20): mov pB, origPB INITv2x2 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Lsgemm_kernel_L2_Mv2_44 + ble L(sgemm_kernel_L2_Mv2_44) .align 5 -.Lsgemm_kernel_L2_Mv2_22: +L(sgemm_kernel_L2_Mv2_22): KERNELv2x2_SUB KERNELv2x2_SUB @@ -1314,57 +1314,57 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_Mv2_22 + bgt L(sgemm_kernel_L2_Mv2_22) -.Lsgemm_kernel_L2_Mv2_44: +L(sgemm_kernel_L2_Mv2_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L2_Mv2_100 + ble L(sgemm_kernel_L2_Mv2_100) .align 5 -.Lsgemm_kernel_L2_Mv2_46: +L(sgemm_kernel_L2_Mv2_46): KERNELv2x2_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L2_Mv2_46 + bne L(sgemm_kernel_L2_Mv2_46) -.Lsgemm_kernel_L2_Mv2_100: +L(sgemm_kernel_L2_Mv2_100): SAVEv2x2 mov pA1, pA2 // pA1 = pA2 mul temp, vec_len, origK // generate address of pA2 add pA2, pA1, temp, lsl #2 // -.Lsgemm_kernel_L2_Mv2_END: +L(sgemm_kernel_L2_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Lsgemm_kernel_L2_Mv2_20 + bge L(sgemm_kernel_L2_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Lsgemm_kernel_L2_END + beq L(sgemm_kernel_L2_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x2 kernel. -.Lsgemm_kernel_L2_Mv1_BEGIN: +L(sgemm_kernel_L2_Mv1_BEGIN): whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s .align 5 -.Lsgemm_kernel_L2_Mv1_20: +L(sgemm_kernel_L2_Mv1_20): mov pB, origPB INITv1x2 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Lsgemm_kernel_L2_Mv1_44 + ble L(sgemm_kernel_L2_Mv1_44) .align 5 -.Lsgemm_kernel_L2_Mv1_22: +L(sgemm_kernel_L2_Mv1_22): KERNELv1x2_SUB KERNELv1x2_SUB @@ -1376,45 +1376,45 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_Mv1_22 + bgt L(sgemm_kernel_L2_Mv1_22) -.Lsgemm_kernel_L2_Mv1_44: +L(sgemm_kernel_L2_Mv1_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L2_Mv1_100 + ble L(sgemm_kernel_L2_Mv1_100) .align 5 -.Lsgemm_kernel_L2_Mv1_46: +L(sgemm_kernel_L2_Mv1_46): KERNELv1x2_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L2_Mv1_46 + bne L(sgemm_kernel_L2_Mv1_46) -.Lsgemm_kernel_L2_Mv1_100: +L(sgemm_kernel_L2_Mv1_100): SAVEv1x2 -.Lsgemm_kernel_L2_Mv1_END: +L(sgemm_kernel_L2_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lsgemm_kernel_L2_Mv1_20 + b.any L(sgemm_kernel_L2_Mv1_20) -.Lsgemm_kernel_L2_END: +L(sgemm_kernel_L2_END): add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 /******************************************************************************/ /* Repeat the same thing if 1 left in N */ .align 5 -.Lsgemm_kernel_L1_BEGIN: +L(sgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lsgemm_kernel_L999 // done + ble L(sgemm_kernel_L999) // done mov pCRow0, pC @@ -1422,11 +1422,11 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Lsgemm_kernel_L1_Mv2_BEGIN: +L(sgemm_kernel_L1_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Lsgemm_kernel_L1_Mv1_BEGIN + blt L(sgemm_kernel_L1_Mv1_BEGIN) mov counterI, origM @@ -1435,17 +1435,17 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ .align 5 -.Lsgemm_kernel_L1_Mv2_20: +L(sgemm_kernel_L1_Mv2_20): mov pB, origPB INITv2x1 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Lsgemm_kernel_L1_Mv2_44 + ble L(sgemm_kernel_L1_Mv2_44) .align 5 -.Lsgemm_kernel_L1_Mv2_22: +L(sgemm_kernel_L1_Mv2_22): KERNELv2x1_SUB KERNELv2x1_SUB @@ -1457,57 +1457,57 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_Mv2_22 + bgt L(sgemm_kernel_L1_Mv2_22) -.Lsgemm_kernel_L1_Mv2_44: +L(sgemm_kernel_L1_Mv2_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L1_Mv2_100 + ble L(sgemm_kernel_L1_Mv2_100) .align 5 -.Lsgemm_kernel_L1_Mv2_46: +L(sgemm_kernel_L1_Mv2_46): KERNELv2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_Mv2_46 + bgt L(sgemm_kernel_L1_Mv2_46) -.Lsgemm_kernel_L1_Mv2_100: +L(sgemm_kernel_L1_Mv2_100): SAVEv2x1 mov pA1, pA2 // pA1 = pA2 mul temp, vec_len, origK // generate address of pA2 add pA2, pA1, temp, lsl #2 // -.Lsgemm_kernel_L1_Mv2_END: +L(sgemm_kernel_L1_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Lsgemm_kernel_L1_Mv2_20 + bge L(sgemm_kernel_L1_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Lsgemm_kernel_L1_END + beq L(sgemm_kernel_L1_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x1 kernel. -.Lsgemm_kernel_L1_Mv1_BEGIN: +L(sgemm_kernel_L1_Mv1_BEGIN): whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s .align 5 -.Lsgemm_kernel_L1_Mv1_20: +L(sgemm_kernel_L1_Mv1_20): mov pB, origPB INITv1x1 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Lsgemm_kernel_L1_Mv1_44 + ble L(sgemm_kernel_L1_Mv1_44) .align 5 -.Lsgemm_kernel_L1_Mv1_22: +L(sgemm_kernel_L1_Mv1_22): KERNELv1x1_SUB KERNELv1x1_SUB @@ -1519,38 +1519,38 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_Mv1_22 + bgt L(sgemm_kernel_L1_Mv1_22) -.Lsgemm_kernel_L1_Mv1_44: +L(sgemm_kernel_L1_Mv1_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L1_Mv1_100 + ble L(sgemm_kernel_L1_Mv1_100) .align 5 -.Lsgemm_kernel_L1_Mv1_46: +L(sgemm_kernel_L1_Mv1_46): KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_Mv1_46 + bgt L(sgemm_kernel_L1_Mv1_46) -.Lsgemm_kernel_L1_Mv1_100: +L(sgemm_kernel_L1_Mv1_100): SAVEv1x1 -.Lsgemm_kernel_L1_Mv1_END: +L(sgemm_kernel_L1_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lsgemm_kernel_L1_Mv1_20 + b.any L(sgemm_kernel_L1_Mv1_20) -.Lsgemm_kernel_L1_END: +L(sgemm_kernel_L1_END): /******************************************************************************/ -.Lsgemm_kernel_L999: +L(sgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/sgemm_ncopy_4.S b/kernel/arm64/sgemm_ncopy_4.S index de8c8eca6f..5570bb1459 100644 --- a/kernel/arm64/sgemm_ncopy_4.S +++ b/kernel/arm64/sgemm_ncopy_4.S @@ -182,14 +182,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. lsl LDA, LDA, #2 // LDA = LDA * SIZE -.Ldgemm_ncopy_L4_BEGIN: +L(dgemm_ncopy_L4_BEGIN): asr J, N, #2 // J = N / 4 cmp J, #0 - ble .Ldgemm_ncopy_L2_BEGIN + ble L(dgemm_ncopy_L2_BEGIN) .align 5 -.Ldgemm_ncopy_L4_M4_BEGIN: +L(dgemm_ncopy_L4_M4_BEGIN): mov A01, A00 add A02, A01, LDA @@ -199,121 +199,121 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #2 // I = M / 4 cmp I, #0 - ble .Ldgemm_ncopy_L4_M4_40 + ble L(dgemm_ncopy_L4_M4_40) .align 5 -.Ldgemm_ncopy_L4_M4_20: +L(dgemm_ncopy_L4_M4_20): COPY4x4 subs I , I , #1 - bne .Ldgemm_ncopy_L4_M4_20 + bne L(dgemm_ncopy_L4_M4_20) -.Ldgemm_ncopy_L4_M4_40: +L(dgemm_ncopy_L4_M4_40): and I, M , #3 cmp I, #0 - ble .Ldgemm_ncopy_L4_M4_END + ble L(dgemm_ncopy_L4_M4_END) .align 5 -.Ldgemm_ncopy_L4_M4_60: +L(dgemm_ncopy_L4_M4_60): COPY1x4 subs I , I , #1 - bne .Ldgemm_ncopy_L4_M4_60 + bne L(dgemm_ncopy_L4_M4_60) -.Ldgemm_ncopy_L4_M4_END: +L(dgemm_ncopy_L4_M4_END): subs J , J, #1 // j-- - bne .Ldgemm_ncopy_L4_M4_BEGIN + bne L(dgemm_ncopy_L4_M4_BEGIN) /*********************************************************************************************/ -.Ldgemm_ncopy_L2_BEGIN: +L(dgemm_ncopy_L2_BEGIN): tst N, #3 - ble .Ldgemm_ncopy_L999 + ble L(dgemm_ncopy_L999) tst N, #2 - ble .Ldgemm_ncopy_L1_BEGIN + ble L(dgemm_ncopy_L1_BEGIN) -.Ldgemm_ncopy_L2_M4_BEGIN: +L(dgemm_ncopy_L2_M4_BEGIN): mov A01, A00 add A02, A01, LDA add A00, A02, LDA asr I, M, #2 // I = M / 4 cmp I, #0 - ble .Ldgemm_ncopy_L2_M4_40 + ble L(dgemm_ncopy_L2_M4_40) .align 5 -.Ldgemm_ncopy_L2_M4_20: +L(dgemm_ncopy_L2_M4_20): COPY4x2 subs I , I , #1 - bne .Ldgemm_ncopy_L2_M4_20 + bne L(dgemm_ncopy_L2_M4_20) -.Ldgemm_ncopy_L2_M4_40: +L(dgemm_ncopy_L2_M4_40): and I, M , #3 cmp I, #0 - ble .Ldgemm_ncopy_L2_M4_END + ble L(dgemm_ncopy_L2_M4_END) .align 5 -.Ldgemm_ncopy_L2_M4_60: +L(dgemm_ncopy_L2_M4_60): COPY1x2 subs I , I , #1 - bne .Ldgemm_ncopy_L2_M4_60 + bne L(dgemm_ncopy_L2_M4_60) -.Ldgemm_ncopy_L2_M4_END: +L(dgemm_ncopy_L2_M4_END): /*********************************************************************************************/ -.Ldgemm_ncopy_L1_BEGIN: +L(dgemm_ncopy_L1_BEGIN): tst N, #1 - ble .Ldgemm_ncopy_L999 + ble L(dgemm_ncopy_L999) -.Ldgemm_ncopy_L1_M4_BEGIN: +L(dgemm_ncopy_L1_M4_BEGIN): mov A01, A00 asr I, M, #2 // I = M / 4 cmp I, #0 - ble .Ldgemm_ncopy_L1_M4_40 + ble L(dgemm_ncopy_L1_M4_40) .align 5 -.Ldgemm_ncopy_L1_M4_20: +L(dgemm_ncopy_L1_M4_20): COPY4x1 subs I , I , #1 - bne .Ldgemm_ncopy_L1_M4_20 + bne L(dgemm_ncopy_L1_M4_20) -.Ldgemm_ncopy_L1_M4_40: +L(dgemm_ncopy_L1_M4_40): and I, M , #3 cmp I, #0 - ble .Ldgemm_ncopy_L1_M4_END + ble L(dgemm_ncopy_L1_M4_END) .align 5 -.Ldgemm_ncopy_L1_M4_60: +L(dgemm_ncopy_L1_M4_60): COPY1x1 subs I , I , #1 - bne .Ldgemm_ncopy_L1_M4_60 + bne L(dgemm_ncopy_L1_M4_60) -.Ldgemm_ncopy_L1_M4_END: +L(dgemm_ncopy_L1_M4_END): -.Ldgemm_ncopy_L999: +L(dgemm_ncopy_L999): mov x0, #0 RESTORE_REGS diff --git a/kernel/arm64/sgemm_ncopy_8.S b/kernel/arm64/sgemm_ncopy_8.S index d941eb3eb4..11b07c0844 100644 --- a/kernel/arm64/sgemm_ncopy_8.S +++ b/kernel/arm64/sgemm_ncopy_8.S @@ -267,14 +267,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. lsl LDA, LDA, #2 // LDA = LDA * SIZE -.Lsgemm_ncopy_L8_BEGIN: +L(sgemm_ncopy_L8_BEGIN): asr J, N, #3 // J = N / 8 cmp J, #0 - ble .Lsgemm_ncopy_L4_BEGIN + ble L(sgemm_ncopy_L4_BEGIN) .align 5 -.Lsgemm_ncopy_L8_M4_BEGIN: +L(sgemm_ncopy_L8_M4_BEGIN): mov A01, A00 add A02, A01, LDA @@ -288,136 +288,136 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #2 // I = M / 4 cmp I, #0 - ble .Lsgemm_ncopy_L8_M4_40 + ble L(sgemm_ncopy_L8_M4_40) asr K, M, #4 // K = M / 16(cacheline) mov TEMP1, A01 .align 5 -.Lsgemm_tcopy_L8_warnup_1: +L(sgemm_tcopy_L8_warnup_1): ldr s0, [TEMP1], #64 subs K, K, #1 - bgt .Lsgemm_tcopy_L8_warnup_1 + bgt L(sgemm_tcopy_L8_warnup_1) asr K, M, #4 // K = M / 16(cacheline) mov TEMP1, A02 .align 5 -.Lsgemm_tcopy_L8_warnup_2: +L(sgemm_tcopy_L8_warnup_2): ldr s0, [TEMP1], #64 subs K, K, #1 - bgt .Lsgemm_tcopy_L8_warnup_2 + bgt L(sgemm_tcopy_L8_warnup_2) asr K, M, #4 // K = M / 16(cacheline) mov TEMP1, A03 .align 5 -.Lsgemm_tcopy_L8_warnup_3: +L(sgemm_tcopy_L8_warnup_3): ldr s0, [TEMP1], #64 subs K, K, #1 - bgt .Lsgemm_tcopy_L8_warnup_3 + bgt L(sgemm_tcopy_L8_warnup_3) asr K, M, #4 // K = M / 16(cacheline) mov TEMP1, A04 .align 5 -.Lsgemm_tcopy_L8_warnup_4: +L(sgemm_tcopy_L8_warnup_4): ldr s0, [TEMP1], #64 subs K, K, #1 - bgt .Lsgemm_tcopy_L8_warnup_4 + bgt L(sgemm_tcopy_L8_warnup_4) asr K, M, #4 // K = M / 16(cacheline) mov TEMP1, A05 .align 5 -.Lsgemm_tcopy_L8_warnup_5: +L(sgemm_tcopy_L8_warnup_5): ldr s0, [TEMP1], #64 subs K, K, #1 - bgt .Lsgemm_tcopy_L8_warnup_5 + bgt L(sgemm_tcopy_L8_warnup_5) asr K, M, #4 // K = M / 16(cacheline) mov TEMP1, A06 .align 5 -.Lsgemm_tcopy_L8_warnup_6: +L(sgemm_tcopy_L8_warnup_6): ldr s0, [TEMP1], #64 subs K, K, #1 - bgt .Lsgemm_tcopy_L8_warnup_6 + bgt L(sgemm_tcopy_L8_warnup_6) asr K, M, #4 // K = M / 16(cacheline) mov TEMP1, A07 .align 5 -.Lsgemm_tcopy_L8_warnup_7: +L(sgemm_tcopy_L8_warnup_7): ldr s0, [TEMP1], #64 subs K, K, #1 - bgt .Lsgemm_tcopy_L8_warnup_7 + bgt L(sgemm_tcopy_L8_warnup_7) asr K, M, #4 // K = M / 16(cacheline) mov TEMP1, A08 .align 5 -.Lsgemm_tcopy_L8_warnup_8: +L(sgemm_tcopy_L8_warnup_8): ldr s0, [TEMP1], #64 subs K, K, #1 - bgt .Lsgemm_tcopy_L8_warnup_8 + bgt L(sgemm_tcopy_L8_warnup_8) .align 5 -.Lsgemm_ncopy_L8_M4_20: +L(sgemm_ncopy_L8_M4_20): COPY4x8 subs I, I, #1 - bne .Lsgemm_ncopy_L8_M4_20 + bne L(sgemm_ncopy_L8_M4_20) -.Lsgemm_ncopy_L8_M4_40: +L(sgemm_ncopy_L8_M4_40): and I, M, #2 cmp I, #0 - ble .Lsgemm_ncopy_L8_M4_60 + ble L(sgemm_ncopy_L8_M4_60) COPY2x8 -.Lsgemm_ncopy_L8_M4_60: +L(sgemm_ncopy_L8_M4_60): and I, M, #1 cmp I, #0 - ble .Lsgemm_ncopy_L8_M4_END + ble L(sgemm_ncopy_L8_M4_END) COPY1x8 -.Lsgemm_ncopy_L8_M4_END: +L(sgemm_ncopy_L8_M4_END): subs J , J, #1 // j-- - bne .Lsgemm_ncopy_L8_M4_BEGIN + bne L(sgemm_ncopy_L8_M4_BEGIN) /*********************************************************************************************/ -.Lsgemm_ncopy_L4_BEGIN: +L(sgemm_ncopy_L4_BEGIN): tst N, #7 - ble .Lsgemm_ncopy_L999 + ble L(sgemm_ncopy_L999) tst N, #4 - ble .Lsgemm_ncopy_L2_BEGIN + ble L(sgemm_ncopy_L2_BEGIN) -.Lsgemm_ncopy_L4_M4_BEGIN: +L(sgemm_ncopy_L4_M4_BEGIN): mov A01, A00 add A02, A01, LDA add A03, A02, LDA @@ -426,43 +426,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #2 // I = M / 4 cmp I, #0 - ble .Lsgemm_ncopy_L4_M4_40 + ble L(sgemm_ncopy_L4_M4_40) .align 5 -.Lsgemm_ncopy_L4_M4_20: +L(sgemm_ncopy_L4_M4_20): COPY4x4 subs I, I, #1 - bne .Lsgemm_ncopy_L4_M4_20 + bne L(sgemm_ncopy_L4_M4_20) -.Lsgemm_ncopy_L4_M4_40: +L(sgemm_ncopy_L4_M4_40): and I, M, #2 cmp I, #0 - ble .Lsgemm_ncopy_L4_M4_60 + ble L(sgemm_ncopy_L4_M4_60) COPY2x4 -.Lsgemm_ncopy_L4_M4_60: +L(sgemm_ncopy_L4_M4_60): and I, M, #1 cmp I, #0 - ble .Lsgemm_ncopy_L4_M4_END + ble L(sgemm_ncopy_L4_M4_END) COPY1x4 -.Lsgemm_ncopy_L4_M4_END: +L(sgemm_ncopy_L4_M4_END): /*********************************************************************************************/ -.Lsgemm_ncopy_L2_BEGIN: +L(sgemm_ncopy_L2_BEGIN): tst N, #2 - ble .Lsgemm_ncopy_L1_BEGIN + ble L(sgemm_ncopy_L1_BEGIN) -.Lsgemm_ncopy_L2_M4_BEGIN: +L(sgemm_ncopy_L2_M4_BEGIN): mov A01, A00 add A02, A01, LDA @@ -470,59 +470,59 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #2 // I = M / 4 cmp I, #0 - ble .Lsgemm_ncopy_L2_M4_40 + ble L(sgemm_ncopy_L2_M4_40) .align 5 -.Lsgemm_ncopy_L2_M4_20: +L(sgemm_ncopy_L2_M4_20): COPY4x2 subs I , I , #1 - bne .Lsgemm_ncopy_L2_M4_20 + bne L(sgemm_ncopy_L2_M4_20) -.Lsgemm_ncopy_L2_M4_40: +L(sgemm_ncopy_L2_M4_40): and I, M, #2 cmp I, #0 - ble .Lsgemm_ncopy_L2_M4_60 + ble L(sgemm_ncopy_L2_M4_60) COPY2x2 -.Lsgemm_ncopy_L2_M4_60: +L(sgemm_ncopy_L2_M4_60): and I, M, #1 cmp I, #0 - ble .Lsgemm_ncopy_L2_M4_END + ble L(sgemm_ncopy_L2_M4_END) COPY1x2 -.Lsgemm_ncopy_L2_M4_END: +L(sgemm_ncopy_L2_M4_END): -.Lsgemm_ncopy_L1_BEGIN: +L(sgemm_ncopy_L1_BEGIN): tst N, #1 - ble .Lsgemm_ncopy_L999 + ble L(sgemm_ncopy_L999) -.Lsgemm_ncopy_L1_M1_BEGIN: +L(sgemm_ncopy_L1_M1_BEGIN): mov A01, A00 mov I, M cmp I, #0 - ble .Lsgemm_ncopy_L1_M1_END + ble L(sgemm_ncopy_L1_M1_END) .align 5 -.Lsgemm_ncopy_L1_M1_20: +L(sgemm_ncopy_L1_M1_20): COPY1x1 subs I, I, #1 - bne .Lsgemm_ncopy_L1_M1_20 + bne L(sgemm_ncopy_L1_M1_20) -.Lsgemm_ncopy_L1_M1_END: +L(sgemm_ncopy_L1_M1_END): -.Lsgemm_ncopy_L999: +L(sgemm_ncopy_L999): mov x0, #0 RESTORE_REGS diff --git a/kernel/arm64/sgemm_tcopy_16.S b/kernel/arm64/sgemm_tcopy_16.S index 3066421bb9..934fe933eb 100644 --- a/kernel/arm64/sgemm_tcopy_16.S +++ b/kernel/arm64/sgemm_tcopy_16.S @@ -566,13 +566,13 @@ All rights reserved. lsl M8, M, #6 // M8 = M * 16 * SIZE -.Lsgemm_tcopy_L8_BEGIN: +L(sgemm_tcopy_L8_BEGIN): asr J, M, #3 // J = M / 8 cmp J, #0 - ble .Lsgemm_tcopy_L4_BEGIN + ble L(sgemm_tcopy_L4_BEGIN) .align 5 -.Lsgemm_tcopy_L8_M16_BEGIN: +L(sgemm_tcopy_L8_M16_BEGIN): mov A01, A add A02, A01, LDA @@ -589,57 +589,57 @@ All rights reserved. asr I, N, #4 // I = N / 16 cmp I, #0 - ble .Lsgemm_tcopy_L8_M16_40 + ble L(sgemm_tcopy_L8_M16_40) .align 5 -.Lsgemm_tcopy_L8_M16_20: +L(sgemm_tcopy_L8_M16_20): COPY16x8 subs I , I , #1 - bne .Lsgemm_tcopy_L8_M16_20 + bne L(sgemm_tcopy_L8_M16_20) -.Lsgemm_tcopy_L8_M16_40: +L(sgemm_tcopy_L8_M16_40): tst N , #8 - ble .Lsgemm_tcopy_L8_M16_60 + ble L(sgemm_tcopy_L8_M16_60) COPY8x8 -.Lsgemm_tcopy_L8_M16_60: +L(sgemm_tcopy_L8_M16_60): tst N , #4 - ble .Lsgemm_tcopy_L8_M16_80 + ble L(sgemm_tcopy_L8_M16_80) COPY4x8 -.Lsgemm_tcopy_L8_M16_80: +L(sgemm_tcopy_L8_M16_80): tst N , #2 - ble .Lsgemm_tcopy_L8_M16_100 + ble L(sgemm_tcopy_L8_M16_100) COPY2x8 -.Lsgemm_tcopy_L8_M16_100: +L(sgemm_tcopy_L8_M16_100): tst N, #1 - ble .Lsgemm_tcopy_L8_M16_END + ble L(sgemm_tcopy_L8_M16_END) COPY1x8 -.Lsgemm_tcopy_L8_M16_END: +L(sgemm_tcopy_L8_M16_END): subs J , J, #1 // j-- - bne .Lsgemm_tcopy_L8_M16_BEGIN + bne L(sgemm_tcopy_L8_M16_BEGIN) /*********************************************************************************************/ -.Lsgemm_tcopy_L4_BEGIN: +L(sgemm_tcopy_L4_BEGIN): tst M, #7 - ble .Lsgemm_tcopy_L999 + ble L(sgemm_tcopy_L999) tst M, #4 - ble .Lsgemm_tcopy_L2_BEGIN + ble L(sgemm_tcopy_L2_BEGIN) -.Lsgemm_tcopy_L4_M16_BEGIN: +L(sgemm_tcopy_L4_M16_BEGIN): mov A01, A add A02, A01, LDA @@ -652,57 +652,57 @@ All rights reserved. asr I, N, #4 // I = N / 16 cmp I, #0 - ble .Lsgemm_tcopy_L4_M16_40 + ble L(sgemm_tcopy_L4_M16_40) .align 5 -.Lsgemm_tcopy_L4_M16_20: +L(sgemm_tcopy_L4_M16_20): COPY16x4 subs I , I , #1 - bne .Lsgemm_tcopy_L4_M16_20 + bne L(sgemm_tcopy_L4_M16_20) -.Lsgemm_tcopy_L4_M16_40: +L(sgemm_tcopy_L4_M16_40): tst N , #8 - ble .Lsgemm_tcopy_L4_M16_60 + ble L(sgemm_tcopy_L4_M16_60) COPY8x4 -.Lsgemm_tcopy_L4_M16_60: +L(sgemm_tcopy_L4_M16_60): tst N , #4 - ble .Lsgemm_tcopy_L4_M16_80 + ble L(sgemm_tcopy_L4_M16_80) COPY4x4 -.Lsgemm_tcopy_L4_M16_80: +L(sgemm_tcopy_L4_M16_80): tst N , #2 - ble .Lsgemm_tcopy_L4_M16_100 + ble L(sgemm_tcopy_L4_M16_100) COPY2x4 -.Lsgemm_tcopy_L4_M16_100: +L(sgemm_tcopy_L4_M16_100): tst N, #1 - ble .Lsgemm_tcopy_L4_M16_END + ble L(sgemm_tcopy_L4_M16_END) COPY1x4 -.Lsgemm_tcopy_L4_M16_END: +L(sgemm_tcopy_L4_M16_END): /*********************************************************************************************/ -.Lsgemm_tcopy_L2_BEGIN: +L(sgemm_tcopy_L2_BEGIN): tst M, #3 - ble .Lsgemm_tcopy_L999 + ble L(sgemm_tcopy_L999) tst M, #2 - ble .Lsgemm_tcopy_L1_BEGIN + ble L(sgemm_tcopy_L1_BEGIN) -.Lsgemm_tcopy_L2_M16_BEGIN: +L(sgemm_tcopy_L2_M16_BEGIN): mov A01, A add A02, A01, LDA add A, A02, LDA @@ -712,99 +712,99 @@ All rights reserved. asr I, N, #4 // I = N / 16 cmp I, #0 - ble .Lsgemm_tcopy_L2_M16_40 + ble L(sgemm_tcopy_L2_M16_40) .align 5 -.Lsgemm_tcopy_L2_M16_20: +L(sgemm_tcopy_L2_M16_20): COPY16x2 subs I , I , #1 - bne .Lsgemm_tcopy_L2_M16_20 + bne L(sgemm_tcopy_L2_M16_20) -.Lsgemm_tcopy_L2_M16_40: +L(sgemm_tcopy_L2_M16_40): tst N , #8 - ble .Lsgemm_tcopy_L2_M16_60 + ble L(sgemm_tcopy_L2_M16_60) COPY8x2 -.Lsgemm_tcopy_L2_M16_60: +L(sgemm_tcopy_L2_M16_60): tst N , #4 - ble .Lsgemm_tcopy_L2_M16_80 + ble L(sgemm_tcopy_L2_M16_80) COPY4x2 -.Lsgemm_tcopy_L2_M16_80: +L(sgemm_tcopy_L2_M16_80): tst N , #2 - ble .Lsgemm_tcopy_L2_M16_100 + ble L(sgemm_tcopy_L2_M16_100) COPY2x2 -.Lsgemm_tcopy_L2_M16_100: +L(sgemm_tcopy_L2_M16_100): tst N , #1 - ble .Lsgemm_tcopy_L2_M16_END + ble L(sgemm_tcopy_L2_M16_END) COPY1x2 -.Lsgemm_tcopy_L2_M16_END: +L(sgemm_tcopy_L2_M16_END): /*********************************************************************************************/ -.Lsgemm_tcopy_L1_BEGIN: +L(sgemm_tcopy_L1_BEGIN): tst M, #1 - ble .Lsgemm_tcopy_L999 + ble L(sgemm_tcopy_L999) -.Lsgemm_tcopy_L1_M16_BEGIN: +L(sgemm_tcopy_L1_M16_BEGIN): mov A01, A // A01 = A mov B00, B asr I, N, #4 // I = M / 16 cmp I, #0 - ble .Lsgemm_tcopy_L1_M16_40 + ble L(sgemm_tcopy_L1_M16_40) .align 5 -.Lsgemm_tcopy_L1_M16_20: +L(sgemm_tcopy_L1_M16_20): COPY16x1 subs I , I , #1 - bne .Lsgemm_tcopy_L1_M16_20 + bne L(sgemm_tcopy_L1_M16_20) -.Lsgemm_tcopy_L1_M16_40: +L(sgemm_tcopy_L1_M16_40): tst N , #8 - ble .Lsgemm_tcopy_L1_M16_60 + ble L(sgemm_tcopy_L1_M16_60) COPY8x1 -.Lsgemm_tcopy_L1_M16_60: +L(sgemm_tcopy_L1_M16_60): tst N , #4 - ble .Lsgemm_tcopy_L1_M16_80 + ble L(sgemm_tcopy_L1_M16_80) COPY4x1 -.Lsgemm_tcopy_L1_M16_80: +L(sgemm_tcopy_L1_M16_80): tst N , #2 - ble .Lsgemm_tcopy_L1_M16_100 + ble L(sgemm_tcopy_L1_M16_100) COPY2x1 -.Lsgemm_tcopy_L1_M16_100: +L(sgemm_tcopy_L1_M16_100): tst N , #1 - ble .Lsgemm_tcopy_L1_M16_END + ble L(sgemm_tcopy_L1_M16_END) COPY1x1 -.Lsgemm_tcopy_L1_M16_END: +L(sgemm_tcopy_L1_M16_END): -.Lsgemm_tcopy_L999: +L(sgemm_tcopy_L999): mov x0, #0 // set return value RESTORE_REGS ret diff --git a/kernel/arm64/sgemm_tcopy_8.S b/kernel/arm64/sgemm_tcopy_8.S index 7d81ba2667..ef52b98b75 100644 --- a/kernel/arm64/sgemm_tcopy_8.S +++ b/kernel/arm64/sgemm_tcopy_8.S @@ -478,14 +478,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. lsl M8, M, #5 // M8 = M * 8 * SIZE -.Lsgemm_tcopy_L8_BEGIN: +L(sgemm_tcopy_L8_BEGIN): asr J, M, #3 // J = M / 8 cmp J, #0 - ble .Lsgemm_tcopy_L4_BEGIN + ble L(sgemm_tcopy_L4_BEGIN) .align 5 -.Lsgemm_tcopy_L8_M8_BEGIN: +L(sgemm_tcopy_L8_M8_BEGIN): mov A01, A add A02, A01, LDA @@ -502,53 +502,53 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, N, #3 // I = N / 8 cmp I, #0 - ble .Lsgemm_tcopy_L8_M8_40 + ble L(sgemm_tcopy_L8_M8_40) .align 5 -.Lsgemm_tcopy_L8_M8_20: +L(sgemm_tcopy_L8_M8_20): COPY8x8 subs I , I , #1 - bne .Lsgemm_tcopy_L8_M8_20 + bne L(sgemm_tcopy_L8_M8_20) -.Lsgemm_tcopy_L8_M8_40: +L(sgemm_tcopy_L8_M8_40): tst N , #4 - ble .Lsgemm_tcopy_L8_M8_60 + ble L(sgemm_tcopy_L8_M8_60) COPY4x8 -.Lsgemm_tcopy_L8_M8_60: +L(sgemm_tcopy_L8_M8_60): tst N , #2 - ble .Lsgemm_tcopy_L8_M8_80 + ble L(sgemm_tcopy_L8_M8_80) COPY2x8 -.Lsgemm_tcopy_L8_M8_80: +L(sgemm_tcopy_L8_M8_80): tst N, #1 - ble .Lsgemm_tcopy_L8_M8_END + ble L(sgemm_tcopy_L8_M8_END) COPY1x8 -.Lsgemm_tcopy_L8_M8_END: +L(sgemm_tcopy_L8_M8_END): subs J, J, #1 // j-- - bne .Lsgemm_tcopy_L8_M8_BEGIN + bne L(sgemm_tcopy_L8_M8_BEGIN) /*********************************************************************************************/ -.Lsgemm_tcopy_L4_BEGIN: +L(sgemm_tcopy_L4_BEGIN): tst M, #7 - ble .Lsgemm_tcopy_L999 + ble L(sgemm_tcopy_L999) tst M, #4 - ble .Lsgemm_tcopy_L2_BEGIN + ble L(sgemm_tcopy_L2_BEGIN) -.Lsgemm_tcopy_L4_M8_BEGIN: +L(sgemm_tcopy_L4_M8_BEGIN): mov A01, A add A02, A01, LDA @@ -561,51 +561,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, N, #3 // I = N / 8 cmp I, #0 - ble .Lsgemm_tcopy_L4_M8_40 + ble L(sgemm_tcopy_L4_M8_40) .align 5 -.Lsgemm_tcopy_L4_M8_20: +L(sgemm_tcopy_L4_M8_20): COPY8x4 subs I , I , #1 - bne .Lsgemm_tcopy_L4_M8_20 + bne L(sgemm_tcopy_L4_M8_20) -.Lsgemm_tcopy_L4_M8_40: +L(sgemm_tcopy_L4_M8_40): tst N , #4 - ble .Lsgemm_tcopy_L4_M8_60 + ble L(sgemm_tcopy_L4_M8_60) COPY4x4 -.Lsgemm_tcopy_L4_M8_60: +L(sgemm_tcopy_L4_M8_60): tst N , #2 - ble .Lsgemm_tcopy_L4_M8_80 + ble L(sgemm_tcopy_L4_M8_80) COPY2x4 -.Lsgemm_tcopy_L4_M8_80: +L(sgemm_tcopy_L4_M8_80): tst N , #1 - ble .Lsgemm_tcopy_L4_M8_END + ble L(sgemm_tcopy_L4_M8_END) COPY1x4 -.Lsgemm_tcopy_L4_M8_END: +L(sgemm_tcopy_L4_M8_END): /*********************************************************************************************/ -.Lsgemm_tcopy_L2_BEGIN: +L(sgemm_tcopy_L2_BEGIN): tst M, #3 - ble .Lsgemm_tcopy_L999 + ble L(sgemm_tcopy_L999) tst M, #2 - ble .Lsgemm_tcopy_L1_BEGIN + ble L(sgemm_tcopy_L1_BEGIN) -.Lsgemm_tcopy_L2_M16_BEGIN: +L(sgemm_tcopy_L2_M16_BEGIN): mov A01, A add A02, A01, LDA @@ -616,89 +616,89 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, N, #3 // I = N / 8 cmp I, #0 - ble .Lsgemm_tcopy_L2_M8_40 + ble L(sgemm_tcopy_L2_M8_40) .align 5 -.Lsgemm_tcopy_L2_M8_20: +L(sgemm_tcopy_L2_M8_20): COPY8x2 subs I , I , #1 - bne .Lsgemm_tcopy_L2_M8_20 + bne L(sgemm_tcopy_L2_M8_20) -.Lsgemm_tcopy_L2_M8_40: +L(sgemm_tcopy_L2_M8_40): tst N , #4 - ble .Lsgemm_tcopy_L2_M8_60 + ble L(sgemm_tcopy_L2_M8_60) COPY4x2 -.Lsgemm_tcopy_L2_M8_60: +L(sgemm_tcopy_L2_M8_60): tst N , #2 - ble .Lsgemm_tcopy_L2_M8_80 + ble L(sgemm_tcopy_L2_M8_80) COPY2x2 -.Lsgemm_tcopy_L2_M8_80: +L(sgemm_tcopy_L2_M8_80): tst N , #1 - ble .Lsgemm_tcopy_L2_M8_END + ble L(sgemm_tcopy_L2_M8_END) COPY1x2 -.Lsgemm_tcopy_L2_M8_END: +L(sgemm_tcopy_L2_M8_END): /*********************************************************************************************/ -.Lsgemm_tcopy_L1_BEGIN: +L(sgemm_tcopy_L1_BEGIN): tst M, #1 - ble .Lsgemm_tcopy_L999 + ble L(sgemm_tcopy_L999) -.Lsgemm_tcopy_L1_M16_BEGIN: +L(sgemm_tcopy_L1_M16_BEGIN): mov A01, A // A01 = A mov B00, B asr I, N, #3 // I = M / 8 cmp I, #0 - ble .Lsgemm_tcopy_L1_M8_40 + ble L(sgemm_tcopy_L1_M8_40) .align 5 -.Lsgemm_tcopy_L1_M8_20: +L(sgemm_tcopy_L1_M8_20): COPY8x1 subs I , I , #1 - bne .Lsgemm_tcopy_L1_M8_20 + bne L(sgemm_tcopy_L1_M8_20) -.Lsgemm_tcopy_L1_M8_40: +L(sgemm_tcopy_L1_M8_40): tst N , #4 - ble .Lsgemm_tcopy_L1_M8_60 + ble L(sgemm_tcopy_L1_M8_60) COPY4x1 -.Lsgemm_tcopy_L1_M8_60: +L(sgemm_tcopy_L1_M8_60): tst N , #2 - ble .Lsgemm_tcopy_L1_M8_80 + ble L(sgemm_tcopy_L1_M8_80) COPY2x1 -.Lsgemm_tcopy_L1_M8_80: +L(sgemm_tcopy_L1_M8_80): tst N , #1 - ble .Lsgemm_tcopy_L1_M8_END + ble L(sgemm_tcopy_L1_M8_END) COPY1x1 -.Lsgemm_tcopy_L1_M8_END: +L(sgemm_tcopy_L1_M8_END): -.Lsgemm_tcopy_L999: +L(sgemm_tcopy_L999): mov x0, #0 // set return value RESTORE_REGS diff --git a/kernel/arm64/strmm_kernel_16x4.S b/kernel/arm64/strmm_kernel_16x4.S index a44326aeb2..c81cf018af 100644 --- a/kernel/arm64/strmm_kernel_16x4.S +++ b/kernel/arm64/strmm_kernel_16x4.S @@ -1036,7 +1036,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE -.Lstrmm_kernel_begin: +L(strmm_kernel_begin): .align 5 add sp, sp, #-(11 * 16) @@ -1067,11 +1067,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lstrmm_kernel_L2_BEGIN + ble L(strmm_kernel_L2_BEGIN) /******************************************************************************/ -.Lstrmm_kernel_L4_BEGIN: +L(strmm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -1085,15 +1085,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = start of A array -.Lstrmm_kernel_L4_M16_BEGIN: +L(strmm_kernel_L4_M16_BEGIN): mov counterI, origM asr counterI, counterI, #4 // counterI = counterI / 16 cmp counterI, #0 - ble .Lstrmm_kernel_L4_M8_BEGIN + ble L(strmm_kernel_L4_M8_BEGIN) .align 5 -.Lstrmm_kernel_L4_M16_20: +L(strmm_kernel_L4_M16_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1115,7 +1115,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 cmp counterL , #2 - blt .Lstrmm_kernel_L4_M16_32 + blt L(strmm_kernel_L4_M16_32) KERNEL16x4_I KERNEL16x4_M2 @@ -1127,10 +1127,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x4_M2 subs counterL, counterL, #2 - ble .Lstrmm_kernel_L4_M16_22a + ble L(strmm_kernel_L4_M16_22a) .align 5 -.Lstrmm_kernel_L4_M16_22: +L(strmm_kernel_L4_M16_22): KERNEL16x4_M1 KERNEL16x4_M2 @@ -1142,10 +1142,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x4_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M16_22 + bgt L(strmm_kernel_L4_M16_22) .align 5 -.Lstrmm_kernel_L4_M16_22a: +L(strmm_kernel_L4_M16_22a): KERNEL16x4_M1 KERNEL16x4_M2 @@ -1156,13 +1156,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x4_M1 KERNEL16x4_E - b .Lstrmm_kernel_L4_M16_44 + b L(strmm_kernel_L4_M16_44) .align 5 -.Lstrmm_kernel_L4_M16_32: +L(strmm_kernel_L4_M16_32): tst counterL, #1 - ble .Lstrmm_kernel_L4_M16_40 + ble L(strmm_kernel_L4_M16_40) KERNEL16x4_I KERNEL16x4_M2 @@ -1173,25 +1173,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x4_M1 KERNEL16x4_E - b .Lstrmm_kernel_L4_M16_44 + b L(strmm_kernel_L4_M16_44) -.Lstrmm_kernel_L4_M16_40: +L(strmm_kernel_L4_M16_40): INIT16x4 -.Lstrmm_kernel_L4_M16_44: +L(strmm_kernel_L4_M16_44): ands counterL , tempK, #7 - ble .Lstrmm_kernel_L4_M16_100 + ble L(strmm_kernel_L4_M16_100) .align 5 -.Lstrmm_kernel_L4_M16_46: +L(strmm_kernel_L4_M16_46): KERNEL16x4_SUB subs counterL, counterL, #1 - bne .Lstrmm_kernel_L4_M16_46 + bne L(strmm_kernel_L4_M16_46) -.Lstrmm_kernel_L4_M16_100: +L(strmm_kernel_L4_M16_100): SAVE16x4 @@ -1214,22 +1214,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] -.Lstrmm_kernel_L4_M16_END: +L(strmm_kernel_L4_M16_END): subs counterI, counterI, #1 - bne .Lstrmm_kernel_L4_M16_20 + bne L(strmm_kernel_L4_M16_20) //------------------------------------------------------------------------------ -.Lstrmm_kernel_L4_M8_BEGIN: +L(strmm_kernel_L4_M8_BEGIN): mov counterI, origM tst counterI , #15 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) tst counterI, #8 - ble .Lstrmm_kernel_L4_M4_BEGIN + ble L(strmm_kernel_L4_M4_BEGIN) -.Lstrmm_kernel_L4_M8_20: +L(strmm_kernel_L4_M8_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1251,54 +1251,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lstrmm_kernel_L4_M8_32 + blt L(strmm_kernel_L4_M8_32) KERNEL8x4_I // do one in the K KERNEL8x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lstrmm_kernel_L4_M8_22a + ble L(strmm_kernel_L4_M8_22a) .align 5 -.Lstrmm_kernel_L4_M8_22: +L(strmm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M8_22 + bgt L(strmm_kernel_L4_M8_22) -.Lstrmm_kernel_L4_M8_22a: +L(strmm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_E - b .Lstrmm_kernel_L4_M8_44 + b L(strmm_kernel_L4_M8_44) -.Lstrmm_kernel_L4_M8_32: +L(strmm_kernel_L4_M8_32): tst counterL, #1 - ble .Lstrmm_kernel_L4_M8_40 + ble L(strmm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_E - b .Lstrmm_kernel_L4_M8_44 + b L(strmm_kernel_L4_M8_44) -.Lstrmm_kernel_L4_M8_40: +L(strmm_kernel_L4_M8_40): INIT8x4 -.Lstrmm_kernel_L4_M8_44: +L(strmm_kernel_L4_M8_44): ands counterL , tempK, #1 - ble .Lstrmm_kernel_L4_M8_100 + ble L(strmm_kernel_L4_M8_100) -.Lstrmm_kernel_L4_M8_46: +L(strmm_kernel_L4_M8_46): KERNEL8x4_SUB -.Lstrmm_kernel_L4_M8_100: +L(strmm_kernel_L4_M8_100): SAVE8x4 @@ -1318,20 +1318,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #8 #endif -.Lstrmm_kernel_L4_M8_END: +L(strmm_kernel_L4_M8_END): //------------------------------------------------------------------------------ -.Lstrmm_kernel_L4_M4_BEGIN: +L(strmm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) tst counterI, #4 - ble .Lstrmm_kernel_L4_M2_BEGIN + ble L(strmm_kernel_L4_M2_BEGIN) -.Lstrmm_kernel_L4_M4_20: +L(strmm_kernel_L4_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1351,54 +1351,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lstrmm_kernel_L4_M4_32 + blt L(strmm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lstrmm_kernel_L4_M4_22a + ble L(strmm_kernel_L4_M4_22a) .align 5 -.Lstrmm_kernel_L4_M4_22: +L(strmm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M4_22 + bgt L(strmm_kernel_L4_M4_22) -.Lstrmm_kernel_L4_M4_22a: +L(strmm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lstrmm_kernel_L4_M4_44 + b L(strmm_kernel_L4_M4_44) -.Lstrmm_kernel_L4_M4_32: +L(strmm_kernel_L4_M4_32): tst counterL, #1 - ble .Lstrmm_kernel_L4_M4_40 + ble L(strmm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lstrmm_kernel_L4_M4_44 + b L(strmm_kernel_L4_M4_44) -.Lstrmm_kernel_L4_M4_40: +L(strmm_kernel_L4_M4_40): INIT4x4 -.Lstrmm_kernel_L4_M4_44: +L(strmm_kernel_L4_M4_44): ands counterL , tempK, #1 - ble .Lstrmm_kernel_L4_M4_100 + ble L(strmm_kernel_L4_M4_100) -.Lstrmm_kernel_L4_M4_46: +L(strmm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lstrmm_kernel_L4_M4_100: +L(strmm_kernel_L4_M4_100): SAVE4x4 @@ -1416,20 +1416,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L4_M4_END: +L(strmm_kernel_L4_M4_END): //------------------------------------------------------------------------------ -.Lstrmm_kernel_L4_M2_BEGIN: +L(strmm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L4_M1_BEGIN + ble L(strmm_kernel_L4_M1_BEGIN) -.Lstrmm_kernel_L4_M2_20: +L(strmm_kernel_L4_M2_20): INIT2x4 @@ -1452,9 +1452,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L4_M2_40 + ble L(strmm_kernel_L4_M2_40) -.Lstrmm_kernel_L4_M2_22: +L(strmm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1467,22 +1467,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M2_22 + bgt L(strmm_kernel_L4_M2_22) -.Lstrmm_kernel_L4_M2_40: +L(strmm_kernel_L4_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L4_M2_100 + ble L(strmm_kernel_L4_M2_100) -.Lstrmm_kernel_L4_M2_42: +L(strmm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M2_42 + bgt L(strmm_kernel_L4_M2_42) -.Lstrmm_kernel_L4_M2_100: +L(strmm_kernel_L4_M2_100): SAVE2x4 @@ -1501,15 +1501,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L4_M2_END: +L(strmm_kernel_L4_M2_END): -.Lstrmm_kernel_L4_M1_BEGIN: +L(strmm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) -.Lstrmm_kernel_L4_M1_20: +L(strmm_kernel_L4_M1_20): INIT1x4 @@ -1532,9 +1532,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L4_M1_40 + ble L(strmm_kernel_L4_M1_40) -.Lstrmm_kernel_L4_M1_22: +L(strmm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1546,22 +1546,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M1_22 + bgt L(strmm_kernel_L4_M1_22) -.Lstrmm_kernel_L4_M1_40: +L(strmm_kernel_L4_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L4_M1_100 + ble L(strmm_kernel_L4_M1_100) -.Lstrmm_kernel_L4_M1_42: +L(strmm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M1_42 + bgt L(strmm_kernel_L4_M1_42) -.Lstrmm_kernel_L4_M1_100: +L(strmm_kernel_L4_M1_100): SAVE1x4 @@ -1580,26 +1580,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #1 #endif -.Lstrmm_kernel_L4_END: +L(strmm_kernel_L4_END): add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4 #if !defined(LEFT) add tempOffset, tempOffset, #4 #endif subs counterJ, counterJ , #1 // j-- - bgt .Lstrmm_kernel_L4_BEGIN + bgt L(strmm_kernel_L4_BEGIN) /******************************************************************************/ -.Lstrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(strmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lstrmm_kernel_L999 + ble L(strmm_kernel_L999) tst counterJ , #2 - ble .Lstrmm_kernel_L1_BEGIN + ble L(strmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1610,14 +1610,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = A -.Lstrmm_kernel_L2_M16_BEGIN: +L(strmm_kernel_L2_M16_BEGIN): mov counterI, origM asr counterI, counterI, #4 // counterI = counterI / 16 cmp counterI,#0 - ble .Lstrmm_kernel_L2_M8_BEGIN + ble L(strmm_kernel_L2_M8_BEGIN) -.Lstrmm_kernel_L2_M16_20: +L(strmm_kernel_L2_M16_20): INIT16x2 @@ -1641,10 +1641,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M16_40 + ble L(strmm_kernel_L2_M16_40) .align 5 -.Lstrmm_kernel_L2_M16_22: +L(strmm_kernel_L2_M16_22): KERNEL16x2_SUB KERNEL16x2_SUB KERNEL16x2_SUB @@ -1656,22 +1656,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M16_22 + bgt L(strmm_kernel_L2_M16_22) -.Lstrmm_kernel_L2_M16_40: +L(strmm_kernel_L2_M16_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M16_100 + ble L(strmm_kernel_L2_M16_100) -.Lstrmm_kernel_L2_M16_42: +L(strmm_kernel_L2_M16_42): KERNEL16x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M16_42 + bgt L(strmm_kernel_L2_M16_42) -.Lstrmm_kernel_L2_M16_100: +L(strmm_kernel_L2_M16_100): SAVE16x2 @@ -1691,22 +1691,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #16 #endif -.Lstrmm_kernel_L2_M16_END: +L(strmm_kernel_L2_M16_END): subs counterI, counterI, #1 - bgt .Lstrmm_kernel_L2_M16_20 + bgt L(strmm_kernel_L2_M16_20) //------------------------------------------------------------------------------ -.Lstrmm_kernel_L2_M8_BEGIN: +L(strmm_kernel_L2_M8_BEGIN): mov counterI, origM tst counterI , #15 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) tst counterI, #8 - ble .Lstrmm_kernel_L2_M4_BEGIN + ble L(strmm_kernel_L2_M4_BEGIN) -.Lstrmm_kernel_L2_M8_20: +L(strmm_kernel_L2_M8_20): INIT8x2 @@ -1730,10 +1730,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M8_40 + ble L(strmm_kernel_L2_M8_40) .align 5 -.Lstrmm_kernel_L2_M8_22: +L(strmm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB KERNEL8x2_SUB @@ -1745,22 +1745,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M8_22 + bgt L(strmm_kernel_L2_M8_22) -.Lstrmm_kernel_L2_M8_40: +L(strmm_kernel_L2_M8_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M8_100 + ble L(strmm_kernel_L2_M8_100) -.Lstrmm_kernel_L2_M8_42: +L(strmm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M8_42 + bgt L(strmm_kernel_L2_M8_42) -.Lstrmm_kernel_L2_M8_100: +L(strmm_kernel_L2_M8_100): SAVE8x2 @@ -1780,19 +1780,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #8 #endif -.Lstrmm_kernel_L2_M8_END: +L(strmm_kernel_L2_M8_END): //------------------------------------------------------------------------------ -.Lstrmm_kernel_L2_M4_BEGIN: +L(strmm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) tst counterI, #4 - ble .Lstrmm_kernel_L2_M2_BEGIN + ble L(strmm_kernel_L2_M2_BEGIN) -.Lstrmm_kernel_L2_M4_20: +L(strmm_kernel_L2_M4_20): INIT4x2 @@ -1815,10 +1815,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M4_40 + ble L(strmm_kernel_L2_M4_40) .align 5 -.Lstrmm_kernel_L2_M4_22: +L(strmm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1830,22 +1830,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M4_22 + bgt L(strmm_kernel_L2_M4_22) -.Lstrmm_kernel_L2_M4_40: +L(strmm_kernel_L2_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M4_100 + ble L(strmm_kernel_L2_M4_100) -.Lstrmm_kernel_L2_M4_42: +L(strmm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M4_42 + bgt L(strmm_kernel_L2_M4_42) -.Lstrmm_kernel_L2_M4_100: +L(strmm_kernel_L2_M4_100): SAVE4x2 @@ -1864,21 +1864,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L2_M4_END: +L(strmm_kernel_L2_M4_END): //------------------------------------------------------------------------------ -.Lstrmm_kernel_L2_M2_BEGIN: +L(strmm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L2_M1_BEGIN + ble L(strmm_kernel_L2_M1_BEGIN) -.Lstrmm_kernel_L2_M2_20: +L(strmm_kernel_L2_M2_20): INIT2x2 @@ -1901,9 +1901,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M2_40 + ble L(strmm_kernel_L2_M2_40) -.Lstrmm_kernel_L2_M2_22: +L(strmm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1916,22 +1916,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M2_22 + bgt L(strmm_kernel_L2_M2_22) -.Lstrmm_kernel_L2_M2_40: +L(strmm_kernel_L2_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M2_100 + ble L(strmm_kernel_L2_M2_100) -.Lstrmm_kernel_L2_M2_42: +L(strmm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M2_42 + bgt L(strmm_kernel_L2_M2_42) -.Lstrmm_kernel_L2_M2_100: +L(strmm_kernel_L2_M2_100): SAVE2x2 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) @@ -1950,15 +1950,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L2_M2_END: +L(strmm_kernel_L2_M2_END): -.Lstrmm_kernel_L2_M1_BEGIN: +L(strmm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) -.Lstrmm_kernel_L2_M1_20: +L(strmm_kernel_L2_M1_20): INIT1x2 @@ -1981,9 +1981,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lstrmm_kernel_L2_M1_40 + ble L(strmm_kernel_L2_M1_40) -.Lstrmm_kernel_L2_M1_22: +L(strmm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1995,22 +1995,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M1_22 + bgt L(strmm_kernel_L2_M1_22) -.Lstrmm_kernel_L2_M1_40: +L(strmm_kernel_L2_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M1_100 + ble L(strmm_kernel_L2_M1_100) -.Lstrmm_kernel_L2_M1_42: +L(strmm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M1_42 + bgt L(strmm_kernel_L2_M1_42) -.Lstrmm_kernel_L2_M1_100: +L(strmm_kernel_L2_M1_100): SAVE1x2 @@ -2029,7 +2029,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #1 #endif -.Lstrmm_kernel_L2_END: +L(strmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -2037,11 +2037,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lstrmm_kernel_L1_BEGIN: +L(strmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lstrmm_kernel_L999 // done + ble L(strmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -2052,14 +2052,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = A -.Lstrmm_kernel_L1_M16_BEGIN: +L(strmm_kernel_L1_M16_BEGIN): mov counterI, origM asr counterI, counterI, #4 // counterI = counterI / 16 cmp counterI, #0 - ble .Lstrmm_kernel_L1_M8_BEGIN + ble L(strmm_kernel_L1_M8_BEGIN) -.Lstrmm_kernel_L1_M16_20: +L(strmm_kernel_L1_M16_20): INIT16x1 @@ -2083,10 +2083,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M16_40 + ble L(strmm_kernel_L1_M16_40) .align 5 -.Lstrmm_kernel_L1_M16_22: +L(strmm_kernel_L1_M16_22): KERNEL16x1_SUB KERNEL16x1_SUB KERNEL16x1_SUB @@ -2098,22 +2098,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M16_22 + bgt L(strmm_kernel_L1_M16_22) -.Lstrmm_kernel_L1_M16_40: +L(strmm_kernel_L1_M16_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M16_100 + ble L(strmm_kernel_L1_M16_100) -.Lstrmm_kernel_L1_M16_42: +L(strmm_kernel_L1_M16_42): KERNEL16x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M16_42 + bgt L(strmm_kernel_L1_M16_42) -.Lstrmm_kernel_L1_M16_100: +L(strmm_kernel_L1_M16_100): SAVE16x1 @@ -2133,23 +2133,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #16 #endif -.Lstrmm_kernel_L1_M16_END: +L(strmm_kernel_L1_M16_END): subs counterI, counterI, #1 - bgt .Lstrmm_kernel_L1_M16_20 + bgt L(strmm_kernel_L1_M16_20) //------------------------------------------------------------------------------ -.Lstrmm_kernel_L1_M8_BEGIN: +L(strmm_kernel_L1_M8_BEGIN): mov counterI, origM tst counterI , #15 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) tst counterI, #8 - ble .Lstrmm_kernel_L1_M4_BEGIN + ble L(strmm_kernel_L1_M4_BEGIN) -.Lstrmm_kernel_L1_M8_20: +L(strmm_kernel_L1_M8_20): INIT8x1 @@ -2173,10 +2173,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M8_40 + ble L(strmm_kernel_L1_M8_40) .align 5 -.Lstrmm_kernel_L1_M8_22: +L(strmm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -2188,22 +2188,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M8_22 + bgt L(strmm_kernel_L1_M8_22) -.Lstrmm_kernel_L1_M8_40: +L(strmm_kernel_L1_M8_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M8_100 + ble L(strmm_kernel_L1_M8_100) -.Lstrmm_kernel_L1_M8_42: +L(strmm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M8_42 + bgt L(strmm_kernel_L1_M8_42) -.Lstrmm_kernel_L1_M8_100: +L(strmm_kernel_L1_M8_100): SAVE8x1 @@ -2223,19 +2223,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #8 #endif -.Lstrmm_kernel_L1_M8_END: +L(strmm_kernel_L1_M8_END): //------------------------------------------------------------------------------ -.Lstrmm_kernel_L1_M4_BEGIN: +L(strmm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) tst counterI, #4 - ble .Lstrmm_kernel_L1_M2_BEGIN + ble L(strmm_kernel_L1_M2_BEGIN) -.Lstrmm_kernel_L1_M4_20: +L(strmm_kernel_L1_M4_20): INIT4x1 @@ -2258,10 +2258,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M4_40 + ble L(strmm_kernel_L1_M4_40) .align 5 -.Lstrmm_kernel_L1_M4_22: +L(strmm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -2273,22 +2273,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M4_22 + bgt L(strmm_kernel_L1_M4_22) -.Lstrmm_kernel_L1_M4_40: +L(strmm_kernel_L1_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M4_100 + ble L(strmm_kernel_L1_M4_100) -.Lstrmm_kernel_L1_M4_42: +L(strmm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M4_42 + bgt L(strmm_kernel_L1_M4_42) -.Lstrmm_kernel_L1_M4_100: +L(strmm_kernel_L1_M4_100): SAVE4x1 @@ -2307,20 +2307,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L1_M4_END: +L(strmm_kernel_L1_M4_END): //------------------------------------------------------------------------------ -.Lstrmm_kernel_L1_M2_BEGIN: +L(strmm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L1_M1_BEGIN + ble L(strmm_kernel_L1_M1_BEGIN) -.Lstrmm_kernel_L1_M2_20: +L(strmm_kernel_L1_M2_20): INIT2x1 @@ -2343,9 +2343,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M2_40 + ble L(strmm_kernel_L1_M2_40) -.Lstrmm_kernel_L1_M2_22: +L(strmm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -2358,22 +2358,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M2_22 + bgt L(strmm_kernel_L1_M2_22) -.Lstrmm_kernel_L1_M2_40: +L(strmm_kernel_L1_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M2_100 + ble L(strmm_kernel_L1_M2_100) -.Lstrmm_kernel_L1_M2_42: +L(strmm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M2_42 + bgt L(strmm_kernel_L1_M2_42) -.Lstrmm_kernel_L1_M2_100: +L(strmm_kernel_L1_M2_100): SAVE2x1 @@ -2392,15 +2392,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L1_M2_END: +L(strmm_kernel_L1_M2_END): -.Lstrmm_kernel_L1_M1_BEGIN: +L(strmm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) -.Lstrmm_kernel_L1_M1_20: +L(strmm_kernel_L1_M1_20): INIT1x1 @@ -2423,9 +2423,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M1_40 + ble L(strmm_kernel_L1_M1_40) -.Lstrmm_kernel_L1_M1_22: +L(strmm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -2437,28 +2437,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M1_22 + bgt L(strmm_kernel_L1_M1_22) -.Lstrmm_kernel_L1_M1_40: +L(strmm_kernel_L1_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M1_100 + ble L(strmm_kernel_L1_M1_100) -.Lstrmm_kernel_L1_M1_42: +L(strmm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M1_42 + bgt L(strmm_kernel_L1_M1_42) -.Lstrmm_kernel_L1_M1_100: +L(strmm_kernel_L1_M1_100): SAVE1x1 -.Lstrmm_kernel_L1_END: +L(strmm_kernel_L1_END): -.Lstrmm_kernel_L999: +L(strmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/strmm_kernel_4x4.S b/kernel/arm64/strmm_kernel_4x4.S index 5f7818c40a..d0cd0129fd 100644 --- a/kernel/arm64/strmm_kernel_4x4.S +++ b/kernel/arm64/strmm_kernel_4x4.S @@ -507,7 +507,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE -.Lstrmm_kernel_begin: +L(strmm_kernel_begin): .align 5 add sp, sp, #-(11 * 16) @@ -539,11 +539,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lstrmm_kernel_L2_BEGIN + ble L(strmm_kernel_L2_BEGIN) /******************************************************************************/ -.Lstrmm_kernel_L4_BEGIN: +L(strmm_kernel_L4_BEGIN): mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #2 @@ -553,14 +553,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lstrmm_kernel_L4_M4_BEGIN: +L(strmm_kernel_L4_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lstrmm_kernel_L4_M2_BEGIN + ble L(strmm_kernel_L4_M2_BEGIN) -.Lstrmm_kernel_L4_M4_20: +L(strmm_kernel_L4_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -581,54 +581,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lstrmm_kernel_L4_M4_32 + blt L(strmm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lstrmm_kernel_L4_M4_22a + ble L(strmm_kernel_L4_M4_22a) .align 5 -.Lstrmm_kernel_L4_M4_22: +L(strmm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M4_22 + bgt L(strmm_kernel_L4_M4_22) -.Lstrmm_kernel_L4_M4_22a: +L(strmm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lstrmm_kernel_L4_M4_44 + b L(strmm_kernel_L4_M4_44) -.Lstrmm_kernel_L4_M4_32: +L(strmm_kernel_L4_M4_32): tst counterL, #1 - ble .Lstrmm_kernel_L4_M4_40 + ble L(strmm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lstrmm_kernel_L4_M4_44 + b L(strmm_kernel_L4_M4_44) -.Lstrmm_kernel_L4_M4_40: +L(strmm_kernel_L4_M4_40): INIT4x4 -.Lstrmm_kernel_L4_M4_44: +L(strmm_kernel_L4_M4_44): ands counterL , tempK, #1 - ble .Lstrmm_kernel_L4_M4_100 + ble L(strmm_kernel_L4_M4_100) -.Lstrmm_kernel_L4_M4_46: +L(strmm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lstrmm_kernel_L4_M4_100: +L(strmm_kernel_L4_M4_100): SAVE4x4 @@ -647,20 +647,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L4_M4_END: +L(strmm_kernel_L4_M4_END): subs counterI, counterI, #1 - bne .Lstrmm_kernel_L4_M4_20 + bne L(strmm_kernel_L4_M4_20) -.Lstrmm_kernel_L4_M2_BEGIN: +L(strmm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L4_M1_BEGIN + ble L(strmm_kernel_L4_M1_BEGIN) -.Lstrmm_kernel_L4_M2_20: +L(strmm_kernel_L4_M2_20): INIT2x4 @@ -684,9 +684,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L4_M2_40 + ble L(strmm_kernel_L4_M2_40) -.Lstrmm_kernel_L4_M2_22: +L(strmm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -699,22 +699,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M2_22 + bgt L(strmm_kernel_L4_M2_22) -.Lstrmm_kernel_L4_M2_40: +L(strmm_kernel_L4_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L4_M2_100 + ble L(strmm_kernel_L4_M2_100) -.Lstrmm_kernel_L4_M2_42: +L(strmm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M2_42 + bgt L(strmm_kernel_L4_M2_42) -.Lstrmm_kernel_L4_M2_100: +L(strmm_kernel_L4_M2_100): SAVE2x4 @@ -735,15 +735,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.Lstrmm_kernel_L4_M2_END: +L(strmm_kernel_L4_M2_END): -.Lstrmm_kernel_L4_M1_BEGIN: +L(strmm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) -.Lstrmm_kernel_L4_M1_20: +L(strmm_kernel_L4_M1_20): INIT1x4 @@ -767,9 +767,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L4_M1_40 + ble L(strmm_kernel_L4_M1_40) -.Lstrmm_kernel_L4_M1_22: +L(strmm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -781,22 +781,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M1_22 + bgt L(strmm_kernel_L4_M1_22) -.Lstrmm_kernel_L4_M1_40: +L(strmm_kernel_L4_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L4_M1_100 + ble L(strmm_kernel_L4_M1_100) -.Lstrmm_kernel_L4_M1_42: +L(strmm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M1_42 + bgt L(strmm_kernel_L4_M1_42) -.Lstrmm_kernel_L4_M1_100: +L(strmm_kernel_L4_M1_100): SAVE1x4 @@ -817,7 +817,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.Lstrmm_kernel_L4_END: +L(strmm_kernel_L4_END): add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4 #if !defined(LEFT) @@ -825,19 +825,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Lstrmm_kernel_L4_BEGIN + bgt L(strmm_kernel_L4_BEGIN) /******************************************************************************/ -.Lstrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(strmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lstrmm_kernel_L999 + ble L(strmm_kernel_L999) tst counterJ , #2 - ble .Lstrmm_kernel_L1_BEGIN + ble L(strmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -849,14 +849,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lstrmm_kernel_L2_M4_BEGIN: +L(strmm_kernel_L2_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI,#0 - ble .Lstrmm_kernel_L2_M2_BEGIN + ble L(strmm_kernel_L2_M2_BEGIN) -.Lstrmm_kernel_L2_M4_20: +L(strmm_kernel_L2_M4_20): INIT4x2 @@ -880,10 +880,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M4_40 + ble L(strmm_kernel_L2_M4_40) .align 5 -.Lstrmm_kernel_L2_M4_22: +L(strmm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -895,22 +895,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M4_22 + bgt L(strmm_kernel_L2_M4_22) -.Lstrmm_kernel_L2_M4_40: +L(strmm_kernel_L2_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M4_100 + ble L(strmm_kernel_L2_M4_100) -.Lstrmm_kernel_L2_M4_42: +L(strmm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M4_42 + bgt L(strmm_kernel_L2_M4_42) -.Lstrmm_kernel_L2_M4_100: +L(strmm_kernel_L2_M4_100): SAVE4x2 @@ -930,22 +930,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L2_M4_END: +L(strmm_kernel_L2_M4_END): subs counterI, counterI, #1 - bgt .Lstrmm_kernel_L2_M4_20 + bgt L(strmm_kernel_L2_M4_20) -.Lstrmm_kernel_L2_M2_BEGIN: +L(strmm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L2_M1_BEGIN + ble L(strmm_kernel_L2_M1_BEGIN) -.Lstrmm_kernel_L2_M2_20: +L(strmm_kernel_L2_M2_20): INIT2x2 @@ -969,9 +969,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M2_40 + ble L(strmm_kernel_L2_M2_40) -.Lstrmm_kernel_L2_M2_22: +L(strmm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -984,22 +984,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M2_22 + bgt L(strmm_kernel_L2_M2_22) -.Lstrmm_kernel_L2_M2_40: +L(strmm_kernel_L2_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M2_100 + ble L(strmm_kernel_L2_M2_100) -.Lstrmm_kernel_L2_M2_42: +L(strmm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M2_42 + bgt L(strmm_kernel_L2_M2_42) -.Lstrmm_kernel_L2_M2_100: +L(strmm_kernel_L2_M2_100): SAVE2x2 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) @@ -1018,15 +1018,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L2_M2_END: +L(strmm_kernel_L2_M2_END): -.Lstrmm_kernel_L2_M1_BEGIN: +L(strmm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) -.Lstrmm_kernel_L2_M1_20: +L(strmm_kernel_L2_M1_20): INIT1x2 @@ -1050,9 +1050,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lstrmm_kernel_L2_M1_40 + ble L(strmm_kernel_L2_M1_40) -.Lstrmm_kernel_L2_M1_22: +L(strmm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1064,22 +1064,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M1_22 + bgt L(strmm_kernel_L2_M1_22) -.Lstrmm_kernel_L2_M1_40: +L(strmm_kernel_L2_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M1_100 + ble L(strmm_kernel_L2_M1_100) -.Lstrmm_kernel_L2_M1_42: +L(strmm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M1_42 + bgt L(strmm_kernel_L2_M1_42) -.Lstrmm_kernel_L2_M1_100: +L(strmm_kernel_L2_M1_100): SAVE1x2 @@ -1099,7 +1099,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Lstrmm_kernel_L2_END: +L(strmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -1107,11 +1107,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lstrmm_kernel_L1_BEGIN: +L(strmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lstrmm_kernel_L999 // done + ble L(strmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1123,14 +1123,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lstrmm_kernel_L1_M4_BEGIN: +L(strmm_kernel_L1_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lstrmm_kernel_L1_M2_BEGIN + ble L(strmm_kernel_L1_M2_BEGIN) -.Lstrmm_kernel_L1_M4_20: +L(strmm_kernel_L1_M4_20): INIT4x1 @@ -1154,10 +1154,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M4_40 + ble L(strmm_kernel_L1_M4_40) .align 5 -.Lstrmm_kernel_L1_M4_22: +L(strmm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1169,22 +1169,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M4_22 + bgt L(strmm_kernel_L1_M4_22) -.Lstrmm_kernel_L1_M4_40: +L(strmm_kernel_L1_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M4_100 + ble L(strmm_kernel_L1_M4_100) -.Lstrmm_kernel_L1_M4_42: +L(strmm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M4_42 + bgt L(strmm_kernel_L1_M4_42) -.Lstrmm_kernel_L1_M4_100: +L(strmm_kernel_L1_M4_100): SAVE4x1 @@ -1204,22 +1204,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L1_M4_END: +L(strmm_kernel_L1_M4_END): subs counterI, counterI, #1 - bgt .Lstrmm_kernel_L1_M4_20 + bgt L(strmm_kernel_L1_M4_20) -.Lstrmm_kernel_L1_M2_BEGIN: +L(strmm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L1_M1_BEGIN + ble L(strmm_kernel_L1_M1_BEGIN) -.Lstrmm_kernel_L1_M2_20: +L(strmm_kernel_L1_M2_20): INIT2x1 @@ -1243,9 +1243,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M2_40 + ble L(strmm_kernel_L1_M2_40) -.Lstrmm_kernel_L1_M2_22: +L(strmm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1258,22 +1258,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M2_22 + bgt L(strmm_kernel_L1_M2_22) -.Lstrmm_kernel_L1_M2_40: +L(strmm_kernel_L1_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M2_100 + ble L(strmm_kernel_L1_M2_100) -.Lstrmm_kernel_L1_M2_42: +L(strmm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M2_42 + bgt L(strmm_kernel_L1_M2_42) -.Lstrmm_kernel_L1_M2_100: +L(strmm_kernel_L1_M2_100): SAVE2x1 @@ -1294,15 +1294,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.Lstrmm_kernel_L1_M2_END: +L(strmm_kernel_L1_M2_END): -.Lstrmm_kernel_L1_M1_BEGIN: +L(strmm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) -.Lstrmm_kernel_L1_M1_20: +L(strmm_kernel_L1_M1_20): INIT1x1 @@ -1326,9 +1326,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M1_40 + ble L(strmm_kernel_L1_M1_40) -.Lstrmm_kernel_L1_M1_22: +L(strmm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1340,22 +1340,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M1_22 + bgt L(strmm_kernel_L1_M1_22) -.Lstrmm_kernel_L1_M1_40: +L(strmm_kernel_L1_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M1_100 + ble L(strmm_kernel_L1_M1_100) -.Lstrmm_kernel_L1_M1_42: +L(strmm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M1_42 + bgt L(strmm_kernel_L1_M1_42) -.Lstrmm_kernel_L1_M1_100: +L(strmm_kernel_L1_M1_100): SAVE1x1 @@ -1377,7 +1377,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif #endif -.Lstrmm_kernel_L1_END: +L(strmm_kernel_L1_END): #if 0 #if !defined(LEFT) @@ -1385,7 +1385,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif #endif -.Lstrmm_kernel_L999: +L(strmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/strmm_kernel_8x8.S b/kernel/arm64/strmm_kernel_8x8.S index cd18e6847d..761bea99da 100644 --- a/kernel/arm64/strmm_kernel_8x8.S +++ b/kernel/arm64/strmm_kernel_8x8.S @@ -1257,7 +1257,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE -.Lstrmm_kernel_begin: +L(strmm_kernel_begin): .align 5 add sp, sp, #-(11 * 16) @@ -1288,12 +1288,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Lstrmm_kernel_L4_BEGIN + ble L(strmm_kernel_L4_BEGIN) /******************************************************************************/ /******************************************************************************/ -.Lstrmm_kernel_L8_BEGIN: +L(strmm_kernel_L8_BEGIN): mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #3 @@ -1305,14 +1305,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lstrmm_kernel_L8_M8_BEGIN: +L(strmm_kernel_L8_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lstrmm_kernel_L8_M4_BEGIN + ble L(strmm_kernel_L8_M4_BEGIN) -.Lstrmm_kernel_L8_M8_20: +L(strmm_kernel_L8_M8_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1333,54 +1333,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lstrmm_kernel_L8_M8_32 + blt L(strmm_kernel_L8_M8_32) KERNEL8x8_I // do one in the K KERNEL8x8_M2 // do another in the K subs counterL, counterL, #2 - ble .Lstrmm_kernel_L8_M8_22a + ble L(strmm_kernel_L8_M8_22a) .align 5 -.Lstrmm_kernel_L8_M8_22: +L(strmm_kernel_L8_M8_22): KERNEL8x8_M1 KERNEL8x8_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M8_22 + bgt L(strmm_kernel_L8_M8_22) -.Lstrmm_kernel_L8_M8_22a: +L(strmm_kernel_L8_M8_22a): KERNEL8x8_M1 KERNEL8x8_E - b .Lstrmm_kernel_L8_M8_44 + b L(strmm_kernel_L8_M8_44) -.Lstrmm_kernel_L8_M8_32: +L(strmm_kernel_L8_M8_32): tst counterL, #1 - ble .Lstrmm_kernel_L8_M8_40 + ble L(strmm_kernel_L8_M8_40) KERNEL8x8_I KERNEL8x8_E - b .Lstrmm_kernel_L8_M8_44 + b L(strmm_kernel_L8_M8_44) -.Lstrmm_kernel_L8_M8_40: +L(strmm_kernel_L8_M8_40): INIT8x8 -.Lstrmm_kernel_L8_M8_44: +L(strmm_kernel_L8_M8_44): ands counterL , tempK, #1 - ble .Lstrmm_kernel_L8_M8_100 + ble L(strmm_kernel_L8_M8_100) -.Lstrmm_kernel_L8_M8_46: +L(strmm_kernel_L8_M8_46): KERNEL8x8_SUB -.Lstrmm_kernel_L8_M8_100: +L(strmm_kernel_L8_M8_100): SAVE8x8 @@ -1399,22 +1399,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #8 #endif -.Lstrmm_kernel_L8_M8_END: +L(strmm_kernel_L8_M8_END): subs counterI, counterI, #1 - bne .Lstrmm_kernel_L8_M8_20 + bne L(strmm_kernel_L8_M8_20) /******************************************************************************/ -.Lstrmm_kernel_L8_M4_BEGIN: +L(strmm_kernel_L8_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lstrmm_kernel_L8_END + ble L(strmm_kernel_L8_END) tst counterI, #4 - ble .Lstrmm_kernel_L8_M2_BEGIN + ble L(strmm_kernel_L8_M2_BEGIN) -.Lstrmm_kernel_L8_M4_20: +L(strmm_kernel_L8_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1436,54 +1436,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lstrmm_kernel_L8_M4_32 + blt L(strmm_kernel_L8_M4_32) KERNEL4x8_I // do one in the K KERNEL4x8_M2 // do another in the K subs counterL, counterL, #2 - ble .Lstrmm_kernel_L8_M4_22a + ble L(strmm_kernel_L8_M4_22a) .align 5 -.Lstrmm_kernel_L8_M4_22: +L(strmm_kernel_L8_M4_22): KERNEL4x8_M1 KERNEL4x8_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M4_22 + bgt L(strmm_kernel_L8_M4_22) -.Lstrmm_kernel_L8_M4_22a: +L(strmm_kernel_L8_M4_22a): KERNEL4x8_M1 KERNEL4x8_E - b .Lstrmm_kernel_L8_M4_44 + b L(strmm_kernel_L8_M4_44) -.Lstrmm_kernel_L8_M4_32: +L(strmm_kernel_L8_M4_32): tst counterL, #1 - ble .Lstrmm_kernel_L8_M4_40 + ble L(strmm_kernel_L8_M4_40) KERNEL4x8_I KERNEL4x8_E - b .Lstrmm_kernel_L8_M4_44 + b L(strmm_kernel_L8_M4_44) -.Lstrmm_kernel_L8_M4_40: +L(strmm_kernel_L8_M4_40): INIT4x8 -.Lstrmm_kernel_L8_M4_44: +L(strmm_kernel_L8_M4_44): ands counterL , tempK, #1 - ble .Lstrmm_kernel_L8_M4_100 + ble L(strmm_kernel_L8_M4_100) -.Lstrmm_kernel_L8_M4_46: +L(strmm_kernel_L8_M4_46): KERNEL4x8_SUB -.Lstrmm_kernel_L8_M4_100: +L(strmm_kernel_L8_M4_100): SAVE4x8 @@ -1503,20 +1503,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L8_M4_END: +L(strmm_kernel_L8_M4_END): /******************************************************************************/ -.Lstrmm_kernel_L8_M2_BEGIN: +L(strmm_kernel_L8_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L8_END + ble L(strmm_kernel_L8_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L8_M1_BEGIN + ble L(strmm_kernel_L8_M1_BEGIN) -.Lstrmm_kernel_L8_M2_20: +L(strmm_kernel_L8_M2_20): INIT2x8 @@ -1540,9 +1540,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L8_M2_40 + ble L(strmm_kernel_L8_M2_40) -.Lstrmm_kernel_L8_M2_22: +L(strmm_kernel_L8_M2_22): KERNEL2x8_SUB KERNEL2x8_SUB @@ -1555,22 +1555,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M2_22 + bgt L(strmm_kernel_L8_M2_22) -.Lstrmm_kernel_L8_M2_40: +L(strmm_kernel_L8_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L8_M2_100 + ble L(strmm_kernel_L8_M2_100) -.Lstrmm_kernel_L8_M2_42: +L(strmm_kernel_L8_M2_42): KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M2_42 + bgt L(strmm_kernel_L8_M2_42) -.Lstrmm_kernel_L8_M2_100: +L(strmm_kernel_L8_M2_100): SAVE2x8 @@ -1590,16 +1590,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L8_M2_END: +L(strmm_kernel_L8_M2_END): /******************************************************************************/ -.Lstrmm_kernel_L8_M1_BEGIN: +L(strmm_kernel_L8_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L8_END + ble L(strmm_kernel_L8_END) -.Lstrmm_kernel_L8_M1_20: +L(strmm_kernel_L8_M1_20): INIT1x8 @@ -1623,9 +1623,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L8_M1_40 + ble L(strmm_kernel_L8_M1_40) -.Lstrmm_kernel_L8_M1_22: +L(strmm_kernel_L8_M1_22): KERNEL1x8_SUB KERNEL1x8_SUB KERNEL1x8_SUB @@ -1637,22 +1637,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M1_22 + bgt L(strmm_kernel_L8_M1_22) -.Lstrmm_kernel_L8_M1_40: +L(strmm_kernel_L8_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L8_M1_100 + ble L(strmm_kernel_L8_M1_100) -.Lstrmm_kernel_L8_M1_42: +L(strmm_kernel_L8_M1_42): KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M1_42 + bgt L(strmm_kernel_L8_M1_42) -.Lstrmm_kernel_L8_M1_100: +L(strmm_kernel_L8_M1_100): SAVE1x8 @@ -1672,7 +1672,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Lstrmm_kernel_L8_END: +L(strmm_kernel_L8_END): lsl temp, origK, #5 // B = B + K * 4 * 8 add origPB, origPB, temp @@ -1681,19 +1681,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Lstrmm_kernel_L8_BEGIN + bgt L(strmm_kernel_L8_BEGIN) /******************************************************************************/ /******************************************************************************/ -.Lstrmm_kernel_L4_BEGIN: +L(strmm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #7 - ble .Lstrmm_kernel_L999 + ble L(strmm_kernel_L999) tst counterJ , #4 - ble .Lstrmm_kernel_L2_BEGIN + ble L(strmm_kernel_L2_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1707,14 +1707,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lstrmm_kernel_L4_M8_BEGIN: +L(strmm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lstrmm_kernel_L4_M4_BEGIN + ble L(strmm_kernel_L4_M4_BEGIN) -.Lstrmm_kernel_L4_M8_20: +L(strmm_kernel_L4_M8_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1736,54 +1736,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lstrmm_kernel_L4_M8_32 + blt L(strmm_kernel_L4_M8_32) KERNEL8x4_I // do one in the K KERNEL8x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lstrmm_kernel_L4_M8_22a + ble L(strmm_kernel_L4_M8_22a) .align 5 -.Lstrmm_kernel_L4_M8_22: +L(strmm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M8_22 + bgt L(strmm_kernel_L4_M8_22) -.Lstrmm_kernel_L4_M8_22a: +L(strmm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_E - b .Lstrmm_kernel_L4_M8_44 + b L(strmm_kernel_L4_M8_44) -.Lstrmm_kernel_L4_M8_32: +L(strmm_kernel_L4_M8_32): tst counterL, #1 - ble .Lstrmm_kernel_L4_M8_40 + ble L(strmm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_E - b .Lstrmm_kernel_L4_M8_44 + b L(strmm_kernel_L4_M8_44) -.Lstrmm_kernel_L4_M8_40: +L(strmm_kernel_L4_M8_40): INIT8x4 -.Lstrmm_kernel_L4_M8_44: +L(strmm_kernel_L4_M8_44): ands counterL , tempK, #1 - ble .Lstrmm_kernel_L4_M8_100 + ble L(strmm_kernel_L4_M8_100) -.Lstrmm_kernel_L4_M8_46: +L(strmm_kernel_L4_M8_46): KERNEL8x4_SUB -.Lstrmm_kernel_L4_M8_100: +L(strmm_kernel_L4_M8_100): SAVE8x4 @@ -1802,22 +1802,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #8 #endif -.Lstrmm_kernel_L4_M8_END: +L(strmm_kernel_L4_M8_END): subs counterI, counterI, #1 - bne .Lstrmm_kernel_L4_M8_20 + bne L(strmm_kernel_L4_M8_20) /******************************************************************************/ -.Lstrmm_kernel_L4_M4_BEGIN: +L(strmm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) tst counterI, #4 - ble .Lstrmm_kernel_L4_M2_BEGIN + ble L(strmm_kernel_L4_M2_BEGIN) -.Lstrmm_kernel_L4_M4_20: +L(strmm_kernel_L4_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1837,54 +1837,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lstrmm_kernel_L4_M4_32 + blt L(strmm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lstrmm_kernel_L4_M4_22a + ble L(strmm_kernel_L4_M4_22a) .align 5 -.Lstrmm_kernel_L4_M4_22: +L(strmm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M4_22 + bgt L(strmm_kernel_L4_M4_22) -.Lstrmm_kernel_L4_M4_22a: +L(strmm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lstrmm_kernel_L4_M4_44 + b L(strmm_kernel_L4_M4_44) -.Lstrmm_kernel_L4_M4_32: +L(strmm_kernel_L4_M4_32): tst counterL, #1 - ble .Lstrmm_kernel_L4_M4_40 + ble L(strmm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lstrmm_kernel_L4_M4_44 + b L(strmm_kernel_L4_M4_44) -.Lstrmm_kernel_L4_M4_40: +L(strmm_kernel_L4_M4_40): INIT4x4 -.Lstrmm_kernel_L4_M4_44: +L(strmm_kernel_L4_M4_44): ands counterL , tempK, #1 - ble .Lstrmm_kernel_L4_M4_100 + ble L(strmm_kernel_L4_M4_100) -.Lstrmm_kernel_L4_M4_46: +L(strmm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lstrmm_kernel_L4_M4_100: +L(strmm_kernel_L4_M4_100): SAVE4x4 @@ -1902,20 +1902,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L4_M4_END: +L(strmm_kernel_L4_M4_END): /******************************************************************************/ -.Lstrmm_kernel_L4_M2_BEGIN: +L(strmm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L4_M1_BEGIN + ble L(strmm_kernel_L4_M1_BEGIN) -.Lstrmm_kernel_L4_M2_20: +L(strmm_kernel_L4_M2_20): INIT2x4 @@ -1938,9 +1938,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L4_M2_40 + ble L(strmm_kernel_L4_M2_40) -.Lstrmm_kernel_L4_M2_22: +L(strmm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1953,22 +1953,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M2_22 + bgt L(strmm_kernel_L4_M2_22) -.Lstrmm_kernel_L4_M2_40: +L(strmm_kernel_L4_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L4_M2_100 + ble L(strmm_kernel_L4_M2_100) -.Lstrmm_kernel_L4_M2_42: +L(strmm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M2_42 + bgt L(strmm_kernel_L4_M2_42) -.Lstrmm_kernel_L4_M2_100: +L(strmm_kernel_L4_M2_100): SAVE2x4 @@ -1987,16 +1987,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L4_M2_END: +L(strmm_kernel_L4_M2_END): /******************************************************************************/ -.Lstrmm_kernel_L4_M1_BEGIN: +L(strmm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) -.Lstrmm_kernel_L4_M1_20: +L(strmm_kernel_L4_M1_20): INIT1x4 @@ -2019,9 +2019,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L4_M1_40 + ble L(strmm_kernel_L4_M1_40) -.Lstrmm_kernel_L4_M1_22: +L(strmm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -2033,22 +2033,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M1_22 + bgt L(strmm_kernel_L4_M1_22) -.Lstrmm_kernel_L4_M1_40: +L(strmm_kernel_L4_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L4_M1_100 + ble L(strmm_kernel_L4_M1_100) -.Lstrmm_kernel_L4_M1_42: +L(strmm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M1_42 + bgt L(strmm_kernel_L4_M1_42) -.Lstrmm_kernel_L4_M1_100: +L(strmm_kernel_L4_M1_100): SAVE1x4 @@ -2067,7 +2067,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #1 #endif -.Lstrmm_kernel_L4_END: +L(strmm_kernel_L4_END): add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4 #if !defined(LEFT) add tempOffset, tempOffset, #4 @@ -2076,14 +2076,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ /******************************************************************************/ -.Lstrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(strmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lstrmm_kernel_L999 + ble L(strmm_kernel_L999) tst counterJ , #2 - ble .Lstrmm_kernel_L1_BEGIN + ble L(strmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -2096,14 +2096,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lstrmm_kernel_L2_M8_BEGIN: +L(strmm_kernel_L2_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI,#0 - ble .Lstrmm_kernel_L2_M4_BEGIN + ble L(strmm_kernel_L2_M4_BEGIN) -.Lstrmm_kernel_L2_M8_20: +L(strmm_kernel_L2_M8_20): INIT8x2 @@ -2126,10 +2126,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M8_40 + ble L(strmm_kernel_L2_M8_40) .align 5 -.Lstrmm_kernel_L2_M8_22: +L(strmm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB KERNEL8x2_SUB @@ -2141,22 +2141,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M8_22 + bgt L(strmm_kernel_L2_M8_22) -.Lstrmm_kernel_L2_M8_40: +L(strmm_kernel_L2_M8_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M8_100 + ble L(strmm_kernel_L2_M8_100) -.Lstrmm_kernel_L2_M8_42: +L(strmm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M8_42 + bgt L(strmm_kernel_L2_M8_42) -.Lstrmm_kernel_L2_M8_100: +L(strmm_kernel_L2_M8_100): SAVE8x2 @@ -2175,23 +2175,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #8 #endif -.Lstrmm_kernel_L2_M8_END: +L(strmm_kernel_L2_M8_END): subs counterI, counterI, #1 - bgt .Lstrmm_kernel_L2_M8_20 + bgt L(strmm_kernel_L2_M8_20) /******************************************************************************/ -.Lstrmm_kernel_L2_M4_BEGIN: +L(strmm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) tst counterI, #4 - ble .Lstrmm_kernel_L2_M2_BEGIN + ble L(strmm_kernel_L2_M2_BEGIN) -.Lstrmm_kernel_L2_M4_20: +L(strmm_kernel_L2_M4_20): INIT4x2 @@ -2214,10 +2214,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M4_40 + ble L(strmm_kernel_L2_M4_40) .align 5 -.Lstrmm_kernel_L2_M4_22: +L(strmm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -2229,22 +2229,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M4_22 + bgt L(strmm_kernel_L2_M4_22) -.Lstrmm_kernel_L2_M4_40: +L(strmm_kernel_L2_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M4_100 + ble L(strmm_kernel_L2_M4_100) -.Lstrmm_kernel_L2_M4_42: +L(strmm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M4_42 + bgt L(strmm_kernel_L2_M4_42) -.Lstrmm_kernel_L2_M4_100: +L(strmm_kernel_L2_M4_100): SAVE4x2 @@ -2263,20 +2263,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L2_M4_END: +L(strmm_kernel_L2_M4_END): /******************************************************************************/ -.Lstrmm_kernel_L2_M2_BEGIN: +L(strmm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L2_M1_BEGIN + ble L(strmm_kernel_L2_M1_BEGIN) -.Lstrmm_kernel_L2_M2_20: +L(strmm_kernel_L2_M2_20): INIT2x2 @@ -2299,9 +2299,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M2_40 + ble L(strmm_kernel_L2_M2_40) -.Lstrmm_kernel_L2_M2_22: +L(strmm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -2314,22 +2314,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M2_22 + bgt L(strmm_kernel_L2_M2_22) -.Lstrmm_kernel_L2_M2_40: +L(strmm_kernel_L2_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M2_100 + ble L(strmm_kernel_L2_M2_100) -.Lstrmm_kernel_L2_M2_42: +L(strmm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M2_42 + bgt L(strmm_kernel_L2_M2_42) -.Lstrmm_kernel_L2_M2_100: +L(strmm_kernel_L2_M2_100): SAVE2x2 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) @@ -2348,16 +2348,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L2_M2_END: +L(strmm_kernel_L2_M2_END): /******************************************************************************/ -.Lstrmm_kernel_L2_M1_BEGIN: +L(strmm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) -.Lstrmm_kernel_L2_M1_20: +L(strmm_kernel_L2_M1_20): INIT1x2 @@ -2380,9 +2380,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lstrmm_kernel_L2_M1_40 + ble L(strmm_kernel_L2_M1_40) -.Lstrmm_kernel_L2_M1_22: +L(strmm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -2394,22 +2394,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M1_22 + bgt L(strmm_kernel_L2_M1_22) -.Lstrmm_kernel_L2_M1_40: +L(strmm_kernel_L2_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M1_100 + ble L(strmm_kernel_L2_M1_100) -.Lstrmm_kernel_L2_M1_42: +L(strmm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M1_42 + bgt L(strmm_kernel_L2_M1_42) -.Lstrmm_kernel_L2_M1_100: +L(strmm_kernel_L2_M1_100): SAVE1x2 @@ -2428,7 +2428,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #1 #endif -.Lstrmm_kernel_L2_END: +L(strmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -2437,11 +2437,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ /******************************************************************************/ -.Lstrmm_kernel_L1_BEGIN: +L(strmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lstrmm_kernel_L999 // done + ble L(strmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -2454,14 +2454,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lstrmm_kernel_L1_M8_BEGIN: +L(strmm_kernel_L1_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 cmp counterI, #0 - ble .Lstrmm_kernel_L1_M4_BEGIN + ble L(strmm_kernel_L1_M4_BEGIN) -.Lstrmm_kernel_L1_M8_20: +L(strmm_kernel_L1_M8_20): INIT8x1 @@ -2484,10 +2484,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M8_40 + ble L(strmm_kernel_L1_M8_40) .align 5 -.Lstrmm_kernel_L1_M8_22: +L(strmm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -2499,22 +2499,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M8_22 + bgt L(strmm_kernel_L1_M8_22) -.Lstrmm_kernel_L1_M8_40: +L(strmm_kernel_L1_M8_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M8_100 + ble L(strmm_kernel_L1_M8_100) -.Lstrmm_kernel_L1_M8_42: +L(strmm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M8_42 + bgt L(strmm_kernel_L1_M8_42) -.Lstrmm_kernel_L1_M8_100: +L(strmm_kernel_L1_M8_100): SAVE8x1 @@ -2533,23 +2533,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #8 #endif -.Lstrmm_kernel_L1_M8_END: +L(strmm_kernel_L1_M8_END): subs counterI, counterI, #1 - bgt .Lstrmm_kernel_L1_M8_20 + bgt L(strmm_kernel_L1_M8_20) /******************************************************************************/ -.Lstrmm_kernel_L1_M4_BEGIN: +L(strmm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) tst counterI, #4 - ble .Lstrmm_kernel_L1_M2_BEGIN + ble L(strmm_kernel_L1_M2_BEGIN) -.Lstrmm_kernel_L1_M4_20: +L(strmm_kernel_L1_M4_20): INIT4x1 @@ -2572,10 +2572,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M4_40 + ble L(strmm_kernel_L1_M4_40) .align 5 -.Lstrmm_kernel_L1_M4_22: +L(strmm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -2587,22 +2587,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M4_22 + bgt L(strmm_kernel_L1_M4_22) -.Lstrmm_kernel_L1_M4_40: +L(strmm_kernel_L1_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M4_100 + ble L(strmm_kernel_L1_M4_100) -.Lstrmm_kernel_L1_M4_42: +L(strmm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M4_42 + bgt L(strmm_kernel_L1_M4_42) -.Lstrmm_kernel_L1_M4_100: +L(strmm_kernel_L1_M4_100): SAVE4x1 @@ -2621,20 +2621,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L1_M4_END: +L(strmm_kernel_L1_M4_END): /******************************************************************************/ -.Lstrmm_kernel_L1_M2_BEGIN: +L(strmm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L1_M1_BEGIN + ble L(strmm_kernel_L1_M1_BEGIN) -.Lstrmm_kernel_L1_M2_20: +L(strmm_kernel_L1_M2_20): INIT2x1 @@ -2657,9 +2657,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M2_40 + ble L(strmm_kernel_L1_M2_40) -.Lstrmm_kernel_L1_M2_22: +L(strmm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -2672,22 +2672,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M2_22 + bgt L(strmm_kernel_L1_M2_22) -.Lstrmm_kernel_L1_M2_40: +L(strmm_kernel_L1_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M2_100 + ble L(strmm_kernel_L1_M2_100) -.Lstrmm_kernel_L1_M2_42: +L(strmm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M2_42 + bgt L(strmm_kernel_L1_M2_42) -.Lstrmm_kernel_L1_M2_100: +L(strmm_kernel_L1_M2_100): SAVE2x1 @@ -2706,16 +2706,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L1_M2_END: +L(strmm_kernel_L1_M2_END): /******************************************************************************/ -.Lstrmm_kernel_L1_M1_BEGIN: +L(strmm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) -.Lstrmm_kernel_L1_M1_20: +L(strmm_kernel_L1_M1_20): INIT1x1 @@ -2738,9 +2738,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M1_40 + ble L(strmm_kernel_L1_M1_40) -.Lstrmm_kernel_L1_M1_22: +L(strmm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -2752,30 +2752,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M1_22 + bgt L(strmm_kernel_L1_M1_22) -.Lstrmm_kernel_L1_M1_40: +L(strmm_kernel_L1_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M1_100 + ble L(strmm_kernel_L1_M1_100) -.Lstrmm_kernel_L1_M1_42: +L(strmm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M1_42 + bgt L(strmm_kernel_L1_M1_42) -.Lstrmm_kernel_L1_M1_100: +L(strmm_kernel_L1_M1_100): SAVE1x1 -.Lstrmm_kernel_L1_END: +L(strmm_kernel_L1_END): /******************************************************************************/ -.Lstrmm_kernel_L999: +L(strmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/strmm_kernel_8x8_cortexa53.S b/kernel/arm64/strmm_kernel_8x8_cortexa53.S index 4b84623f32..e65a2b9981 100644 --- a/kernel/arm64/strmm_kernel_8x8_cortexa53.S +++ b/kernel/arm64/strmm_kernel_8x8_cortexa53.S @@ -1258,7 +1258,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE -.Lstrmm_kernel_begin: +L(strmm_kernel_begin): .align 5 add sp, sp, #-(11 * 16) @@ -1289,12 +1289,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Lstrmm_kernel_L4_BEGIN + ble L(strmm_kernel_L4_BEGIN) /******************************************************************************/ /******************************************************************************/ -.Lstrmm_kernel_L8_BEGIN: +L(strmm_kernel_L8_BEGIN): mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #3 @@ -1306,14 +1306,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lstrmm_kernel_L8_M8_BEGIN: +L(strmm_kernel_L8_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lstrmm_kernel_L8_M4_BEGIN + ble L(strmm_kernel_L8_M4_BEGIN) -.Lstrmm_kernel_L8_M8_20: +L(strmm_kernel_L8_M8_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1334,7 +1334,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // L = K / 8 cmp counterL , #2 // is there at least 16 to do? - blt .Lstrmm_kernel_L8_M8_32 + blt L(strmm_kernel_L8_M8_32) KERNEL8x8_I // do one in the K KERNEL8x8_M2 // do another in the K @@ -1346,10 +1346,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x8_M2 subs counterL, counterL, #2 - ble .Lstrmm_kernel_L8_M8_22a + ble L(strmm_kernel_L8_M8_22a) .align 5 -.Lstrmm_kernel_L8_M8_22: +L(strmm_kernel_L8_M8_22): KERNEL8x8_M1 KERNEL8x8_M2 @@ -1361,9 +1361,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x8_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M8_22 + bgt L(strmm_kernel_L8_M8_22) -.Lstrmm_kernel_L8_M8_22a: +L(strmm_kernel_L8_M8_22a): KERNEL8x8_M1 KERNEL8x8_M2 @@ -1374,12 +1374,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x8_M1 KERNEL8x8_E - b .Lstrmm_kernel_L8_M8_44 + b L(strmm_kernel_L8_M8_44) -.Lstrmm_kernel_L8_M8_32: +L(strmm_kernel_L8_M8_32): tst counterL, #1 - ble .Lstrmm_kernel_L8_M8_40 + ble L(strmm_kernel_L8_M8_40) KERNEL8x8_I KERNEL8x8_M2 @@ -1390,25 +1390,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x8_M1 KERNEL8x8_E - b .Lstrmm_kernel_L8_M8_44 + b L(strmm_kernel_L8_M8_44) -.Lstrmm_kernel_L8_M8_40: +L(strmm_kernel_L8_M8_40): INIT8x8 -.Lstrmm_kernel_L8_M8_44: +L(strmm_kernel_L8_M8_44): ands counterL , tempK, #7 - ble .Lstrmm_kernel_L8_M8_100 + ble L(strmm_kernel_L8_M8_100) -.Lstrmm_kernel_L8_M8_46: +L(strmm_kernel_L8_M8_46): KERNEL8x8_SUB subs counterL, counterL, 1 - bgt .Lstrmm_kernel_L8_M8_46 + bgt L(strmm_kernel_L8_M8_46) -.Lstrmm_kernel_L8_M8_100: +L(strmm_kernel_L8_M8_100): SAVE8x8 @@ -1427,22 +1427,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #8 #endif -.Lstrmm_kernel_L8_M8_END: +L(strmm_kernel_L8_M8_END): subs counterI, counterI, #1 - bne .Lstrmm_kernel_L8_M8_20 + bne L(strmm_kernel_L8_M8_20) /******************************************************************************/ -.Lstrmm_kernel_L8_M4_BEGIN: +L(strmm_kernel_L8_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lstrmm_kernel_L8_END + ble L(strmm_kernel_L8_END) tst counterI, #4 - ble .Lstrmm_kernel_L8_M2_BEGIN + ble L(strmm_kernel_L8_M2_BEGIN) -.Lstrmm_kernel_L8_M4_20: +L(strmm_kernel_L8_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1464,54 +1464,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lstrmm_kernel_L8_M4_32 + blt L(strmm_kernel_L8_M4_32) KERNEL4x8_I // do one in the K KERNEL4x8_M2 // do another in the K subs counterL, counterL, #2 - ble .Lstrmm_kernel_L8_M4_22a + ble L(strmm_kernel_L8_M4_22a) .align 5 -.Lstrmm_kernel_L8_M4_22: +L(strmm_kernel_L8_M4_22): KERNEL4x8_M1 KERNEL4x8_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M4_22 + bgt L(strmm_kernel_L8_M4_22) -.Lstrmm_kernel_L8_M4_22a: +L(strmm_kernel_L8_M4_22a): KERNEL4x8_M1 KERNEL4x8_E - b .Lstrmm_kernel_L8_M4_44 + b L(strmm_kernel_L8_M4_44) -.Lstrmm_kernel_L8_M4_32: +L(strmm_kernel_L8_M4_32): tst counterL, #1 - ble .Lstrmm_kernel_L8_M4_40 + ble L(strmm_kernel_L8_M4_40) KERNEL4x8_I KERNEL4x8_E - b .Lstrmm_kernel_L8_M4_44 + b L(strmm_kernel_L8_M4_44) -.Lstrmm_kernel_L8_M4_40: +L(strmm_kernel_L8_M4_40): INIT4x8 -.Lstrmm_kernel_L8_M4_44: +L(strmm_kernel_L8_M4_44): ands counterL , tempK, #1 - ble .Lstrmm_kernel_L8_M4_100 + ble L(strmm_kernel_L8_M4_100) -.Lstrmm_kernel_L8_M4_46: +L(strmm_kernel_L8_M4_46): KERNEL4x8_SUB -.Lstrmm_kernel_L8_M4_100: +L(strmm_kernel_L8_M4_100): SAVE4x8 @@ -1531,20 +1531,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L8_M4_END: +L(strmm_kernel_L8_M4_END): /******************************************************************************/ -.Lstrmm_kernel_L8_M2_BEGIN: +L(strmm_kernel_L8_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L8_END + ble L(strmm_kernel_L8_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L8_M1_BEGIN + ble L(strmm_kernel_L8_M1_BEGIN) -.Lstrmm_kernel_L8_M2_20: +L(strmm_kernel_L8_M2_20): INIT2x8 @@ -1568,9 +1568,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L8_M2_40 + ble L(strmm_kernel_L8_M2_40) -.Lstrmm_kernel_L8_M2_22: +L(strmm_kernel_L8_M2_22): KERNEL2x8_SUB KERNEL2x8_SUB @@ -1583,22 +1583,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M2_22 + bgt L(strmm_kernel_L8_M2_22) -.Lstrmm_kernel_L8_M2_40: +L(strmm_kernel_L8_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L8_M2_100 + ble L(strmm_kernel_L8_M2_100) -.Lstrmm_kernel_L8_M2_42: +L(strmm_kernel_L8_M2_42): KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M2_42 + bgt L(strmm_kernel_L8_M2_42) -.Lstrmm_kernel_L8_M2_100: +L(strmm_kernel_L8_M2_100): SAVE2x8 @@ -1618,16 +1618,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L8_M2_END: +L(strmm_kernel_L8_M2_END): /******************************************************************************/ -.Lstrmm_kernel_L8_M1_BEGIN: +L(strmm_kernel_L8_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L8_END + ble L(strmm_kernel_L8_END) -.Lstrmm_kernel_L8_M1_20: +L(strmm_kernel_L8_M1_20): INIT1x8 @@ -1651,9 +1651,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L8_M1_40 + ble L(strmm_kernel_L8_M1_40) -.Lstrmm_kernel_L8_M1_22: +L(strmm_kernel_L8_M1_22): KERNEL1x8_SUB KERNEL1x8_SUB KERNEL1x8_SUB @@ -1665,22 +1665,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M1_22 + bgt L(strmm_kernel_L8_M1_22) -.Lstrmm_kernel_L8_M1_40: +L(strmm_kernel_L8_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L8_M1_100 + ble L(strmm_kernel_L8_M1_100) -.Lstrmm_kernel_L8_M1_42: +L(strmm_kernel_L8_M1_42): KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M1_42 + bgt L(strmm_kernel_L8_M1_42) -.Lstrmm_kernel_L8_M1_100: +L(strmm_kernel_L8_M1_100): SAVE1x8 @@ -1700,7 +1700,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Lstrmm_kernel_L8_END: +L(strmm_kernel_L8_END): lsl temp, origK, #5 // B = B + K * 4 * 8 add origPB, origPB, temp @@ -1709,19 +1709,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Lstrmm_kernel_L8_BEGIN + bgt L(strmm_kernel_L8_BEGIN) /******************************************************************************/ /******************************************************************************/ -.Lstrmm_kernel_L4_BEGIN: +L(strmm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #7 - ble .Lstrmm_kernel_L999 + ble L(strmm_kernel_L999) tst counterJ , #4 - ble .Lstrmm_kernel_L2_BEGIN + ble L(strmm_kernel_L2_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1735,14 +1735,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lstrmm_kernel_L4_M8_BEGIN: +L(strmm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lstrmm_kernel_L4_M4_BEGIN + ble L(strmm_kernel_L4_M4_BEGIN) -.Lstrmm_kernel_L4_M8_20: +L(strmm_kernel_L4_M8_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1764,54 +1764,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lstrmm_kernel_L4_M8_32 + blt L(strmm_kernel_L4_M8_32) KERNEL8x4_I // do one in the K KERNEL8x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lstrmm_kernel_L4_M8_22a + ble L(strmm_kernel_L4_M8_22a) .align 5 -.Lstrmm_kernel_L4_M8_22: +L(strmm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M8_22 + bgt L(strmm_kernel_L4_M8_22) -.Lstrmm_kernel_L4_M8_22a: +L(strmm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_E - b .Lstrmm_kernel_L4_M8_44 + b L(strmm_kernel_L4_M8_44) -.Lstrmm_kernel_L4_M8_32: +L(strmm_kernel_L4_M8_32): tst counterL, #1 - ble .Lstrmm_kernel_L4_M8_40 + ble L(strmm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_E - b .Lstrmm_kernel_L4_M8_44 + b L(strmm_kernel_L4_M8_44) -.Lstrmm_kernel_L4_M8_40: +L(strmm_kernel_L4_M8_40): INIT8x4 -.Lstrmm_kernel_L4_M8_44: +L(strmm_kernel_L4_M8_44): ands counterL , tempK, #1 - ble .Lstrmm_kernel_L4_M8_100 + ble L(strmm_kernel_L4_M8_100) -.Lstrmm_kernel_L4_M8_46: +L(strmm_kernel_L4_M8_46): KERNEL8x4_SUB -.Lstrmm_kernel_L4_M8_100: +L(strmm_kernel_L4_M8_100): SAVE8x4 @@ -1830,22 +1830,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #8 #endif -.Lstrmm_kernel_L4_M8_END: +L(strmm_kernel_L4_M8_END): subs counterI, counterI, #1 - bne .Lstrmm_kernel_L4_M8_20 + bne L(strmm_kernel_L4_M8_20) /******************************************************************************/ -.Lstrmm_kernel_L4_M4_BEGIN: +L(strmm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) tst counterI, #4 - ble .Lstrmm_kernel_L4_M2_BEGIN + ble L(strmm_kernel_L4_M2_BEGIN) -.Lstrmm_kernel_L4_M4_20: +L(strmm_kernel_L4_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1865,54 +1865,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lstrmm_kernel_L4_M4_32 + blt L(strmm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lstrmm_kernel_L4_M4_22a + ble L(strmm_kernel_L4_M4_22a) .align 5 -.Lstrmm_kernel_L4_M4_22: +L(strmm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M4_22 + bgt L(strmm_kernel_L4_M4_22) -.Lstrmm_kernel_L4_M4_22a: +L(strmm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lstrmm_kernel_L4_M4_44 + b L(strmm_kernel_L4_M4_44) -.Lstrmm_kernel_L4_M4_32: +L(strmm_kernel_L4_M4_32): tst counterL, #1 - ble .Lstrmm_kernel_L4_M4_40 + ble L(strmm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lstrmm_kernel_L4_M4_44 + b L(strmm_kernel_L4_M4_44) -.Lstrmm_kernel_L4_M4_40: +L(strmm_kernel_L4_M4_40): INIT4x4 -.Lstrmm_kernel_L4_M4_44: +L(strmm_kernel_L4_M4_44): ands counterL , tempK, #1 - ble .Lstrmm_kernel_L4_M4_100 + ble L(strmm_kernel_L4_M4_100) -.Lstrmm_kernel_L4_M4_46: +L(strmm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lstrmm_kernel_L4_M4_100: +L(strmm_kernel_L4_M4_100): SAVE4x4 @@ -1930,20 +1930,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L4_M4_END: +L(strmm_kernel_L4_M4_END): /******************************************************************************/ -.Lstrmm_kernel_L4_M2_BEGIN: +L(strmm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L4_M1_BEGIN + ble L(strmm_kernel_L4_M1_BEGIN) -.Lstrmm_kernel_L4_M2_20: +L(strmm_kernel_L4_M2_20): INIT2x4 @@ -1966,9 +1966,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L4_M2_40 + ble L(strmm_kernel_L4_M2_40) -.Lstrmm_kernel_L4_M2_22: +L(strmm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1981,22 +1981,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M2_22 + bgt L(strmm_kernel_L4_M2_22) -.Lstrmm_kernel_L4_M2_40: +L(strmm_kernel_L4_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L4_M2_100 + ble L(strmm_kernel_L4_M2_100) -.Lstrmm_kernel_L4_M2_42: +L(strmm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M2_42 + bgt L(strmm_kernel_L4_M2_42) -.Lstrmm_kernel_L4_M2_100: +L(strmm_kernel_L4_M2_100): SAVE2x4 @@ -2015,16 +2015,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L4_M2_END: +L(strmm_kernel_L4_M2_END): /******************************************************************************/ -.Lstrmm_kernel_L4_M1_BEGIN: +L(strmm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) -.Lstrmm_kernel_L4_M1_20: +L(strmm_kernel_L4_M1_20): INIT1x4 @@ -2047,9 +2047,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L4_M1_40 + ble L(strmm_kernel_L4_M1_40) -.Lstrmm_kernel_L4_M1_22: +L(strmm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -2061,22 +2061,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M1_22 + bgt L(strmm_kernel_L4_M1_22) -.Lstrmm_kernel_L4_M1_40: +L(strmm_kernel_L4_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L4_M1_100 + ble L(strmm_kernel_L4_M1_100) -.Lstrmm_kernel_L4_M1_42: +L(strmm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M1_42 + bgt L(strmm_kernel_L4_M1_42) -.Lstrmm_kernel_L4_M1_100: +L(strmm_kernel_L4_M1_100): SAVE1x4 @@ -2095,7 +2095,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #1 #endif -.Lstrmm_kernel_L4_END: +L(strmm_kernel_L4_END): add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4 #if !defined(LEFT) add tempOffset, tempOffset, #4 @@ -2104,14 +2104,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ /******************************************************************************/ -.Lstrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(strmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lstrmm_kernel_L999 + ble L(strmm_kernel_L999) tst counterJ , #2 - ble .Lstrmm_kernel_L1_BEGIN + ble L(strmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -2124,14 +2124,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lstrmm_kernel_L2_M8_BEGIN: +L(strmm_kernel_L2_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI,#0 - ble .Lstrmm_kernel_L2_M4_BEGIN + ble L(strmm_kernel_L2_M4_BEGIN) -.Lstrmm_kernel_L2_M8_20: +L(strmm_kernel_L2_M8_20): INIT8x2 @@ -2154,10 +2154,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M8_40 + ble L(strmm_kernel_L2_M8_40) .align 5 -.Lstrmm_kernel_L2_M8_22: +L(strmm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB KERNEL8x2_SUB @@ -2169,22 +2169,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M8_22 + bgt L(strmm_kernel_L2_M8_22) -.Lstrmm_kernel_L2_M8_40: +L(strmm_kernel_L2_M8_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M8_100 + ble L(strmm_kernel_L2_M8_100) -.Lstrmm_kernel_L2_M8_42: +L(strmm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M8_42 + bgt L(strmm_kernel_L2_M8_42) -.Lstrmm_kernel_L2_M8_100: +L(strmm_kernel_L2_M8_100): SAVE8x2 @@ -2203,23 +2203,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #8 #endif -.Lstrmm_kernel_L2_M8_END: +L(strmm_kernel_L2_M8_END): subs counterI, counterI, #1 - bgt .Lstrmm_kernel_L2_M8_20 + bgt L(strmm_kernel_L2_M8_20) /******************************************************************************/ -.Lstrmm_kernel_L2_M4_BEGIN: +L(strmm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) tst counterI, #4 - ble .Lstrmm_kernel_L2_M2_BEGIN + ble L(strmm_kernel_L2_M2_BEGIN) -.Lstrmm_kernel_L2_M4_20: +L(strmm_kernel_L2_M4_20): INIT4x2 @@ -2242,10 +2242,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M4_40 + ble L(strmm_kernel_L2_M4_40) .align 5 -.Lstrmm_kernel_L2_M4_22: +L(strmm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -2257,22 +2257,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M4_22 + bgt L(strmm_kernel_L2_M4_22) -.Lstrmm_kernel_L2_M4_40: +L(strmm_kernel_L2_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M4_100 + ble L(strmm_kernel_L2_M4_100) -.Lstrmm_kernel_L2_M4_42: +L(strmm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M4_42 + bgt L(strmm_kernel_L2_M4_42) -.Lstrmm_kernel_L2_M4_100: +L(strmm_kernel_L2_M4_100): SAVE4x2 @@ -2291,20 +2291,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L2_M4_END: +L(strmm_kernel_L2_M4_END): /******************************************************************************/ -.Lstrmm_kernel_L2_M2_BEGIN: +L(strmm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L2_M1_BEGIN + ble L(strmm_kernel_L2_M1_BEGIN) -.Lstrmm_kernel_L2_M2_20: +L(strmm_kernel_L2_M2_20): INIT2x2 @@ -2327,9 +2327,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M2_40 + ble L(strmm_kernel_L2_M2_40) -.Lstrmm_kernel_L2_M2_22: +L(strmm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -2342,22 +2342,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M2_22 + bgt L(strmm_kernel_L2_M2_22) -.Lstrmm_kernel_L2_M2_40: +L(strmm_kernel_L2_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M2_100 + ble L(strmm_kernel_L2_M2_100) -.Lstrmm_kernel_L2_M2_42: +L(strmm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M2_42 + bgt L(strmm_kernel_L2_M2_42) -.Lstrmm_kernel_L2_M2_100: +L(strmm_kernel_L2_M2_100): SAVE2x2 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) @@ -2376,16 +2376,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L2_M2_END: +L(strmm_kernel_L2_M2_END): /******************************************************************************/ -.Lstrmm_kernel_L2_M1_BEGIN: +L(strmm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) -.Lstrmm_kernel_L2_M1_20: +L(strmm_kernel_L2_M1_20): INIT1x2 @@ -2408,9 +2408,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lstrmm_kernel_L2_M1_40 + ble L(strmm_kernel_L2_M1_40) -.Lstrmm_kernel_L2_M1_22: +L(strmm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -2422,22 +2422,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M1_22 + bgt L(strmm_kernel_L2_M1_22) -.Lstrmm_kernel_L2_M1_40: +L(strmm_kernel_L2_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M1_100 + ble L(strmm_kernel_L2_M1_100) -.Lstrmm_kernel_L2_M1_42: +L(strmm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M1_42 + bgt L(strmm_kernel_L2_M1_42) -.Lstrmm_kernel_L2_M1_100: +L(strmm_kernel_L2_M1_100): SAVE1x2 @@ -2456,7 +2456,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #1 #endif -.Lstrmm_kernel_L2_END: +L(strmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -2465,11 +2465,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ /******************************************************************************/ -.Lstrmm_kernel_L1_BEGIN: +L(strmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lstrmm_kernel_L999 // done + ble L(strmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -2482,14 +2482,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lstrmm_kernel_L1_M8_BEGIN: +L(strmm_kernel_L1_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 cmp counterI, #0 - ble .Lstrmm_kernel_L1_M4_BEGIN + ble L(strmm_kernel_L1_M4_BEGIN) -.Lstrmm_kernel_L1_M8_20: +L(strmm_kernel_L1_M8_20): INIT8x1 @@ -2512,10 +2512,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M8_40 + ble L(strmm_kernel_L1_M8_40) .align 5 -.Lstrmm_kernel_L1_M8_22: +L(strmm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -2527,22 +2527,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M8_22 + bgt L(strmm_kernel_L1_M8_22) -.Lstrmm_kernel_L1_M8_40: +L(strmm_kernel_L1_M8_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M8_100 + ble L(strmm_kernel_L1_M8_100) -.Lstrmm_kernel_L1_M8_42: +L(strmm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M8_42 + bgt L(strmm_kernel_L1_M8_42) -.Lstrmm_kernel_L1_M8_100: +L(strmm_kernel_L1_M8_100): SAVE8x1 @@ -2561,23 +2561,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #8 #endif -.Lstrmm_kernel_L1_M8_END: +L(strmm_kernel_L1_M8_END): subs counterI, counterI, #1 - bgt .Lstrmm_kernel_L1_M8_20 + bgt L(strmm_kernel_L1_M8_20) /******************************************************************************/ -.Lstrmm_kernel_L1_M4_BEGIN: +L(strmm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) tst counterI, #4 - ble .Lstrmm_kernel_L1_M2_BEGIN + ble L(strmm_kernel_L1_M2_BEGIN) -.Lstrmm_kernel_L1_M4_20: +L(strmm_kernel_L1_M4_20): INIT4x1 @@ -2600,10 +2600,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M4_40 + ble L(strmm_kernel_L1_M4_40) .align 5 -.Lstrmm_kernel_L1_M4_22: +L(strmm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -2615,22 +2615,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M4_22 + bgt L(strmm_kernel_L1_M4_22) -.Lstrmm_kernel_L1_M4_40: +L(strmm_kernel_L1_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M4_100 + ble L(strmm_kernel_L1_M4_100) -.Lstrmm_kernel_L1_M4_42: +L(strmm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M4_42 + bgt L(strmm_kernel_L1_M4_42) -.Lstrmm_kernel_L1_M4_100: +L(strmm_kernel_L1_M4_100): SAVE4x1 @@ -2649,20 +2649,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L1_M4_END: +L(strmm_kernel_L1_M4_END): /******************************************************************************/ -.Lstrmm_kernel_L1_M2_BEGIN: +L(strmm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L1_M1_BEGIN + ble L(strmm_kernel_L1_M1_BEGIN) -.Lstrmm_kernel_L1_M2_20: +L(strmm_kernel_L1_M2_20): INIT2x1 @@ -2685,9 +2685,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M2_40 + ble L(strmm_kernel_L1_M2_40) -.Lstrmm_kernel_L1_M2_22: +L(strmm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -2700,22 +2700,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M2_22 + bgt L(strmm_kernel_L1_M2_22) -.Lstrmm_kernel_L1_M2_40: +L(strmm_kernel_L1_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M2_100 + ble L(strmm_kernel_L1_M2_100) -.Lstrmm_kernel_L1_M2_42: +L(strmm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M2_42 + bgt L(strmm_kernel_L1_M2_42) -.Lstrmm_kernel_L1_M2_100: +L(strmm_kernel_L1_M2_100): SAVE2x1 @@ -2734,16 +2734,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L1_M2_END: +L(strmm_kernel_L1_M2_END): /******************************************************************************/ -.Lstrmm_kernel_L1_M1_BEGIN: +L(strmm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) -.Lstrmm_kernel_L1_M1_20: +L(strmm_kernel_L1_M1_20): INIT1x1 @@ -2766,9 +2766,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M1_40 + ble L(strmm_kernel_L1_M1_40) -.Lstrmm_kernel_L1_M1_22: +L(strmm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -2780,30 +2780,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M1_22 + bgt L(strmm_kernel_L1_M1_22) -.Lstrmm_kernel_L1_M1_40: +L(strmm_kernel_L1_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M1_100 + ble L(strmm_kernel_L1_M1_100) -.Lstrmm_kernel_L1_M1_42: +L(strmm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M1_42 + bgt L(strmm_kernel_L1_M1_42) -.Lstrmm_kernel_L1_M1_100: +L(strmm_kernel_L1_M1_100): SAVE1x1 -.Lstrmm_kernel_L1_END: +L(strmm_kernel_L1_END): /******************************************************************************/ -.Lstrmm_kernel_L999: +L(strmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/strmm_kernel_sve_v1x8.S b/kernel/arm64/strmm_kernel_sve_v1x8.S index 3c45e3e298..106b6d2051 100644 --- a/kernel/arm64/strmm_kernel_sve_v1x8.S +++ b/kernel/arm64/strmm_kernel_sve_v1x8.S @@ -472,13 +472,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Lstrmm_kernel_L4_BEGIN + ble L(strmm_kernel_L4_BEGIN) /******************************************************************************/ /* Repeat this as long as there are 8 left in N */ .align 5 -.Lstrmm_kernel_L8_BEGIN: +L(strmm_kernel_L8_BEGIN): mov pCRow0, pC add pC, pC, LDC, lsl #3 // add 8 x LDC @@ -489,7 +489,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lstrmm_kernel_L8_Mv1_BEGIN: +L(strmm_kernel_L8_Mv1_BEGIN): /* Loop over M is done in an SVE fashion. This has the benefit of the last M%SVE_LEN iterations being done in a single sweep */ mov counterI, #0 @@ -497,7 +497,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension .align 5 -.Lstrmm_kernel_L8_Mv1_20: +L(strmm_kernel_L8_Mv1_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -521,7 +521,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Lstrmm_kernel_L8_Mv1_32 + blt L(strmm_kernel_L8_Mv1_32) KERNELv1x8_I KERNELv1x8_M2 @@ -533,10 +533,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lstrmm_kernel_L8_Mv1_22a + ble L(strmm_kernel_L8_Mv1_22a) .align 5 -.Lstrmm_kernel_L8_Mv1_22: +L(strmm_kernel_L8_Mv1_22): KERNELv1x8_M1 KERNELv1x8_M2 @@ -548,10 +548,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_Mv1_22 + bgt L(strmm_kernel_L8_Mv1_22) .align 5 -.Lstrmm_kernel_L8_Mv1_22a: +L(strmm_kernel_L8_Mv1_22a): KERNELv1x8_M1 KERNELv1x8_M2 @@ -562,13 +562,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M1 KERNELv1x8_E - b .Lstrmm_kernel_L8_Mv1_44 + b L(strmm_kernel_L8_Mv1_44) .align 5 -.Lstrmm_kernel_L8_Mv1_32: +L(strmm_kernel_L8_Mv1_32): tst counterL, #1 - ble .Lstrmm_kernel_L8_Mv1_40 + ble L(strmm_kernel_L8_Mv1_40) KERNELv1x8_I KERNELv1x8_M2 @@ -580,26 +580,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_E - b .Lstrmm_kernel_L8_Mv1_44 + b L(strmm_kernel_L8_Mv1_44) -.Lstrmm_kernel_L8_Mv1_40: +L(strmm_kernel_L8_Mv1_40): INITv1x8 -.Lstrmm_kernel_L8_Mv1_44: +L(strmm_kernel_L8_Mv1_44): ands counterL , tempK, #7 - ble .Lstrmm_kernel_L8_Mv1_100 + ble L(strmm_kernel_L8_Mv1_100) .align 5 -.Lstrmm_kernel_L8_Mv1_46: +L(strmm_kernel_L8_Mv1_46): KERNELv1x8_SUB subs counterL, counterL, #1 - bne .Lstrmm_kernel_L8_Mv1_46 + bne L(strmm_kernel_L8_Mv1_46) -.Lstrmm_kernel_L8_Mv1_100: +L(strmm_kernel_L8_Mv1_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] @@ -622,14 +622,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, lanes #endif -.Lstrmm_kernel_L8_Mv1_END: +L(strmm_kernel_L8_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lstrmm_kernel_L8_Mv1_20 + b.any L(strmm_kernel_L8_Mv1_20) -.Lstrmm_kernel_L8_END: +L(strmm_kernel_L8_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 8 * 4 @@ -639,17 +639,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Lstrmm_kernel_L8_BEGIN + bgt L(strmm_kernel_L8_BEGIN) /******************************************************************************/ /* Repeat the same thing if 4 left in N */ .align 5 -.Lstrmm_kernel_L4_BEGIN: +L(strmm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #4 - ble .Lstrmm_kernel_L2_BEGIN + ble L(strmm_kernel_L2_BEGIN) #if defined(LEFT) mov tempOffset, offset @@ -661,14 +661,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lstrmm_kernel_L4_Mv1_BEGIN: +L(strmm_kernel_L4_Mv1_BEGIN): mov counterI, #0 whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s .align 5 -.Lstrmm_kernel_L4_Mv1_20: +L(strmm_kernel_L4_Mv1_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -692,10 +692,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Lstrmm_kernel_L4_Mv1_44 + ble L(strmm_kernel_L4_Mv1_44) .align 5 -.Lstrmm_kernel_L4_Mv1_22: +L(strmm_kernel_L4_Mv1_22): KERNELv1x4_SUB KERNELv1x4_SUB @@ -707,22 +707,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_Mv1_22 + bgt L(strmm_kernel_L4_Mv1_22) -.Lstrmm_kernel_L4_Mv1_44: +L(strmm_kernel_L4_Mv1_44): ands counterL , tempK, #7 - ble .Lstrmm_kernel_L4_Mv1_100 + ble L(strmm_kernel_L4_Mv1_100) .align 5 -.Lstrmm_kernel_L4_Mv1_46: +L(strmm_kernel_L4_Mv1_46): KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Lstrmm_kernel_L4_Mv1_46 + bne L(strmm_kernel_L4_Mv1_46) -.Lstrmm_kernel_L4_Mv1_100: +L(strmm_kernel_L4_Mv1_100): SAVEv1x4 @@ -742,15 +742,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, lanes #endif -.Lstrmm_kernel_L4_Mv1_END: +L(strmm_kernel_L4_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lstrmm_kernel_L4_Mv1_20 + b.any L(strmm_kernel_L4_Mv1_20) -.Lstrmm_kernel_L4_END: +L(strmm_kernel_L4_END): lsl temp, origK, #4 add origPB, origPB, temp // B = B + K * 4 * 4 #if !defined(LEFT) @@ -761,11 +761,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /* Repeat the same thing if 2 left in N */ .align 5 -.Lstrmm_kernel_L2_BEGIN: +L(strmm_kernel_L2_BEGIN): mov counterJ , origN tst counterJ , #2 - ble .Lstrmm_kernel_L1_BEGIN + ble L(strmm_kernel_L1_BEGIN) mov pCRow0, pC @@ -777,14 +777,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lstrmm_kernel_L2_Mv1_BEGIN: +L(strmm_kernel_L2_Mv1_BEGIN): mov counterI, #0 whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s .align 5 -.Lstrmm_kernel_L2_Mv1_20: +L(strmm_kernel_L2_Mv1_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -808,10 +808,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Lstrmm_kernel_L2_Mv1_44 + ble L(strmm_kernel_L2_Mv1_44) .align 5 -.Lstrmm_kernel_L2_Mv1_22: +L(strmm_kernel_L2_Mv1_22): KERNELv1x2_SUB KERNELv1x2_SUB @@ -823,22 +823,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_Mv1_22 + bgt L(strmm_kernel_L2_Mv1_22) -.Lstrmm_kernel_L2_Mv1_44: +L(strmm_kernel_L2_Mv1_44): ands counterL , tempK, #7 - ble .Lstrmm_kernel_L2_Mv1_100 + ble L(strmm_kernel_L2_Mv1_100) .align 5 -.Lstrmm_kernel_L2_Mv1_46: +L(strmm_kernel_L2_Mv1_46): KERNELv1x2_SUB subs counterL, counterL, #1 - bne .Lstrmm_kernel_L2_Mv1_46 + bne L(strmm_kernel_L2_Mv1_46) -.Lstrmm_kernel_L2_Mv1_100: +L(strmm_kernel_L2_Mv1_100): SAVEv1x2 @@ -859,15 +859,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.Lstrmm_kernel_L2_Mv1_END: +L(strmm_kernel_L2_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lstrmm_kernel_L2_Mv1_20 + b.any L(strmm_kernel_L2_Mv1_20) -.Lstrmm_kernel_L2_END: +L(strmm_kernel_L2_END): add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 #if !defined(LEFT) add tempOffset, tempOffset, #2 @@ -877,11 +877,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /* Repeat the same thing if 1 left in N */ .align 5 -.Lstrmm_kernel_L1_BEGIN: +L(strmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lstrmm_kernel_L999 // done + ble L(strmm_kernel_L999) // done mov pCRow0, pC @@ -893,14 +893,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lstrmm_kernel_L1_Mv1_BEGIN: +L(strmm_kernel_L1_Mv1_BEGIN): mov counterI, #0 whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s .align 5 -.Lstrmm_kernel_L1_Mv1_20: +L(strmm_kernel_L1_Mv1_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -924,10 +924,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Lstrmm_kernel_L1_Mv1_44 + ble L(strmm_kernel_L1_Mv1_44) .align 5 -.Lstrmm_kernel_L1_Mv1_22: +L(strmm_kernel_L1_Mv1_22): KERNELv1x1_SUB KERNELv1x1_SUB @@ -939,22 +939,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_Mv1_22 + bgt L(strmm_kernel_L1_Mv1_22) -.Lstrmm_kernel_L1_Mv1_44: +L(strmm_kernel_L1_Mv1_44): ands counterL , tempK, #7 - ble .Lstrmm_kernel_L1_Mv1_100 + ble L(strmm_kernel_L1_Mv1_100) .align 5 -.Lstrmm_kernel_L1_Mv1_46: +L(strmm_kernel_L1_Mv1_46): KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_Mv1_46 + bgt L(strmm_kernel_L1_Mv1_46) -.Lstrmm_kernel_L1_Mv1_100: +L(strmm_kernel_L1_Mv1_100): SAVEv1x1 @@ -976,19 +976,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lstrmm_kernel_L1_Mv1_END: +L(strmm_kernel_L1_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lstrmm_kernel_L1_Mv1_20 + b.any L(strmm_kernel_L1_Mv1_20) -.Lstrmm_kernel_L1_END: +L(strmm_kernel_L1_END): /******************************************************************************/ -.Lstrmm_kernel_L999: +L(strmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/sum.S b/kernel/arm64/sum.S index 16d0dc4e44..6308b86a15 100644 --- a/kernel/arm64/sum.S +++ b/kernel/arm64/sum.S @@ -114,52 +114,52 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif cmp N, xzr - ble .Lsum_kernel_L999 + ble L(sum_kernel_L999) cmp INC_X, xzr - ble .Lsum_kernel_L999 + ble L(sum_kernel_L999) cmp INC_X, #1 - bne .Lsum_kernel_S_BEGIN + bne L(sum_kernel_S_BEGIN) -.Lsum_kernel_F_BEGIN: +L(sum_kernel_F_BEGIN): asr I, N, #3 cmp I, xzr - beq .Lsum_kernel_F1 + beq L(sum_kernel_F1) -.Lsum_kernel_F8: +L(sum_kernel_F8): KERNEL_F8 subs I, I, #1 - bne .Lsum_kernel_F8 + bne L(sum_kernel_F8) KERNEL_F8_FINALIZE -.Lsum_kernel_F1: +L(sum_kernel_F1): ands I, N, #7 - ble .Lsum_kernel_L999 + ble L(sum_kernel_L999) -.Lsum_kernel_F10: +L(sum_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lsum_kernel_F10 + bne L(sum_kernel_F10) -.Lsum_kernel_L999: +L(sum_kernel_L999): ret -.Lsum_kernel_S_BEGIN: +L(sum_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lsum_kernel_S1 + ble L(sum_kernel_S1) -.Lsum_kernel_S4: +L(sum_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -167,19 +167,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lsum_kernel_S4 + bne L(sum_kernel_S4) -.Lsum_kernel_S1: +L(sum_kernel_S1): ands I, N, #3 - ble .Lsum_kernel_L999 + ble L(sum_kernel_L999) -.Lsum_kernel_S10: +L(sum_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lsum_kernel_S10 + bne L(sum_kernel_S10) ret diff --git a/kernel/arm64/swap.S b/kernel/arm64/swap.S index 184e02e9cf..a4076fd851 100644 --- a/kernel/arm64/swap.S +++ b/kernel/arm64/swap.S @@ -193,50 +193,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Lswap_kernel_L999 + ble L(swap_kernel_L999) cmp INC_X, #1 - bne .Lswap_kernel_S_BEGIN + bne L(swap_kernel_S_BEGIN) cmp INC_Y, #1 - bne .Lswap_kernel_S_BEGIN + bne L(swap_kernel_S_BEGIN) -.Lswap_kernel_F_BEGIN: +L(swap_kernel_F_BEGIN): asr I, N, #3 cmp I, xzr - beq .Lswap_kernel_F1 + beq L(swap_kernel_F1) -.Lswap_kernel_F8: +L(swap_kernel_F8): KERNEL_F8 subs I, I, #1 - bne .Lswap_kernel_F8 + bne L(swap_kernel_F8) -.Lswap_kernel_F1: +L(swap_kernel_F1): ands I, N, #7 - ble .Lswap_kernel_L999 + ble L(swap_kernel_L999) -.Lswap_kernel_F10: +L(swap_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lswap_kernel_F10 + bne L(swap_kernel_F10) - b .Lswap_kernel_L999 + b L(swap_kernel_L999) -.Lswap_kernel_S_BEGIN: +L(swap_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lswap_kernel_S1 + ble L(swap_kernel_S1) -.Lswap_kernel_S4: +L(swap_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -244,21 +244,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lswap_kernel_S4 + bne L(swap_kernel_S4) -.Lswap_kernel_S1: +L(swap_kernel_S1): ands I, N, #3 - ble .Lswap_kernel_L999 + ble L(swap_kernel_L999) -.Lswap_kernel_S10: +L(swap_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lswap_kernel_S10 + bne L(swap_kernel_S10) -.Lswap_kernel_L999: +L(swap_kernel_L999): mov w0, wzr ret diff --git a/kernel/arm64/swap_thunderx2t99.S b/kernel/arm64/swap_thunderx2t99.S index 8b97622f2c..9783196c02 100644 --- a/kernel/arm64/swap_thunderx2t99.S +++ b/kernel/arm64/swap_thunderx2t99.S @@ -104,56 +104,56 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Lswap_kernel_L999 + ble L(swap_kernel_L999) cmp INC_X, #1 - bne .Lswap_kernel_S_BEGIN + bne L(swap_kernel_S_BEGIN) cmp INC_Y, #1 - bne .Lswap_kernel_S_BEGIN + bne L(swap_kernel_S_BEGIN) -.Lswap_kernel_F_BEGIN: +L(swap_kernel_F_BEGIN): INIT asr I, N, #N_DIV_SHIFT cmp I, xzr - beq .Lswap_kernel_F1 + beq L(swap_kernel_F1) .align 5 -.Lswap_kernel_F: +L(swap_kernel_F): KERNEL_F subs I, I, #1 - bne .Lswap_kernel_F + bne L(swap_kernel_F) -.Lswap_kernel_F1: +L(swap_kernel_F1): #if defined(DOUBLE) && defined(COMPLEX) - b .Lswap_kernel_L999 + b L(swap_kernel_L999) #else ands I, N, #N_REM_MASK - ble .Lswap_kernel_L999 + ble L(swap_kernel_L999) #endif -.Lswap_kernel_F10: +L(swap_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lswap_kernel_F10 + bne L(swap_kernel_F10) - b .Lswap_kernel_L999 + b L(swap_kernel_L999) -.Lswap_kernel_S_BEGIN: +L(swap_kernel_S_BEGIN): INIT asr I, N, #2 cmp I, xzr - ble .Lswap_kernel_S1 + ble L(swap_kernel_S1) -.Lswap_kernel_S4: +L(swap_kernel_S4): KERNEL_F1 KERNEL_F1 @@ -161,21 +161,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_F1 subs I, I, #1 - bne .Lswap_kernel_S4 + bne L(swap_kernel_S4) -.Lswap_kernel_S1: +L(swap_kernel_S1): ands I, N, #3 - ble .Lswap_kernel_L999 + ble L(swap_kernel_L999) -.Lswap_kernel_S10: +L(swap_kernel_S10): KERNEL_F1 subs I, I, #1 - bne .Lswap_kernel_S10 + bne L(swap_kernel_S10) -.Lswap_kernel_L999: +L(swap_kernel_L999): mov w0, wzr ret diff --git a/kernel/arm64/zamax.S b/kernel/arm64/zamax.S index 3b49c4fffa..5c7457130b 100644 --- a/kernel/arm64/zamax.S +++ b/kernel/arm64/zamax.S @@ -184,62 +184,62 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Lzamax_kernel_zero + ble L(zamax_kernel_zero) cmp INC_X, xzr - ble .Lzamax_kernel_zero + ble L(zamax_kernel_zero) cmp INC_X, #1 - bne .Lzamax_kernel_S_BEGIN + bne L(zamax_kernel_S_BEGIN) -.Lzamax_kernel_F_BEGIN: +L(zamax_kernel_F_BEGIN): asr I, N, #2 cmp I, xzr - beq .Lzamax_kernel_F1_INIT + beq L(zamax_kernel_F1_INIT) INIT_F4 subs I, I, #1 - beq .Lzamax_kernel_F1 + beq L(zamax_kernel_F1) -.Lzamax_kernel_F4: +L(zamax_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lzamax_kernel_F4 + bne L(zamax_kernel_F4) -.Lzamax_kernel_F1: +L(zamax_kernel_F1): ands I, N, #3 - ble .Lzamax_kernel_L999 + ble L(zamax_kernel_L999) -.Lzamax_kernel_F10: +L(zamax_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lzamax_kernel_F10 + bne L(zamax_kernel_F10) ret -.Lzamax_kernel_F1_INIT: +L(zamax_kernel_F1_INIT): INIT_F1 subs N, N, #1 - b .Lzamax_kernel_F1 + b L(zamax_kernel_F1) -.Lzamax_kernel_S_BEGIN: +L(zamax_kernel_S_BEGIN): INIT_S subs N, N, #1 - ble .Lzamax_kernel_L999 + ble L(zamax_kernel_L999) asr I, N, #2 cmp I, xzr - ble .Lzamax_kernel_S1 + ble L(zamax_kernel_S1) -.Lzamax_kernel_S4: +L(zamax_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -247,25 +247,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lzamax_kernel_S4 + bne L(zamax_kernel_S4) -.Lzamax_kernel_S1: +L(zamax_kernel_S1): ands I, N, #3 - ble .Lzamax_kernel_L999 + ble L(zamax_kernel_L999) -.Lzamax_kernel_S10: +L(zamax_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lzamax_kernel_S10 + bne L(zamax_kernel_S10) -.Lzamax_kernel_L999: +L(zamax_kernel_L999): ret -.Lzamax_kernel_zero: +L(zamax_kernel_zero): fmov MAXF, REG0 ret diff --git a/kernel/arm64/zasum.S b/kernel/arm64/zasum.S index 0d5ec952bb..038dedcc5e 100644 --- a/kernel/arm64/zasum.S +++ b/kernel/arm64/zasum.S @@ -92,52 +92,52 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. fmov SUMF, REG0 cmp N, xzr - ble .Lzasum_kernel_L999 + ble L(zasum_kernel_L999) cmp INC_X, xzr - ble .Lzasum_kernel_L999 + ble L(zasum_kernel_L999) cmp INC_X, #1 - bne .Lzasum_kernel_S_BEGIN + bne L(zasum_kernel_S_BEGIN) -.Lzasum_kernel_F_BEGIN: +L(zasum_kernel_F_BEGIN): asr I, N, #2 cmp I, xzr - beq .Lzasum_kernel_F1 + beq L(zasum_kernel_F1) -.Lzasum_kernel_F4: +L(zasum_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lzasum_kernel_F4 + bne L(zasum_kernel_F4) KERNEL_F4_FINALIZE -.Lzasum_kernel_F1: +L(zasum_kernel_F1): ands I, N, #3 - ble .Lzasum_kernel_L999 + ble L(zasum_kernel_L999) -.Lzasum_kernel_F10: +L(zasum_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lzasum_kernel_F10 + bne L(zasum_kernel_F10) -.Lzasum_kernel_L999: +L(zasum_kernel_L999): ret -.Lzasum_kernel_S_BEGIN: +L(zasum_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lzasum_kernel_S1 + ble L(zasum_kernel_S1) -.Lzasum_kernel_S4: +L(zasum_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -145,19 +145,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lzasum_kernel_S4 + bne L(zasum_kernel_S4) -.Lzasum_kernel_S1: +L(zasum_kernel_S1): ands I, N, #3 - ble .Lzasum_kernel_L999 + ble L(zasum_kernel_L999) -.Lzasum_kernel_S10: +L(zasum_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lzasum_kernel_S10 + bne L(zasum_kernel_S10) ret diff --git a/kernel/arm64/zaxpy.S b/kernel/arm64/zaxpy.S index 46d7b04788..ea29968fd3 100644 --- a/kernel/arm64/zaxpy.S +++ b/kernel/arm64/zaxpy.S @@ -241,62 +241,62 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Lzaxpy_kernel_L999 + ble L(zaxpy_kernel_L999) mov Y_COPY, Y fcmp DA_R, #0.0 - bne .L1 + bne L(1) fcmp DA_I, #0.0 - beq .Lzaxpy_kernel_L999 + beq L(zaxpy_kernel_L999) -.L1: +L(1): INIT cmp INC_X, #1 - bne .Lzaxpy_kernel_S_BEGIN + bne L(zaxpy_kernel_S_BEGIN) cmp INC_Y, #1 - bne .Lzaxpy_kernel_S_BEGIN + bne L(zaxpy_kernel_S_BEGIN) -.Lzaxpy_kernel_F_BEGIN: +L(zaxpy_kernel_F_BEGIN): asr I, N, #2 cmp I, xzr - beq .Lzaxpy_kernel_F1 + beq L(zaxpy_kernel_F1) KERNEL_INIT_F4 -.Lzaxpy_kernel_F4: +L(zaxpy_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lzaxpy_kernel_F4 + bne L(zaxpy_kernel_F4) -.Lzaxpy_kernel_F1: +L(zaxpy_kernel_F1): ands I, N, #3 - ble .Lzaxpy_kernel_L999 + ble L(zaxpy_kernel_L999) -.Lzaxpy_kernel_F10: +L(zaxpy_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lzaxpy_kernel_F10 + bne L(zaxpy_kernel_F10) mov w0, wzr ret -.Lzaxpy_kernel_S_BEGIN: +L(zaxpy_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lzaxpy_kernel_S1 + ble L(zaxpy_kernel_S1) -.Lzaxpy_kernel_S4: +L(zaxpy_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -304,21 +304,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lzaxpy_kernel_S4 + bne L(zaxpy_kernel_S4) -.Lzaxpy_kernel_S1: +L(zaxpy_kernel_S1): ands I, N, #3 - ble .Lzaxpy_kernel_L999 + ble L(zaxpy_kernel_L999) -.Lzaxpy_kernel_S10: +L(zaxpy_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lzaxpy_kernel_S10 + bne L(zaxpy_kernel_S10) -.Lzaxpy_kernel_L999: +L(zaxpy_kernel_L999): mov w0, wzr ret + + EPILOGUE diff --git a/kernel/arm64/zdot.S b/kernel/arm64/zdot.S index 044ace3b82..0e6bc1f608 100644 --- a/kernel/arm64/zdot.S +++ b/kernel/arm64/zdot.S @@ -229,51 +229,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif cmp N, xzr - ble .Lzdot_kernel_L999 + ble L(zdot_kernel_L999) cmp INC_X, #1 - bne .Lzdot_kernel_S_BEGIN + bne L(zdot_kernel_S_BEGIN) cmp INC_Y, #1 - bne .Lzdot_kernel_S_BEGIN + bne L(zdot_kernel_S_BEGIN) -.Lzdot_kernel_F_BEGIN: +L(zdot_kernel_F_BEGIN): asr I, N, #2 cmp I, xzr - beq .Lzdot_kernel_F1 + beq L(zdot_kernel_F1) -.Lzdot_kernel_F4: +L(zdot_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lzdot_kernel_F4 + bne L(zdot_kernel_F4) KERNEL_F4_FINALIZE -.Lzdot_kernel_F1: +L(zdot_kernel_F1): ands I, N, #3 - ble .Lzdot_kernel_L999 + ble L(zdot_kernel_L999) -.Lzdot_kernel_F10: +L(zdot_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lzdot_kernel_F10 + bne L(zdot_kernel_F10) ret -.Lzdot_kernel_S_BEGIN: +L(zdot_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lzdot_kernel_S1 + ble L(zdot_kernel_S1) -.Lzdot_kernel_S4: +L(zdot_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -281,21 +281,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lzdot_kernel_S4 + bne L(zdot_kernel_S4) -.Lzdot_kernel_S1: +L(zdot_kernel_S1): ands I, N, #3 - ble .Lzdot_kernel_L999 + ble L(zdot_kernel_L999) -.Lzdot_kernel_S10: +L(zdot_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lzdot_kernel_S10 + bne L(zdot_kernel_S10) -.Lzdot_kernel_L999: +L(zdot_kernel_L999): ret diff --git a/kernel/arm64/zgemm_kernel_4x4.S b/kernel/arm64/zgemm_kernel_4x4.S index a65c4f581d..d4f2145609 100644 --- a/kernel/arm64/zgemm_kernel_4x4.S +++ b/kernel/arm64/zgemm_kernel_4x4.S @@ -1099,9 +1099,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lzgemm_kernel_L2_BEGIN + ble L(zgemm_kernel_L2_BEGIN) -.Lzgemm_kernel_L4_BEGIN: +L(zgemm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -1111,20 +1111,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lzgemm_kernel_L4_M4_BEGIN: +L(zgemm_kernel_L4_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lzgemm_kernel_L4_M2_BEGIN + ble L(zgemm_kernel_L4_M2_BEGIN) .align 5 -.Lzgemm_kernel_L4_M4_20: +L(zgemm_kernel_L4_M4_20): mov pB, origPB asr counterL , origK, #3 cmp counterL , #2 - blt .Lzgemm_kernel_L4_M4_32 + blt L(zgemm_kernel_L4_M4_32) KERNEL4x4_I KERNEL4x4_M2 @@ -1136,10 +1136,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lzgemm_kernel_L4_M4_22a + ble L(zgemm_kernel_L4_M4_22a) .align 5 -.Lzgemm_kernel_L4_M4_22: +L(zgemm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 @@ -1151,10 +1151,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L4_M4_22 + bgt L(zgemm_kernel_L4_M4_22) .align 5 -.Lzgemm_kernel_L4_M4_22a: +L(zgemm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_M2 @@ -1165,13 +1165,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - b .Lzgemm_kernel_L4_M4_44 + b L(zgemm_kernel_L4_M4_44) .align 5 -.Lzgemm_kernel_L4_M4_32: +L(zgemm_kernel_L4_M4_32): tst counterL, #1 - ble .Lzgemm_kernel_L4_M4_40 + ble L(zgemm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_M2 @@ -1182,55 +1182,55 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - b .Lzgemm_kernel_L4_M4_44 + b L(zgemm_kernel_L4_M4_44) -.Lzgemm_kernel_L4_M4_40: +L(zgemm_kernel_L4_M4_40): INIT4x4 -.Lzgemm_kernel_L4_M4_44: +L(zgemm_kernel_L4_M4_44): ands counterL , origK, #7 - ble .Lzgemm_kernel_L4_M4_100 + ble L(zgemm_kernel_L4_M4_100) .align 5 -.Lzgemm_kernel_L4_M4_46: +L(zgemm_kernel_L4_M4_46): KERNEL4x4_SUB subs counterL, counterL, #1 - bne .Lzgemm_kernel_L4_M4_46 + bne L(zgemm_kernel_L4_M4_46) -.Lzgemm_kernel_L4_M4_100: +L(zgemm_kernel_L4_M4_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVE4x4 -.Lzgemm_kernel_L4_M4_END: +L(zgemm_kernel_L4_M4_END): subs counterI, counterI, #1 - bne .Lzgemm_kernel_L4_M4_20 + bne L(zgemm_kernel_L4_M4_20) -.Lzgemm_kernel_L4_M2_BEGIN: +L(zgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lzgemm_kernel_L4_END + ble L(zgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lzgemm_kernel_L4_M1_BEGIN + ble L(zgemm_kernel_L4_M1_BEGIN) -.Lzgemm_kernel_L4_M2_20: +L(zgemm_kernel_L4_M2_20): INIT2x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lzgemm_kernel_L4_M2_40 + ble L(zgemm_kernel_L4_M2_40) -.Lzgemm_kernel_L4_M2_22: +L(zgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1243,43 +1243,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L4_M2_22 + bgt L(zgemm_kernel_L4_M2_22) -.Lzgemm_kernel_L4_M2_40: +L(zgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L4_M2_100 + ble L(zgemm_kernel_L4_M2_100) -.Lzgemm_kernel_L4_M2_42: +L(zgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L4_M2_42 + bgt L(zgemm_kernel_L4_M2_42) -.Lzgemm_kernel_L4_M2_100: +L(zgemm_kernel_L4_M2_100): SAVE2x4 -.Lzgemm_kernel_L4_M2_END: +L(zgemm_kernel_L4_M2_END): -.Lzgemm_kernel_L4_M1_BEGIN: +L(zgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lzgemm_kernel_L4_END + ble L(zgemm_kernel_L4_END) -.Lzgemm_kernel_L4_M1_20: +L(zgemm_kernel_L4_M1_20): INIT1x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lzgemm_kernel_L4_M1_40 + ble L(zgemm_kernel_L4_M1_40) -.Lzgemm_kernel_L4_M1_22: +L(zgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1291,45 +1291,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L4_M1_22 + bgt L(zgemm_kernel_L4_M1_22) -.Lzgemm_kernel_L4_M1_40: +L(zgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L4_M1_100 + ble L(zgemm_kernel_L4_M1_100) -.Lzgemm_kernel_L4_M1_42: +L(zgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L4_M1_42 + bgt L(zgemm_kernel_L4_M1_42) -.Lzgemm_kernel_L4_M1_100: +L(zgemm_kernel_L4_M1_100): SAVE1x4 -.Lzgemm_kernel_L4_END: +L(zgemm_kernel_L4_END): lsl temp, origK, #6 add origPB, origPB, temp // B = B + K * 4 * 8 * 2 subs counterJ, counterJ , #1 // j-- - bgt .Lzgemm_kernel_L4_BEGIN + bgt L(zgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Lzgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(zgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lzgemm_kernel_L999 + ble L(zgemm_kernel_L999) tst counterJ , #2 - ble .Lzgemm_kernel_L1_BEGIN + ble L(zgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1339,24 +1339,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lzgemm_kernel_L2_M4_BEGIN: +L(zgemm_kernel_L2_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI,#0 - ble .Lzgemm_kernel_L2_M2_BEGIN + ble L(zgemm_kernel_L2_M2_BEGIN) -.Lzgemm_kernel_L2_M4_20: +L(zgemm_kernel_L2_M4_20): INIT4x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lzgemm_kernel_L2_M4_40 + ble L(zgemm_kernel_L2_M4_40) .align 5 -.Lzgemm_kernel_L2_M4_22: +L(zgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1368,50 +1368,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M4_22 + bgt L(zgemm_kernel_L2_M4_22) -.Lzgemm_kernel_L2_M4_40: +L(zgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L2_M4_100 + ble L(zgemm_kernel_L2_M4_100) -.Lzgemm_kernel_L2_M4_42: +L(zgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M4_42 + bgt L(zgemm_kernel_L2_M4_42) -.Lzgemm_kernel_L2_M4_100: +L(zgemm_kernel_L2_M4_100): SAVE4x2 -.Lzgemm_kernel_L2_M4_END: +L(zgemm_kernel_L2_M4_END): subs counterI, counterI, #1 - bgt .Lzgemm_kernel_L2_M4_20 + bgt L(zgemm_kernel_L2_M4_20) -.Lzgemm_kernel_L2_M2_BEGIN: +L(zgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lzgemm_kernel_L2_END + ble L(zgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lzgemm_kernel_L2_M1_BEGIN + ble L(zgemm_kernel_L2_M1_BEGIN) -.Lzgemm_kernel_L2_M2_20: +L(zgemm_kernel_L2_M2_20): INIT2x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lzgemm_kernel_L2_M2_40 + ble L(zgemm_kernel_L2_M2_40) -.Lzgemm_kernel_L2_M2_22: +L(zgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1424,43 +1424,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M2_22 + bgt L(zgemm_kernel_L2_M2_22) -.Lzgemm_kernel_L2_M2_40: +L(zgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L2_M2_100 + ble L(zgemm_kernel_L2_M2_100) -.Lzgemm_kernel_L2_M2_42: +L(zgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M2_42 + bgt L(zgemm_kernel_L2_M2_42) -.Lzgemm_kernel_L2_M2_100: +L(zgemm_kernel_L2_M2_100): SAVE2x2 -.Lzgemm_kernel_L2_M2_END: +L(zgemm_kernel_L2_M2_END): -.Lzgemm_kernel_L2_M1_BEGIN: +L(zgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lzgemm_kernel_L2_END + ble L(zgemm_kernel_L2_END) -.Lzgemm_kernel_L2_M1_20: +L(zgemm_kernel_L2_M1_20): INIT1x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lzgemm_kernel_L2_M1_40 + ble L(zgemm_kernel_L2_M1_40) -.Lzgemm_kernel_L2_M1_22: +L(zgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1472,37 +1472,37 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M1_22 + bgt L(zgemm_kernel_L2_M1_22) -.Lzgemm_kernel_L2_M1_40: +L(zgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L2_M1_100 + ble L(zgemm_kernel_L2_M1_100) -.Lzgemm_kernel_L2_M1_42: +L(zgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M1_42 + bgt L(zgemm_kernel_L2_M1_42) -.Lzgemm_kernel_L2_M1_100: +L(zgemm_kernel_L2_M1_100): SAVE1x2 -.Lzgemm_kernel_L2_END: +L(zgemm_kernel_L2_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 2 * 8 * 2 /******************************************************************************/ -.Lzgemm_kernel_L1_BEGIN: +L(zgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lzgemm_kernel_L999 // done + ble L(zgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1512,24 +1512,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lzgemm_kernel_L1_M4_BEGIN: +L(zgemm_kernel_L1_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lzgemm_kernel_L1_M2_BEGIN + ble L(zgemm_kernel_L1_M2_BEGIN) -.Lzgemm_kernel_L1_M4_20: +L(zgemm_kernel_L1_M4_20): INIT4x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lzgemm_kernel_L1_M4_40 + ble L(zgemm_kernel_L1_M4_40) .align 5 -.Lzgemm_kernel_L1_M4_22: +L(zgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1541,50 +1541,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M4_22 + bgt L(zgemm_kernel_L1_M4_22) -.Lzgemm_kernel_L1_M4_40: +L(zgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L1_M4_100 + ble L(zgemm_kernel_L1_M4_100) -.Lzgemm_kernel_L1_M4_42: +L(zgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M4_42 + bgt L(zgemm_kernel_L1_M4_42) -.Lzgemm_kernel_L1_M4_100: +L(zgemm_kernel_L1_M4_100): SAVE4x1 -.Lzgemm_kernel_L1_M4_END: +L(zgemm_kernel_L1_M4_END): subs counterI, counterI, #1 - bgt .Lzgemm_kernel_L1_M4_20 + bgt L(zgemm_kernel_L1_M4_20) -.Lzgemm_kernel_L1_M2_BEGIN: +L(zgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lzgemm_kernel_L1_END + ble L(zgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lzgemm_kernel_L1_M1_BEGIN + ble L(zgemm_kernel_L1_M1_BEGIN) -.Lzgemm_kernel_L1_M2_20: +L(zgemm_kernel_L1_M2_20): INIT2x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lzgemm_kernel_L1_M2_40 + ble L(zgemm_kernel_L1_M2_40) -.Lzgemm_kernel_L1_M2_22: +L(zgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1597,43 +1597,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M2_22 + bgt L(zgemm_kernel_L1_M2_22) -.Lzgemm_kernel_L1_M2_40: +L(zgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L1_M2_100 + ble L(zgemm_kernel_L1_M2_100) -.Lzgemm_kernel_L1_M2_42: +L(zgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M2_42 + bgt L(zgemm_kernel_L1_M2_42) -.Lzgemm_kernel_L1_M2_100: +L(zgemm_kernel_L1_M2_100): SAVE2x1 -.Lzgemm_kernel_L1_M2_END: +L(zgemm_kernel_L1_M2_END): -.Lzgemm_kernel_L1_M1_BEGIN: +L(zgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lzgemm_kernel_L1_END + ble L(zgemm_kernel_L1_END) -.Lzgemm_kernel_L1_M1_20: +L(zgemm_kernel_L1_M1_20): INIT1x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lzgemm_kernel_L1_M1_40 + ble L(zgemm_kernel_L1_M1_40) -.Lzgemm_kernel_L1_M1_22: +L(zgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1645,30 +1645,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M1_22 + bgt L(zgemm_kernel_L1_M1_22) -.Lzgemm_kernel_L1_M1_40: +L(zgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L1_M1_100 + ble L(zgemm_kernel_L1_M1_100) -.Lzgemm_kernel_L1_M1_42: +L(zgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M1_42 + bgt L(zgemm_kernel_L1_M1_42) -.Lzgemm_kernel_L1_M1_100: +L(zgemm_kernel_L1_M1_100): SAVE1x1 -.Lzgemm_kernel_L1_END: +L(zgemm_kernel_L1_END): -.Lzgemm_kernel_L999: +L(zgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/zgemm_kernel_4x4_thunderx2t99.S b/kernel/arm64/zgemm_kernel_4x4_thunderx2t99.S index 8e6ff655de..6db92d1c6c 100644 --- a/kernel/arm64/zgemm_kernel_4x4_thunderx2t99.S +++ b/kernel/arm64/zgemm_kernel_4x4_thunderx2t99.S @@ -1109,9 +1109,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lzgemm_kernel_L2_BEGIN + ble L(zgemm_kernel_L2_BEGIN) -.Lzgemm_kernel_L4_BEGIN: +L(zgemm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -1121,20 +1121,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lzgemm_kernel_L4_M4_BEGIN: +L(zgemm_kernel_L4_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lzgemm_kernel_L4_M2_BEGIN + ble L(zgemm_kernel_L4_M2_BEGIN) .align 5 -.Lzgemm_kernel_L4_M4_20: +L(zgemm_kernel_L4_M4_20): mov pB, origPB asr counterL , origK, #3 cmp counterL , #2 - blt .Lzgemm_kernel_L4_M4_32 + blt L(zgemm_kernel_L4_M4_32) KERNEL4x4_I KERNEL4x4_M2 @@ -1146,10 +1146,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lzgemm_kernel_L4_M4_22a + ble L(zgemm_kernel_L4_M4_22a) .align 5 -.Lzgemm_kernel_L4_M4_22: +L(zgemm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 @@ -1161,10 +1161,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L4_M4_22 + bgt L(zgemm_kernel_L4_M4_22) .align 5 -.Lzgemm_kernel_L4_M4_22a: +L(zgemm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_M2 @@ -1175,13 +1175,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - b .Lzgemm_kernel_L4_M4_44 + b L(zgemm_kernel_L4_M4_44) .align 5 -.Lzgemm_kernel_L4_M4_32: +L(zgemm_kernel_L4_M4_32): tst counterL, #1 - ble .Lzgemm_kernel_L4_M4_40 + ble L(zgemm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_M2 @@ -1192,55 +1192,55 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - b .Lzgemm_kernel_L4_M4_44 + b L(zgemm_kernel_L4_M4_44) -.Lzgemm_kernel_L4_M4_40: +L(zgemm_kernel_L4_M4_40): INIT4x4 -.Lzgemm_kernel_L4_M4_44: +L(zgemm_kernel_L4_M4_44): ands counterL , origK, #7 - ble .Lzgemm_kernel_L4_M4_100 + ble L(zgemm_kernel_L4_M4_100) .align 5 -.Lzgemm_kernel_L4_M4_46: +L(zgemm_kernel_L4_M4_46): KERNEL4x4_SUB subs counterL, counterL, #1 - bne .Lzgemm_kernel_L4_M4_46 + bne L(zgemm_kernel_L4_M4_46) -.Lzgemm_kernel_L4_M4_100: +L(zgemm_kernel_L4_M4_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVE4x4 -.Lzgemm_kernel_L4_M4_END: +L(zgemm_kernel_L4_M4_END): subs counterI, counterI, #1 - bne .Lzgemm_kernel_L4_M4_20 + bne L(zgemm_kernel_L4_M4_20) -.Lzgemm_kernel_L4_M2_BEGIN: +L(zgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lzgemm_kernel_L4_END + ble L(zgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lzgemm_kernel_L4_M1_BEGIN + ble L(zgemm_kernel_L4_M1_BEGIN) -.Lzgemm_kernel_L4_M2_20: +L(zgemm_kernel_L4_M2_20): INIT2x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lzgemm_kernel_L4_M2_40 + ble L(zgemm_kernel_L4_M2_40) -.Lzgemm_kernel_L4_M2_22: +L(zgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1253,43 +1253,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L4_M2_22 + bgt L(zgemm_kernel_L4_M2_22) -.Lzgemm_kernel_L4_M2_40: +L(zgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L4_M2_100 + ble L(zgemm_kernel_L4_M2_100) -.Lzgemm_kernel_L4_M2_42: +L(zgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L4_M2_42 + bgt L(zgemm_kernel_L4_M2_42) -.Lzgemm_kernel_L4_M2_100: +L(zgemm_kernel_L4_M2_100): SAVE2x4 -.Lzgemm_kernel_L4_M2_END: +L(zgemm_kernel_L4_M2_END): -.Lzgemm_kernel_L4_M1_BEGIN: +L(zgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lzgemm_kernel_L4_END + ble L(zgemm_kernel_L4_END) -.Lzgemm_kernel_L4_M1_20: +L(zgemm_kernel_L4_M1_20): INIT1x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lzgemm_kernel_L4_M1_40 + ble L(zgemm_kernel_L4_M1_40) -.Lzgemm_kernel_L4_M1_22: +L(zgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1301,45 +1301,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L4_M1_22 + bgt L(zgemm_kernel_L4_M1_22) -.Lzgemm_kernel_L4_M1_40: +L(zgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L4_M1_100 + ble L(zgemm_kernel_L4_M1_100) -.Lzgemm_kernel_L4_M1_42: +L(zgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L4_M1_42 + bgt L(zgemm_kernel_L4_M1_42) -.Lzgemm_kernel_L4_M1_100: +L(zgemm_kernel_L4_M1_100): SAVE1x4 -.Lzgemm_kernel_L4_END: +L(zgemm_kernel_L4_END): lsl temp, origK, #6 add origPB, origPB, temp // B = B + K * 4 * 8 * 2 subs counterJ, counterJ , #1 // j-- - bgt .Lzgemm_kernel_L4_BEGIN + bgt L(zgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Lzgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(zgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lzgemm_kernel_L999 + ble L(zgemm_kernel_L999) tst counterJ , #2 - ble .Lzgemm_kernel_L1_BEGIN + ble L(zgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1349,24 +1349,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lzgemm_kernel_L2_M4_BEGIN: +L(zgemm_kernel_L2_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI,#0 - ble .Lzgemm_kernel_L2_M2_BEGIN + ble L(zgemm_kernel_L2_M2_BEGIN) -.Lzgemm_kernel_L2_M4_20: +L(zgemm_kernel_L2_M4_20): INIT4x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lzgemm_kernel_L2_M4_40 + ble L(zgemm_kernel_L2_M4_40) .align 5 -.Lzgemm_kernel_L2_M4_22: +L(zgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1378,50 +1378,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M4_22 + bgt L(zgemm_kernel_L2_M4_22) -.Lzgemm_kernel_L2_M4_40: +L(zgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L2_M4_100 + ble L(zgemm_kernel_L2_M4_100) -.Lzgemm_kernel_L2_M4_42: +L(zgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M4_42 + bgt L(zgemm_kernel_L2_M4_42) -.Lzgemm_kernel_L2_M4_100: +L(zgemm_kernel_L2_M4_100): SAVE4x2 -.Lzgemm_kernel_L2_M4_END: +L(zgemm_kernel_L2_M4_END): subs counterI, counterI, #1 - bgt .Lzgemm_kernel_L2_M4_20 + bgt L(zgemm_kernel_L2_M4_20) -.Lzgemm_kernel_L2_M2_BEGIN: +L(zgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lzgemm_kernel_L2_END + ble L(zgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lzgemm_kernel_L2_M1_BEGIN + ble L(zgemm_kernel_L2_M1_BEGIN) -.Lzgemm_kernel_L2_M2_20: +L(zgemm_kernel_L2_M2_20): INIT2x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lzgemm_kernel_L2_M2_40 + ble L(zgemm_kernel_L2_M2_40) -.Lzgemm_kernel_L2_M2_22: +L(zgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1434,43 +1434,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M2_22 + bgt L(zgemm_kernel_L2_M2_22) -.Lzgemm_kernel_L2_M2_40: +L(zgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L2_M2_100 + ble L(zgemm_kernel_L2_M2_100) -.Lzgemm_kernel_L2_M2_42: +L(zgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M2_42 + bgt L(zgemm_kernel_L2_M2_42) -.Lzgemm_kernel_L2_M2_100: +L(zgemm_kernel_L2_M2_100): SAVE2x2 -.Lzgemm_kernel_L2_M2_END: +L(zgemm_kernel_L2_M2_END): -.Lzgemm_kernel_L2_M1_BEGIN: +L(zgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lzgemm_kernel_L2_END + ble L(zgemm_kernel_L2_END) -.Lzgemm_kernel_L2_M1_20: +L(zgemm_kernel_L2_M1_20): INIT1x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lzgemm_kernel_L2_M1_40 + ble L(zgemm_kernel_L2_M1_40) -.Lzgemm_kernel_L2_M1_22: +L(zgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1482,37 +1482,37 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M1_22 + bgt L(zgemm_kernel_L2_M1_22) -.Lzgemm_kernel_L2_M1_40: +L(zgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L2_M1_100 + ble L(zgemm_kernel_L2_M1_100) -.Lzgemm_kernel_L2_M1_42: +L(zgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M1_42 + bgt L(zgemm_kernel_L2_M1_42) -.Lzgemm_kernel_L2_M1_100: +L(zgemm_kernel_L2_M1_100): SAVE1x2 -.Lzgemm_kernel_L2_END: +L(zgemm_kernel_L2_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 2 * 8 * 2 /******************************************************************************/ -.Lzgemm_kernel_L1_BEGIN: +L(zgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lzgemm_kernel_L999 // done + ble L(zgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1522,24 +1522,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lzgemm_kernel_L1_M4_BEGIN: +L(zgemm_kernel_L1_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lzgemm_kernel_L1_M2_BEGIN + ble L(zgemm_kernel_L1_M2_BEGIN) -.Lzgemm_kernel_L1_M4_20: +L(zgemm_kernel_L1_M4_20): INIT4x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lzgemm_kernel_L1_M4_40 + ble L(zgemm_kernel_L1_M4_40) .align 5 -.Lzgemm_kernel_L1_M4_22: +L(zgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1551,50 +1551,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M4_22 + bgt L(zgemm_kernel_L1_M4_22) -.Lzgemm_kernel_L1_M4_40: +L(zgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L1_M4_100 + ble L(zgemm_kernel_L1_M4_100) -.Lzgemm_kernel_L1_M4_42: +L(zgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M4_42 + bgt L(zgemm_kernel_L1_M4_42) -.Lzgemm_kernel_L1_M4_100: +L(zgemm_kernel_L1_M4_100): SAVE4x1 -.Lzgemm_kernel_L1_M4_END: +L(zgemm_kernel_L1_M4_END): subs counterI, counterI, #1 - bgt .Lzgemm_kernel_L1_M4_20 + bgt L(zgemm_kernel_L1_M4_20) -.Lzgemm_kernel_L1_M2_BEGIN: +L(zgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lzgemm_kernel_L1_END + ble L(zgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lzgemm_kernel_L1_M1_BEGIN + ble L(zgemm_kernel_L1_M1_BEGIN) -.Lzgemm_kernel_L1_M2_20: +L(zgemm_kernel_L1_M2_20): INIT2x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lzgemm_kernel_L1_M2_40 + ble L(zgemm_kernel_L1_M2_40) -.Lzgemm_kernel_L1_M2_22: +L(zgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1607,43 +1607,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M2_22 + bgt L(zgemm_kernel_L1_M2_22) -.Lzgemm_kernel_L1_M2_40: +L(zgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L1_M2_100 + ble L(zgemm_kernel_L1_M2_100) -.Lzgemm_kernel_L1_M2_42: +L(zgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M2_42 + bgt L(zgemm_kernel_L1_M2_42) -.Lzgemm_kernel_L1_M2_100: +L(zgemm_kernel_L1_M2_100): SAVE2x1 -.Lzgemm_kernel_L1_M2_END: +L(zgemm_kernel_L1_M2_END): -.Lzgemm_kernel_L1_M1_BEGIN: +L(zgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lzgemm_kernel_L1_END + ble L(zgemm_kernel_L1_END) -.Lzgemm_kernel_L1_M1_20: +L(zgemm_kernel_L1_M1_20): INIT1x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lzgemm_kernel_L1_M1_40 + ble L(zgemm_kernel_L1_M1_40) -.Lzgemm_kernel_L1_M1_22: +L(zgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1655,30 +1655,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M1_22 + bgt L(zgemm_kernel_L1_M1_22) -.Lzgemm_kernel_L1_M1_40: +L(zgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L1_M1_100 + ble L(zgemm_kernel_L1_M1_100) -.Lzgemm_kernel_L1_M1_42: +L(zgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M1_42 + bgt L(zgemm_kernel_L1_M1_42) -.Lzgemm_kernel_L1_M1_100: +L(zgemm_kernel_L1_M1_100): SAVE1x1 -.Lzgemm_kernel_L1_END: +L(zgemm_kernel_L1_END): -.Lzgemm_kernel_L999: +L(zgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/zgemm_kernel_sve_v1x4.S b/kernel/arm64/zgemm_kernel_sve_v1x4.S index a043948d69..9331f119b8 100644 --- a/kernel/arm64/zgemm_kernel_sve_v1x4.S +++ b/kernel/arm64/zgemm_kernel_sve_v1x4.S @@ -533,10 +533,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lzgemm_kernel_L2_BEGIN + ble L(zgemm_kernel_L2_BEGIN) /******************************************************************************/ -.Lzgemm_kernel_L4_BEGIN: +L(zgemm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -546,7 +546,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lzgemm_kernel_L4_Mv1_BEGIN: +L(zgemm_kernel_L4_Mv1_BEGIN): /* Loop over M is done in an SVE fashion. This has the benefit of the last M%SVE_LEN iterations being done in a single sweep */ mov counterI, #0 @@ -554,14 +554,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension .align 5 -.Lzgemm_kernel_L4_Mv1_20: +L(zgemm_kernel_L4_Mv1_20): mov pB, origPB INITv1x4 // fill with zeros asr counterL , origK, #3 cmp counterL , #2 - blt .Lzgemm_kernel_L4_Mv1_32 + blt L(zgemm_kernel_L4_Mv1_32) KERNELv1x4_I KERNELv1x4_M2 @@ -573,10 +573,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lzgemm_kernel_L4_Mv1_22a + ble L(zgemm_kernel_L4_Mv1_22a) .align 5 -.Lzgemm_kernel_L4_Mv1_22: +L(zgemm_kernel_L4_Mv1_22): KERNELv1x4_M1 KERNELv1x4_M2 @@ -588,10 +588,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M2 subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L4_Mv1_22 + bgt L(zgemm_kernel_L4_Mv1_22) .align 5 -.Lzgemm_kernel_L4_Mv1_22a: +L(zgemm_kernel_L4_Mv1_22a): KERNELv1x4_M1 KERNELv1x4_M2 @@ -602,13 +602,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M1 KERNELv1x4_E - b .Lzgemm_kernel_L4_Mv1_44 + b L(zgemm_kernel_L4_Mv1_44) .align 5 -.Lzgemm_kernel_L4_Mv1_32: +L(zgemm_kernel_L4_Mv1_32): tst counterL, #1 - ble .Lzgemm_kernel_L4_Mv1_40 + ble L(zgemm_kernel_L4_Mv1_40) KERNELv1x4_I KERNELv1x4_M2 @@ -619,56 +619,56 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M1 KERNELv1x4_E - b .Lzgemm_kernel_L4_Mv1_44 + b L(zgemm_kernel_L4_Mv1_44) -.Lzgemm_kernel_L4_Mv1_40: +L(zgemm_kernel_L4_Mv1_40): INITv1x4 -.Lzgemm_kernel_L4_Mv1_44: +L(zgemm_kernel_L4_Mv1_44): ands counterL , origK, #7 - ble .Lzgemm_kernel_L4_Mv1_100 + ble L(zgemm_kernel_L4_Mv1_100) .align 5 -.Lzgemm_kernel_L4_Mv1_46: +L(zgemm_kernel_L4_Mv1_46): KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Lzgemm_kernel_L4_Mv1_46 + bne L(zgemm_kernel_L4_Mv1_46) -.Lzgemm_kernel_L4_Mv1_100: +L(zgemm_kernel_L4_Mv1_100): SAVEv1x4 -.Lzgemm_kernel_L4_Mv1_END: +L(zgemm_kernel_L4_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension - b.any .Lzgemm_kernel_L4_Mv1_20 + b.any L(zgemm_kernel_L4_Mv1_20) -.Lzgemm_kernel_L4_END: +L(zgemm_kernel_L4_END): lsl temp, origK, #6 add origPB, origPB, temp // B = B + K * 4 * 8 * 2 subs counterJ, counterJ , #1 // j-- - bgt .Lzgemm_kernel_L4_BEGIN + bgt L(zgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Lzgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(zgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lzgemm_kernel_L999 + ble L(zgemm_kernel_L999) tst counterJ , #2 - ble .Lzgemm_kernel_L1_BEGIN + ble L(zgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC add pCRow1, pCRow0, LDC @@ -679,24 +679,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lzgemm_kernel_L2_Mv1_BEGIN: +L(zgemm_kernel_L2_Mv1_BEGIN): mov counterI, #0 whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d -.Lzgemm_kernel_L2_Mv1_20: +L(zgemm_kernel_L2_Mv1_20): INITv1x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lzgemm_kernel_L2_Mv1_40 + ble L(zgemm_kernel_L2_Mv1_40) .align 5 -.Lzgemm_kernel_L2_Mv1_22: +L(zgemm_kernel_L2_Mv1_22): KERNELv1x2_SUB KERNELv1x2_SUB KERNELv1x2_SUB @@ -708,45 +708,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_Mv1_22 + bgt L(zgemm_kernel_L2_Mv1_22) -.Lzgemm_kernel_L2_Mv1_40: +L(zgemm_kernel_L2_Mv1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L2_Mv1_100 + ble L(zgemm_kernel_L2_Mv1_100) -.Lzgemm_kernel_L2_Mv1_42: +L(zgemm_kernel_L2_Mv1_42): KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_Mv1_42 + bgt L(zgemm_kernel_L2_Mv1_42) -.Lzgemm_kernel_L2_Mv1_100: +L(zgemm_kernel_L2_Mv1_100): SAVEv1x2 -.Lzgemm_kernel_L2_Mv1_END: +L(zgemm_kernel_L2_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Lzgemm_kernel_L2_Mv1_20 + b.any L(zgemm_kernel_L2_Mv1_20) -.Lzgemm_kernel_L2_END: +L(zgemm_kernel_L2_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 2 * 8 * 2 /******************************************************************************/ -.Lzgemm_kernel_L1_BEGIN: +L(zgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lzgemm_kernel_L999 // done + ble L(zgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -754,24 +754,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lzgemm_kernel_L1_Mv1_BEGIN: +L(zgemm_kernel_L1_Mv1_BEGIN): mov counterI, #0 whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d -.Lzgemm_kernel_L1_Mv1_20: +L(zgemm_kernel_L1_Mv1_20): INITv1x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lzgemm_kernel_L1_Mv1_40 + ble L(zgemm_kernel_L1_Mv1_40) .align 5 -.Lzgemm_kernel_L1_Mv1_22: +L(zgemm_kernel_L1_Mv1_22): KERNELv1x1_SUB KERNELv1x1_SUB KERNELv1x1_SUB @@ -783,37 +783,37 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_Mv1_22 + bgt L(zgemm_kernel_L1_Mv1_22) -.Lzgemm_kernel_L1_Mv1_40: +L(zgemm_kernel_L1_Mv1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L1_Mv1_100 + ble L(zgemm_kernel_L1_Mv1_100) -.Lzgemm_kernel_L1_Mv1_42: +L(zgemm_kernel_L1_Mv1_42): KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_Mv1_42 + bgt L(zgemm_kernel_L1_Mv1_42) -.Lzgemm_kernel_L1_Mv1_100: +L(zgemm_kernel_L1_Mv1_100): SAVEv1x1 -.Lzgemm_kernel_L1_Mv1_END: +L(zgemm_kernel_L1_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Lzgemm_kernel_L1_Mv1_20 + b.any L(zgemm_kernel_L1_Mv1_20) -.Lzgemm_kernel_L1_END: +L(zgemm_kernel_L1_END): /******************************************************************************/ -.Lzgemm_kernel_L999: +L(zgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/zgemv_n.S b/kernel/arm64/zgemv_n.S index 28afcada5a..143e28cffc 100644 --- a/kernel/arm64/zgemv_n.S +++ b/kernel/arm64/zgemv_n.S @@ -364,9 +364,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SAVE_REGS cmp N, xzr - ble .Lzgemv_n_kernel_L999 + ble L(zgemv_n_kernel_L999) cmp M, xzr - ble .Lzgemv_n_kernel_L999 + ble L(zgemv_n_kernel_L999) lsl LDA, LDA, #SHZ lsl INC_X, INC_X, #SHZ @@ -375,9 +375,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT cmp INC_Y, #1 - bne .Lzgemv_n_kernel_S_BEGIN + bne L(zgemv_n_kernel_S_BEGIN) -.Lzgemv_n_kernel_F_LOOP: +L(zgemv_n_kernel_F_LOOP): mov A_PTR, A mov Y_IPTR, Y mov Y_OPTR, Y @@ -387,40 +387,40 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #2 cmp I, xzr - beq .Lzgemv_n_kernel_F1 + beq L(zgemv_n_kernel_F1) -.Lzgemv_n_kernel_F4: +L(zgemv_n_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lzgemv_n_kernel_F4 + bne L(zgemv_n_kernel_F4) -.Lzgemv_n_kernel_F1: +L(zgemv_n_kernel_F1): ands I, M, #3 - ble .Lzgemv_n_kernel_F_END + ble L(zgemv_n_kernel_F_END) -.Lzgemv_n_kernel_F10: +L(zgemv_n_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lzgemv_n_kernel_F10 + bne L(zgemv_n_kernel_F10) -.Lzgemv_n_kernel_F_END: +L(zgemv_n_kernel_F_END): add A, A, LDA subs J, J, #1 - bne .Lzgemv_n_kernel_F_LOOP + bne L(zgemv_n_kernel_F_LOOP) - b .Lzgemv_n_kernel_L999 + b L(zgemv_n_kernel_L999) -.Lzgemv_n_kernel_S_BEGIN: +L(zgemv_n_kernel_S_BEGIN): INIT_S -.Lzgemv_n_kernel_S_LOOP: +L(zgemv_n_kernel_S_LOOP): mov A_PTR, A mov Y_IPTR, Y mov Y_OPTR, Y @@ -430,9 +430,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #2 cmp I, xzr - ble .Lzgemv_n_kernel_S1 + ble L(zgemv_n_kernel_S1) -.Lzgemv_n_kernel_S4: +L(zgemv_n_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -440,27 +440,27 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lzgemv_n_kernel_S4 + bne L(zgemv_n_kernel_S4) -.Lzgemv_n_kernel_S1: +L(zgemv_n_kernel_S1): ands I, M, #3 - ble .Lzgemv_n_kernel_S_END + ble L(zgemv_n_kernel_S_END) -.Lzgemv_n_kernel_S10: +L(zgemv_n_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lzgemv_n_kernel_S10 + bne L(zgemv_n_kernel_S10) -.Lzgemv_n_kernel_S_END: +L(zgemv_n_kernel_S_END): add A, A, LDA subs J, J, #1 - bne .Lzgemv_n_kernel_S_LOOP + bne L(zgemv_n_kernel_S_LOOP) -.Lzgemv_n_kernel_L999: +L(zgemv_n_kernel_L999): RESTORE_REGS mov w0, wzr diff --git a/kernel/arm64/zgemv_t.S b/kernel/arm64/zgemv_t.S index 0151029c77..38320c33da 100644 --- a/kernel/arm64/zgemv_t.S +++ b/kernel/arm64/zgemv_t.S @@ -292,9 +292,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SAVE_REGS cmp N, xzr - ble .Lzgemv_t_kernel_L999 + ble L(zgemv_t_kernel_L999) cmp M, xzr - ble .Lzgemv_t_kernel_L999 + ble L(zgemv_t_kernel_L999) lsl LDA, LDA, #SHZ lsl INC_Y, INC_Y, #SHZ @@ -303,9 +303,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT cmp INC_X, #1 - bne .Lzgemv_t_kernel_S_BEGIN + bne L(zgemv_t_kernel_S_BEGIN) -.Lzgemv_t_kernel_F_LOOP: +L(zgemv_t_kernel_F_LOOP): mov A_PTR, A mov X_PTR, X @@ -314,30 +314,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #2 cmp I, xzr - beq .Lzgemv_t_kernel_F1 + beq L(zgemv_t_kernel_F1) -.Lzgemv_t_kernel_F4: +L(zgemv_t_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lzgemv_t_kernel_F4 + bne L(zgemv_t_kernel_F4) KERNEL_F4_FINALIZE -.Lzgemv_t_kernel_F1: +L(zgemv_t_kernel_F1): ands I, M, #3 - ble .Lzgemv_t_kernel_F_END + ble L(zgemv_t_kernel_F_END) -.Lzgemv_t_kernel_F10: +L(zgemv_t_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lzgemv_t_kernel_F10 + bne L(zgemv_t_kernel_F10) -.Lzgemv_t_kernel_F_END: +L(zgemv_t_kernel_F_END): #if !defined(DOUBLE) ld1 {v4.2s}, [Y] @@ -355,15 +355,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add A, A, LDA subs J, J, #1 - bne .Lzgemv_t_kernel_F_LOOP + bne L(zgemv_t_kernel_F_LOOP) - b .Lzgemv_t_kernel_L999 + b L(zgemv_t_kernel_L999) -.Lzgemv_t_kernel_S_BEGIN: +L(zgemv_t_kernel_S_BEGIN): INIT_S -.Lzgemv_t_kernel_S_LOOP: +L(zgemv_t_kernel_S_LOOP): mov A_PTR, A mov X_PTR, X @@ -371,9 +371,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #2 cmp I, xzr - ble .Lzgemv_t_kernel_S1 + ble L(zgemv_t_kernel_S1) -.Lzgemv_t_kernel_S4: +L(zgemv_t_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -381,21 +381,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lzgemv_t_kernel_S4 + bne L(zgemv_t_kernel_S4) -.Lzgemv_t_kernel_S1: +L(zgemv_t_kernel_S1): ands I, M, #3 - ble .Lzgemv_t_kernel_S_END + ble L(zgemv_t_kernel_S_END) -.Lzgemv_t_kernel_S10: +L(zgemv_t_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lzgemv_t_kernel_S10 + bne L(zgemv_t_kernel_S10) -.Lzgemv_t_kernel_S_END: +L(zgemv_t_kernel_S_END): #if !defined(DOUBLE) ld1 {v4.2s}, [Y] @@ -413,9 +413,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add A, A, LDA subs J, J, #1 - bne .Lzgemv_t_kernel_S_LOOP + bne L(zgemv_t_kernel_S_LOOP) -.Lzgemv_t_kernel_L999: +L(zgemv_t_kernel_L999): RESTORE_REGS mov w0, wzr ret diff --git a/kernel/arm64/znrm2.S b/kernel/arm64/znrm2.S index a530b80f0d..03a72ae70d 100644 --- a/kernel/arm64/znrm2.S +++ b/kernel/arm64/znrm2.S @@ -123,69 +123,69 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if !defined(DOUBLE) ldr s4, [X] fcmp s4, REGZERO - beq KERNEL_S1_NEXT + beq L(KERNEL_S1_NEXT) fabs s4, s4 fcmp SCALE, s4 - bge KERNEL_S1_SCALE_GE_XR + bge L(KERNEL_S1_SCALE_GE_XR) fdiv s2, SCALE, s4 fmul s2, s2, s2 fmul s3, SSQ, s2 fadd SSQ, REGONE, s3 fmov SCALE, s4 - b KERNEL_S1_NEXT -KERNEL_S1_SCALE_GE_XR: + b L(KERNEL_S1_NEXT) +L(KERNEL_S1_SCALE_GE_XR): fdiv s2, s4, SCALE fmla SSQ, s2, v2.s[0] -KERNEL_S1_NEXT: +L(KERNEL_S1_NEXT): ldr s5, [X, #4] fcmp s5, REGZERO - beq KERNEL_S1_END + beq L(KERNEL_S1_END) fabs s5, s5 fcmp SCALE, s5 - bge KERNEL_S1_SCALE_GE_XI + bge L(KERNEL_S1_SCALE_GE_XI) fdiv s2, SCALE, s5 fmul s2, s2, s2 fmul s3, SSQ, s2 fadd SSQ, REGONE, s3 fmov SCALE, s5 - b KERNEL_S1_END -KERNEL_S1_SCALE_GE_XI: + b L(KERNEL_S1_END) +L(KERNEL_S1_SCALE_GE_XI): fdiv s2, s5, SCALE fmla SSQ, s2, v2.s[0] #else ldr d4, [X] fcmp d4, REGZERO - beq KERNEL_S1_NEXT + beq L(KERNEL_S1_NEXT) fabs d4, d4 fcmp SCALE, d4 - bge KERNEL_S1_SCALE_GE_XR + bge L(KERNEL_S1_SCALE_GE_XR) fdiv d2, SCALE, d4 fmul d2, d2, d2 fmul d3, SSQ, d2 fadd SSQ, REGONE, d3 fmov SCALE, d4 - b KERNEL_S1_NEXT -KERNEL_S1_SCALE_GE_XR: + b L(KERNEL_S1_NEXT) +L(KERNEL_S1_SCALE_GE_XR): fdiv d2, d4, SCALE fmla SSQ, d2, v2.d[0] -KERNEL_S1_NEXT: +L(KERNEL_S1_NEXT): ldr d5, [X, #8] fcmp d5, REGZERO - beq KERNEL_S1_END + beq L(KERNEL_S1_END) fabs d5, d5 fcmp SCALE, d5 - bge KERNEL_S1_SCALE_GE_XI + bge L(KERNEL_S1_SCALE_GE_XI) fdiv d2, SCALE, d5 fmul d2, d2, d2 fmul d3, SSQ, d2 fadd SSQ, REGONE, d3 fmov SCALE, d5 - b KERNEL_S1_END -KERNEL_S1_SCALE_GE_XI: + b L(KERNEL_S1_END) +L(KERNEL_S1_SCALE_GE_XI): fdiv d2, d5, SCALE fmla SSQ, d2, v2.d[0] #endif -KERNEL_S1_END: +L(KERNEL_S1_END): add X, X, INC_X .endm @@ -226,43 +226,43 @@ KERNEL_S1_END: INIT cmp N, #0 - ble .Lznrm2_kernel_L999 + ble L(znrm2_kernel_L999) cmp INC_X, #0 - beq .Lznrm2_kernel_L999 + beq L(znrm2_kernel_L999) cmp INC_X, #1 - bne .Lznrm2_kernel_S_BEGIN + bne L(znrm2_kernel_S_BEGIN) -.Lznrm2_kernel_F_BEGIN: +L(znrm2_kernel_F_BEGIN): asr I, N, #3 // I = N / 8 cmp I, xzr - ble .Lznrm2_kernel_F1 + ble L(znrm2_kernel_F1) -.Lznrm2_kernel_F8: +L(znrm2_kernel_F8): KERNEL_F8 subs I, I, #1 - bne .Lznrm2_kernel_F8 + bne L(znrm2_kernel_F8) -.Lznrm2_kernel_F1: +L(znrm2_kernel_F1): ands I, N, #7 - ble .Lznrm2_kernel_L999 + ble L(znrm2_kernel_L999) -.Lznrm2_kernel_F10: +L(znrm2_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lznrm2_kernel_F10 + bne L(znrm2_kernel_F10) - b .Lznrm2_kernel_L999 + b L(znrm2_kernel_L999) -.Lznrm2_kernel_S_BEGIN: +L(znrm2_kernel_S_BEGIN): INIT_S @@ -270,15 +270,15 @@ KERNEL_S1_END: .align 5 -.Lznrm2_kernel_S10: +L(znrm2_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lznrm2_kernel_S10 + bne L(znrm2_kernel_S10) -.Lznrm2_kernel_L999: +L(znrm2_kernel_L999): fsqrt SSQ, SSQ fmul SSQ, SCALE, SSQ diff --git a/kernel/arm64/zrot.S b/kernel/arm64/zrot.S index b5e510ebea..8186e58092 100644 --- a/kernel/arm64/zrot.S +++ b/kernel/arm64/zrot.S @@ -181,54 +181,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Lzrot_kernel_L999 + ble L(zrot_kernel_L999) INIT cmp INC_X, #1 - bne .Lzrot_kernel_S_BEGIN + bne L(zrot_kernel_S_BEGIN) cmp INC_Y, #1 - bne .Lzrot_kernel_S_BEGIN + bne L(zrot_kernel_S_BEGIN) -.Lzrot_kernel_F_BEGIN: +L(zrot_kernel_F_BEGIN): asr I, N, #2 cmp I, xzr - beq .Lzrot_kernel_F1 + beq L(zrot_kernel_F1) KERNEL_INIT_F4 -.Lzrot_kernel_F4: +L(zrot_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lzrot_kernel_F4 + bne L(zrot_kernel_F4) -.Lzrot_kernel_F1: +L(zrot_kernel_F1): ands I, N, #3 - ble .Lzrot_kernel_L999 + ble L(zrot_kernel_L999) -.Lzrot_kernel_F10: +L(zrot_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lzrot_kernel_F10 + bne L(zrot_kernel_F10) mov w0, wzr ret -.Lzrot_kernel_S_BEGIN: +L(zrot_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lzrot_kernel_S1 + ble L(zrot_kernel_S1) -.Lzrot_kernel_S4: +L(zrot_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -236,21 +236,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lzrot_kernel_S4 + bne L(zrot_kernel_S4) -.Lzrot_kernel_S1: +L(zrot_kernel_S1): ands I, N, #3 - ble .Lzrot_kernel_L999 + ble L(zrot_kernel_L999) -.Lzrot_kernel_S10: +L(zrot_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lzrot_kernel_S10 + bne L(zrot_kernel_S10) -.Lzrot_kernel_L999: +L(zrot_kernel_L999): mov w0, wzr ret + + EPILOGUE diff --git a/kernel/arm64/zscal.S b/kernel/arm64/zscal.S index 97d8a8b7ad..63d71deeaa 100644 --- a/kernel/arm64/zscal.S +++ b/kernel/arm64/zscal.S @@ -188,22 +188,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE - b zscal_begin -data_ar: + b L(zscal_begin) +L(data_ar): .word 0x3e44fae6 -data_ai: +L(data_ai): .word 0x3d320fa2 -data_xr: +L(data_xr): .word 0x3f4baff1 -data_xi: +L(data_xi): .word 0xbe8ef0bd -zscal_begin: +L(zscal_begin): - ldr s20, data_ar - ldr s21, data_ai - ldr s22, data_xr - ldr s23, data_xi + ldr s20, L(data_ar) + ldr s21, L(data_ai) + ldr s22, L(data_xr) + ldr s23, L(data_xi) fmul s24, s22, s21 fmla s24, s23, v20.s[0] @@ -215,74 +215,74 @@ zscal_begin: mov X_COPY, X cmp N, xzr - ble .Lzscal_kernel_L999 + ble L(zscal_kernel_L999) ldr FLAG, [sp] cmp FLAG, #1 -beq .Lzscal_kernel_RI_non_zero +beq L(zscal_kernel_RI_non_zero) fcmp DA_R, #0.0 - bne .Lzscal_kernel_R_non_zero + bne L(zscal_kernel_R_non_zero) fcmp DA_I, #0.0 - beq .Lzscal_kernel_RI_zero + beq L(zscal_kernel_RI_zero) -// b .Lzscal_kernel_R_zero +// b L(zscal_kernel_R_zero) -.Lzscal_kernel_R_non_zero: +L(zscal_kernel_R_non_zero): fcmp DA_I, #0.0 -//QUAK beq .Lzscal_kernel_I_zero +//QUAK beq L(zscal_kernel_I_zero) /******************************************************************************* * A_R != 0 && A_I != 0 *******************************************************************************/ -.Lzscal_kernel_RI_non_zero: +L(zscal_kernel_RI_non_zero): INIT cmp INC_X, #1 - bne .Lzscal_kernel_S_BEGIN + bne L(zscal_kernel_S_BEGIN) -.Lzscal_kernel_F_BEGIN: +L(zscal_kernel_F_BEGIN): asr I, N, #2 cmp I, xzr - beq .Lzscal_kernel_F1 + beq L(zscal_kernel_F1) KERNEL_INIT_F4 -.Lzscal_kernel_F4: +L(zscal_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lzscal_kernel_F4 + bne L(zscal_kernel_F4) -.Lzscal_kernel_F1: +L(zscal_kernel_F1): ands I, N, #3 - ble .Lzscal_kernel_L999 + ble L(zscal_kernel_L999) -.Lzscal_kernel_F10: +L(zscal_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lzscal_kernel_F10 + bne L(zscal_kernel_F10) mov w0, wzr ret -.Lzscal_kernel_S_BEGIN: +L(zscal_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lzscal_kernel_S1 + ble L(zscal_kernel_S1) -.Lzscal_kernel_S4: +L(zscal_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -290,21 +290,21 @@ beq .Lzscal_kernel_RI_non_zero KERNEL_S1 subs I, I, #1 - bne .Lzscal_kernel_S4 + bne L(zscal_kernel_S4) -.Lzscal_kernel_S1: +L(zscal_kernel_S1): ands I, N, #3 - ble .Lzscal_kernel_L999 + ble L(zscal_kernel_L999) -.Lzscal_kernel_S10: +L(zscal_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lzscal_kernel_S10 + bne L(zscal_kernel_S10) -.Lzscal_kernel_L999: +L(zscal_kernel_L999): mov w0, wzr ret @@ -313,7 +313,7 @@ beq .Lzscal_kernel_RI_non_zero * A_R == 0 && A_I != 0 *******************************************************************************/ -.Lzscal_kernel_R_zero: +L(zscal_kernel_R_zero): INIT_S #if !defined(DOUBLE) @@ -326,7 +326,7 @@ beq .Lzscal_kernel_RI_non_zero ins v1.d[1], v2.d[0] // v1 = -DA_I, DA_I #endif -.Lzscal_kernel_R_zero_1: +L(zscal_kernel_R_zero_1): #if !defined(DOUBLE) ld1 {v2.2s}, [X] // X1, X0 fmul v2.2s, v2.2s, v1.2s // -DA_I*X1, DA_I*X0 @@ -340,7 +340,7 @@ beq .Lzscal_kernel_RI_non_zero #endif add X, X, INC_X subs N, N, #1 - bne .Lzscal_kernel_R_zero_1 + bne L(zscal_kernel_R_zero_1) mov w0, wzr ret @@ -349,7 +349,7 @@ beq .Lzscal_kernel_RI_non_zero * A_R != 0 && A_I == 0 *******************************************************************************/ -.Lzscal_kernel_I_zero: +L(zscal_kernel_I_zero): INIT_S #if !defined(DOUBLE) ins v0.s[1], v0.s[0] // v0 = DA_R, DA_R @@ -357,7 +357,7 @@ beq .Lzscal_kernel_RI_non_zero ins v0.d[1], v0.d[0] // v0 = DA_R, DA_R #endif -.Lzscal_kernel_I_zero_1: +L(zscal_kernel_I_zero_1): #if !defined(DOUBLE) ld1 {v2.2s}, [X] // X1, X0 fmul v2.2s, v2.2s, v0.2s // DA_R*X1, DA_R*X0 @@ -369,7 +369,7 @@ beq .Lzscal_kernel_RI_non_zero #endif add X, X, INC_X subs N, N, #1 - bne .Lzscal_kernel_I_zero_1 + bne L(zscal_kernel_I_zero_1) mov w0, wzr ret @@ -378,16 +378,16 @@ beq .Lzscal_kernel_RI_non_zero * A_R == 0 && A_I == 0 *******************************************************************************/ -.Lzscal_kernel_RI_zero: +L(zscal_kernel_RI_zero): INIT_S -.Lzscal_kernel_RI_zero_1: +L(zscal_kernel_RI_zero_1): stp DA_R, DA_I, [X] add X, X, INC_X subs N, N, #1 - bne .Lzscal_kernel_RI_zero_1 + bne L(zscal_kernel_RI_zero_1) mov w0, wzr ret diff --git a/kernel/arm64/zsum.S b/kernel/arm64/zsum.S index 67ea3cb4d0..27d8716991 100644 --- a/kernel/arm64/zsum.S +++ b/kernel/arm64/zsum.S @@ -86,52 +86,52 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. fmov SUMF, REG0 cmp N, xzr - ble .Lzsum_kernel_L999 + ble L(zsum_kernel_L999) cmp INC_X, xzr - ble .Lzsum_kernel_L999 + ble L(zsum_kernel_L999) cmp INC_X, #1 - bne .Lzsum_kernel_S_BEGIN + bne L(zsum_kernel_S_BEGIN) -.Lzsum_kernel_F_BEGIN: +L(zsum_kernel_F_BEGIN): asr I, N, #2 cmp I, xzr - beq .Lzsum_kernel_F1 + beq L(zsum_kernel_F1) -.Lzsum_kernel_F4: +L(zsum_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lzsum_kernel_F4 + bne L(zsum_kernel_F4) KERNEL_F4_FINALIZE -.Lzsum_kernel_F1: +L(zsum_kernel_F1): ands I, N, #3 - ble .Lzsum_kernel_L999 + ble L(zsum_kernel_L999) -.Lzsum_kernel_F10: +L(zsum_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lzsum_kernel_F10 + bne L(zsum_kernel_F10) -.Lzsum_kernel_L999: +L(zsum_kernel_L999): ret -.Lzsum_kernel_S_BEGIN: +L(zsum_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lzsum_kernel_S1 + ble L(zsum_kernel_S1) -.Lzsum_kernel_S4: +L(zsum_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -139,19 +139,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lzsum_kernel_S4 + bne L(zsum_kernel_S4) -.Lzsum_kernel_S1: +L(zsum_kernel_S1): ands I, N, #3 - ble .Lzsum_kernel_L999 + ble L(zsum_kernel_L999) -.Lzsum_kernel_S10: +L(zsum_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lzsum_kernel_S10 + bne L(zsum_kernel_S10) ret diff --git a/kernel/arm64/ztrmm_kernel_4x4.S b/kernel/arm64/ztrmm_kernel_4x4.S index cd053b896c..b1874903dd 100644 --- a/kernel/arm64/ztrmm_kernel_4x4.S +++ b/kernel/arm64/ztrmm_kernel_4x4.S @@ -1078,9 +1078,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lztrmm_kernel_L2_BEGIN + ble L(ztrmm_kernel_L2_BEGIN) -.Lztrmm_kernel_L4_BEGIN: +L(ztrmm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -1094,15 +1094,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = start of A array -.Lztrmm_kernel_L4_M4_BEGIN: +L(ztrmm_kernel_L4_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lztrmm_kernel_L4_M2_BEGIN + ble L(ztrmm_kernel_L4_M2_BEGIN) .align 5 -.Lztrmm_kernel_L4_M4_20: +L(ztrmm_kernel_L4_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1123,7 +1123,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 cmp counterL , #2 - blt .Lztrmm_kernel_L4_M4_32 + blt L(ztrmm_kernel_L4_M4_32) KERNEL4x4_I KERNEL4x4_M2 @@ -1135,10 +1135,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 subs counterL, counterL, #2 - ble .Lztrmm_kernel_L4_M4_22a + ble L(ztrmm_kernel_L4_M4_22a) .align 5 -.Lztrmm_kernel_L4_M4_22: +L(ztrmm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 @@ -1150,10 +1150,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L4_M4_22 + bgt L(ztrmm_kernel_L4_M4_22) .align 5 -.Lztrmm_kernel_L4_M4_22a: +L(ztrmm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_M2 @@ -1164,13 +1164,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - b .Lztrmm_kernel_L4_M4_44 + b L(ztrmm_kernel_L4_M4_44) .align 5 -.Lztrmm_kernel_L4_M4_32: +L(ztrmm_kernel_L4_M4_32): tst counterL, #1 - ble .Lztrmm_kernel_L4_M4_40 + ble L(ztrmm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_M2 @@ -1181,26 +1181,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - b .Lztrmm_kernel_L4_M4_44 + b L(ztrmm_kernel_L4_M4_44) -.Lztrmm_kernel_L4_M4_40: +L(ztrmm_kernel_L4_M4_40): INIT4x4 -.Lztrmm_kernel_L4_M4_44: +L(ztrmm_kernel_L4_M4_44): ands counterL , tempK, #7 - ble .Lztrmm_kernel_L4_M4_100 + ble L(ztrmm_kernel_L4_M4_100) .align 5 -.Lztrmm_kernel_L4_M4_46: +L(ztrmm_kernel_L4_M4_46): KERNEL4x4_SUB subs counterL, counterL, #1 - bne .Lztrmm_kernel_L4_M4_46 + bne L(ztrmm_kernel_L4_M4_46) -.Lztrmm_kernel_L4_M4_100: +L(ztrmm_kernel_L4_M4_100): SAVE4x4 @@ -1223,20 +1223,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] -.Lztrmm_kernel_L4_M4_END: +L(ztrmm_kernel_L4_M4_END): subs counterI, counterI, #1 - bne .Lztrmm_kernel_L4_M4_20 + bne L(ztrmm_kernel_L4_M4_20) -.Lztrmm_kernel_L4_M2_BEGIN: +L(ztrmm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lztrmm_kernel_L4_END + ble L(ztrmm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lztrmm_kernel_L4_M1_BEGIN + ble L(ztrmm_kernel_L4_M1_BEGIN) -.Lztrmm_kernel_L4_M2_20: +L(ztrmm_kernel_L4_M2_20): INIT2x4 @@ -1260,9 +1260,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lztrmm_kernel_L4_M2_40 + ble L(ztrmm_kernel_L4_M2_40) -.Lztrmm_kernel_L4_M2_22: +L(ztrmm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1275,22 +1275,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L4_M2_22 + bgt L(ztrmm_kernel_L4_M2_22) -.Lztrmm_kernel_L4_M2_40: +L(ztrmm_kernel_L4_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lztrmm_kernel_L4_M2_100 + ble L(ztrmm_kernel_L4_M2_100) -.Lztrmm_kernel_L4_M2_42: +L(ztrmm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L4_M2_42 + bgt L(ztrmm_kernel_L4_M2_42) -.Lztrmm_kernel_L4_M2_100: +L(ztrmm_kernel_L4_M2_100): SAVE2x4 @@ -1310,15 +1310,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lztrmm_kernel_L4_M2_END: +L(ztrmm_kernel_L4_M2_END): -.Lztrmm_kernel_L4_M1_BEGIN: +L(ztrmm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lztrmm_kernel_L4_END + ble L(ztrmm_kernel_L4_END) -.Lztrmm_kernel_L4_M1_20: +L(ztrmm_kernel_L4_M1_20): INIT1x4 @@ -1342,9 +1342,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lztrmm_kernel_L4_M1_40 + ble L(ztrmm_kernel_L4_M1_40) -.Lztrmm_kernel_L4_M1_22: +L(ztrmm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1356,22 +1356,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L4_M1_22 + bgt L(ztrmm_kernel_L4_M1_22) -.Lztrmm_kernel_L4_M1_40: +L(ztrmm_kernel_L4_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lztrmm_kernel_L4_M1_100 + ble L(ztrmm_kernel_L4_M1_100) -.Lztrmm_kernel_L4_M1_42: +L(ztrmm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L4_M1_42 + bgt L(ztrmm_kernel_L4_M1_42) -.Lztrmm_kernel_L4_M1_100: +L(ztrmm_kernel_L4_M1_100): SAVE1x4 @@ -1392,7 +1392,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.Lztrmm_kernel_L4_END: +L(ztrmm_kernel_L4_END): lsl temp, origK, #6 add origPB, origPB, temp // B = B + K * 4 * 8 * 2 @@ -1402,19 +1402,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Lztrmm_kernel_L4_BEGIN + bgt L(ztrmm_kernel_L4_BEGIN) /******************************************************************************/ -.Lztrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(ztrmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lztrmm_kernel_L999 // error, N was less than 4? + ble L(ztrmm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Lztrmm_kernel_L1_BEGIN + ble L(ztrmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1426,14 +1426,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lztrmm_kernel_L2_M4_BEGIN: +L(ztrmm_kernel_L2_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI,#0 - ble .Lztrmm_kernel_L2_M2_BEGIN + ble L(ztrmm_kernel_L2_M2_BEGIN) -.Lztrmm_kernel_L2_M4_20: +L(ztrmm_kernel_L2_M4_20): INIT4x2 @@ -1457,10 +1457,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lztrmm_kernel_L2_M4_40 + ble L(ztrmm_kernel_L2_M4_40) .align 5 -.Lztrmm_kernel_L2_M4_22: +L(ztrmm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1472,22 +1472,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L2_M4_22 + bgt L(ztrmm_kernel_L2_M4_22) -.Lztrmm_kernel_L2_M4_40: +L(ztrmm_kernel_L2_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lztrmm_kernel_L2_M4_100 + ble L(ztrmm_kernel_L2_M4_100) -.Lztrmm_kernel_L2_M4_42: +L(ztrmm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L2_M4_42 + bgt L(ztrmm_kernel_L2_M4_42) -.Lztrmm_kernel_L2_M4_100: +L(ztrmm_kernel_L2_M4_100): SAVE4x2 @@ -1507,22 +1507,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lztrmm_kernel_L2_M4_END: +L(ztrmm_kernel_L2_M4_END): subs counterI, counterI, #1 - bgt .Lztrmm_kernel_L2_M4_20 + bgt L(ztrmm_kernel_L2_M4_20) -.Lztrmm_kernel_L2_M2_BEGIN: +L(ztrmm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lztrmm_kernel_L2_END + ble L(ztrmm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lztrmm_kernel_L2_M1_BEGIN + ble L(ztrmm_kernel_L2_M1_BEGIN) -.Lztrmm_kernel_L2_M2_20: +L(ztrmm_kernel_L2_M2_20): INIT2x2 @@ -1546,9 +1546,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lztrmm_kernel_L2_M2_40 + ble L(ztrmm_kernel_L2_M2_40) -.Lztrmm_kernel_L2_M2_22: +L(ztrmm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1561,22 +1561,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L2_M2_22 + bgt L(ztrmm_kernel_L2_M2_22) -.Lztrmm_kernel_L2_M2_40: +L(ztrmm_kernel_L2_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lztrmm_kernel_L2_M2_100 + ble L(ztrmm_kernel_L2_M2_100) -.Lztrmm_kernel_L2_M2_42: +L(ztrmm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L2_M2_42 + bgt L(ztrmm_kernel_L2_M2_42) -.Lztrmm_kernel_L2_M2_100: +L(ztrmm_kernel_L2_M2_100): SAVE2x2 @@ -1596,15 +1596,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lztrmm_kernel_L2_M2_END: +L(ztrmm_kernel_L2_M2_END): -.Lztrmm_kernel_L2_M1_BEGIN: +L(ztrmm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lztrmm_kernel_L2_END + ble L(ztrmm_kernel_L2_END) -.Lztrmm_kernel_L2_M1_20: +L(ztrmm_kernel_L2_M1_20): INIT1x2 @@ -1628,9 +1628,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lztrmm_kernel_L2_M1_40 + ble L(ztrmm_kernel_L2_M1_40) -.Lztrmm_kernel_L2_M1_22: +L(ztrmm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1642,22 +1642,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L2_M1_22 + bgt L(ztrmm_kernel_L2_M1_22) -.Lztrmm_kernel_L2_M1_40: +L(ztrmm_kernel_L2_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lztrmm_kernel_L2_M1_100 + ble L(ztrmm_kernel_L2_M1_100) -.Lztrmm_kernel_L2_M1_42: +L(ztrmm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L2_M1_42 + bgt L(ztrmm_kernel_L2_M1_42) -.Lztrmm_kernel_L2_M1_100: +L(ztrmm_kernel_L2_M1_100): SAVE1x2 @@ -1678,7 +1678,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.Lztrmm_kernel_L2_END: +L(ztrmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -1688,11 +1688,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lztrmm_kernel_L1_BEGIN: +L(ztrmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lztrmm_kernel_L999 // done + ble L(ztrmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1706,14 +1706,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lztrmm_kernel_L1_M4_BEGIN: +L(ztrmm_kernel_L1_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lztrmm_kernel_L1_M2_BEGIN + ble L(ztrmm_kernel_L1_M2_BEGIN) -.Lztrmm_kernel_L1_M4_20: +L(ztrmm_kernel_L1_M4_20): INIT4x1 @@ -1737,10 +1737,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lztrmm_kernel_L1_M4_40 + ble L(ztrmm_kernel_L1_M4_40) .align 5 -.Lztrmm_kernel_L1_M4_22: +L(ztrmm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1752,22 +1752,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L1_M4_22 + bgt L(ztrmm_kernel_L1_M4_22) -.Lztrmm_kernel_L1_M4_40: +L(ztrmm_kernel_L1_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lztrmm_kernel_L1_M4_100 + ble L(ztrmm_kernel_L1_M4_100) -.Lztrmm_kernel_L1_M4_42: +L(ztrmm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L1_M4_42 + bgt L(ztrmm_kernel_L1_M4_42) -.Lztrmm_kernel_L1_M4_100: +L(ztrmm_kernel_L1_M4_100): SAVE4x1 @@ -1787,22 +1787,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lztrmm_kernel_L1_M4_END: +L(ztrmm_kernel_L1_M4_END): subs counterI, counterI, #1 - bgt .Lztrmm_kernel_L1_M4_20 + bgt L(ztrmm_kernel_L1_M4_20) -.Lztrmm_kernel_L1_M2_BEGIN: +L(ztrmm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lztrmm_kernel_L1_END + ble L(ztrmm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lztrmm_kernel_L1_M1_BEGIN + ble L(ztrmm_kernel_L1_M1_BEGIN) -.Lztrmm_kernel_L1_M2_20: +L(ztrmm_kernel_L1_M2_20): INIT2x1 @@ -1826,9 +1826,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lztrmm_kernel_L1_M2_40 + ble L(ztrmm_kernel_L1_M2_40) -.Lztrmm_kernel_L1_M2_22: +L(ztrmm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1841,22 +1841,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L1_M2_22 + bgt L(ztrmm_kernel_L1_M2_22) -.Lztrmm_kernel_L1_M2_40: +L(ztrmm_kernel_L1_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lztrmm_kernel_L1_M2_100 + ble L(ztrmm_kernel_L1_M2_100) -.Lztrmm_kernel_L1_M2_42: +L(ztrmm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L1_M2_42 + bgt L(ztrmm_kernel_L1_M2_42) -.Lztrmm_kernel_L1_M2_100: +L(ztrmm_kernel_L1_M2_100): SAVE2x1 @@ -1876,15 +1876,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lztrmm_kernel_L1_M2_END: +L(ztrmm_kernel_L1_M2_END): -.Lztrmm_kernel_L1_M1_BEGIN: +L(ztrmm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lztrmm_kernel_L1_END + ble L(ztrmm_kernel_L1_END) -.Lztrmm_kernel_L1_M1_20: +L(ztrmm_kernel_L1_M1_20): INIT1x1 @@ -1908,9 +1908,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lztrmm_kernel_L1_M1_40 + ble L(ztrmm_kernel_L1_M1_40) -.Lztrmm_kernel_L1_M1_22: +L(ztrmm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1922,30 +1922,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L1_M1_22 + bgt L(ztrmm_kernel_L1_M1_22) -.Lztrmm_kernel_L1_M1_40: +L(ztrmm_kernel_L1_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lztrmm_kernel_L1_M1_100 + ble L(ztrmm_kernel_L1_M1_100) -.Lztrmm_kernel_L1_M1_42: +L(ztrmm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L1_M1_42 + bgt L(ztrmm_kernel_L1_M1_42) -.Lztrmm_kernel_L1_M1_100: +L(ztrmm_kernel_L1_M1_100): SAVE1x1 -.Lztrmm_kernel_L1_END: +L(ztrmm_kernel_L1_END): -.Lztrmm_kernel_L999: +L(ztrmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/ztrmm_kernel_sve_v1x4.S b/kernel/arm64/ztrmm_kernel_sve_v1x4.S index b71a3d39ed..22be5a82c5 100644 --- a/kernel/arm64/ztrmm_kernel_sve_v1x4.S +++ b/kernel/arm64/ztrmm_kernel_sve_v1x4.S @@ -584,10 +584,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lztrmm_kernel_L2_BEGIN + ble L(ztrmm_kernel_L2_BEGIN) /******************************************************************************/ -.Lztrmm_kernel_L4_BEGIN: +L(ztrmm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -600,7 +600,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = start of A array -.Lztrmm_kernel_L4_Mv1_BEGIN: +L(ztrmm_kernel_L4_Mv1_BEGIN): /* Loop over M is done in an SVE fashion. This has the benefit of the last M%SVE_LEN iterations being done in a single sweep */ mov counterI, #0 @@ -608,7 +608,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension .align 5 -.Lztrmm_kernel_L4_Mv1_20: +L(ztrmm_kernel_L4_Mv1_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -631,7 +631,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 cmp counterL , #2 - blt .Lztrmm_kernel_L4_Mv1_32 + blt L(ztrmm_kernel_L4_Mv1_32) KERNELv1x4_I KERNELv1x4_M2 @@ -643,10 +643,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lztrmm_kernel_L4_Mv1_22a + ble L(ztrmm_kernel_L4_Mv1_22a) .align 5 -.Lztrmm_kernel_L4_Mv1_22: +L(ztrmm_kernel_L4_Mv1_22): KERNELv1x4_M1 KERNELv1x4_M2 @@ -658,10 +658,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M2 subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L4_Mv1_22 + bgt L(ztrmm_kernel_L4_Mv1_22) .align 5 -.Lztrmm_kernel_L4_Mv1_22a: +L(ztrmm_kernel_L4_Mv1_22a): KERNELv1x4_M1 KERNELv1x4_M2 @@ -672,13 +672,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M1 KERNELv1x4_E - b .Lztrmm_kernel_L4_Mv1_44 + b L(ztrmm_kernel_L4_Mv1_44) .align 5 -.Lztrmm_kernel_L4_Mv1_32: +L(ztrmm_kernel_L4_Mv1_32): tst counterL, #1 - ble .Lztrmm_kernel_L4_Mv1_40 + ble L(ztrmm_kernel_L4_Mv1_40) KERNELv1x4_I KERNELv1x4_M2 @@ -689,26 +689,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M1 KERNELv1x4_E - b .Lztrmm_kernel_L4_Mv1_44 + b L(ztrmm_kernel_L4_Mv1_44) -.Lztrmm_kernel_L4_Mv1_40: +L(ztrmm_kernel_L4_Mv1_40): INITv1x4 -.Lztrmm_kernel_L4_Mv1_44: +L(ztrmm_kernel_L4_Mv1_44): ands counterL , tempK, #7 - ble .Lztrmm_kernel_L4_Mv1_100 + ble L(ztrmm_kernel_L4_Mv1_100) .align 5 -.Lztrmm_kernel_L4_Mv1_46: +L(ztrmm_kernel_L4_Mv1_46): KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Lztrmm_kernel_L4_Mv1_46 + bne L(ztrmm_kernel_L4_Mv1_46) -.Lztrmm_kernel_L4_Mv1_100: +L(ztrmm_kernel_L4_Mv1_100): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -732,16 +732,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SAVEv1x4 -.Lztrmm_kernel_L4_Mv1_END: +L(ztrmm_kernel_L4_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension - b.any .Lztrmm_kernel_L4_Mv1_20 + b.any L(ztrmm_kernel_L4_Mv1_20) -.Lztrmm_kernel_L4_END: +L(ztrmm_kernel_L4_END): lsl temp, origK, #6 add origPB, origPB, temp // B = B + K * 4 * 8 * 2 @@ -751,19 +751,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Lztrmm_kernel_L4_BEGIN + bgt L(ztrmm_kernel_L4_BEGIN) /******************************************************************************/ -.Lztrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(ztrmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lztrmm_kernel_L999 + ble L(ztrmm_kernel_L999) tst counterJ , #2 - ble .Lztrmm_kernel_L1_BEGIN + ble L(ztrmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC add pCRow1, pCRow0, LDC @@ -778,14 +778,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lztrmm_kernel_L2_Mv1_BEGIN: +L(ztrmm_kernel_L2_Mv1_BEGIN): mov counterI, #0 whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d -.Lztrmm_kernel_L2_Mv1_20: +L(ztrmm_kernel_L2_Mv1_20): INITv1x2 @@ -809,10 +809,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lztrmm_kernel_L2_Mv1_40 + ble L(ztrmm_kernel_L2_Mv1_40) .align 5 -.Lztrmm_kernel_L2_Mv1_22: +L(ztrmm_kernel_L2_Mv1_22): KERNELv1x2_SUB KERNELv1x2_SUB KERNELv1x2_SUB @@ -824,22 +824,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L2_Mv1_22 + bgt L(ztrmm_kernel_L2_Mv1_22) -.Lztrmm_kernel_L2_Mv1_40: +L(ztrmm_kernel_L2_Mv1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lztrmm_kernel_L2_Mv1_100 + ble L(ztrmm_kernel_L2_Mv1_100) -.Lztrmm_kernel_L2_Mv1_42: +L(ztrmm_kernel_L2_Mv1_42): KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L2_Mv1_42 + bgt L(ztrmm_kernel_L2_Mv1_42) -.Lztrmm_kernel_L2_Mv1_100: +L(ztrmm_kernel_L2_Mv1_100): SAVEv1x2 @@ -859,16 +859,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, lanes #endif -.Lztrmm_kernel_L2_Mv1_END: +L(ztrmm_kernel_L2_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Lztrmm_kernel_L2_Mv1_20 + b.any L(ztrmm_kernel_L2_Mv1_20) -.Lztrmm_kernel_L2_END: +L(ztrmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -878,11 +878,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lztrmm_kernel_L1_BEGIN: +L(ztrmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lztrmm_kernel_L999 // done + ble L(ztrmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -894,14 +894,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lztrmm_kernel_L1_Mv1_BEGIN: +L(ztrmm_kernel_L1_Mv1_BEGIN): mov counterI, #0 whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d -.Lztrmm_kernel_L1_Mv1_20: +L(ztrmm_kernel_L1_Mv1_20): INITv1x1 @@ -925,10 +925,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lztrmm_kernel_L1_Mv1_40 + ble L(ztrmm_kernel_L1_Mv1_40) .align 5 -.Lztrmm_kernel_L1_Mv1_22: +L(ztrmm_kernel_L1_Mv1_22): KERNELv1x1_SUB KERNELv1x1_SUB KERNELv1x1_SUB @@ -940,22 +940,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L1_Mv1_22 + bgt L(ztrmm_kernel_L1_Mv1_22) -.Lztrmm_kernel_L1_Mv1_40: +L(ztrmm_kernel_L1_Mv1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lztrmm_kernel_L1_Mv1_100 + ble L(ztrmm_kernel_L1_Mv1_100) -.Lztrmm_kernel_L1_Mv1_42: +L(ztrmm_kernel_L1_Mv1_42): KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L1_Mv1_42 + bgt L(ztrmm_kernel_L1_Mv1_42) -.Lztrmm_kernel_L1_Mv1_100: +L(ztrmm_kernel_L1_Mv1_100): SAVEv1x1 @@ -975,18 +975,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, lanes #endif -.Lztrmm_kernel_L1_Mv1_END: +L(ztrmm_kernel_L1_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Lztrmm_kernel_L1_Mv1_20 + b.any L(ztrmm_kernel_L1_Mv1_20) -.Lztrmm_kernel_L1_END: +L(ztrmm_kernel_L1_END): /******************************************************************************/ -.Lztrmm_kernel_L999: +L(ztrmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/x86_64/amax.S b/kernel/x86_64/amax.S index 1498bb226c..62cc7d3a29 100644 --- a/kernel/x86_64/amax.S +++ b/kernel/x86_64/amax.S @@ -64,9 +64,9 @@ fldz testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) ffreep %st @@ -76,17 +76,17 @@ #endif addq INCX, X decq M - jle .L999 + jle L(999) cmpq $SIZE, INCX - jne .L40 + jne L(40) movq M, I sarq $3, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -166,17 +166,17 @@ addq $8 * SIZE, X decq I - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): movq M, I andq $7, I - jle .L999 + jle L(999) ALIGN_4 -.L21: +L(21): FLD 0 * SIZE(X) #ifdef USE_ABS fabs @@ -188,17 +188,17 @@ addq $1 * SIZE, X decq I - jg .L21 - jmp .L999 + jg L(21) + jmp L(999) ALIGN_4 -.L40: +L(40): movq M, I sarq $3, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): FLD 0 * SIZE(X) addq INCX, X #ifdef USE_ABS @@ -280,17 +280,17 @@ ffreep %st decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): movq M, I andq $7, I - jle .L999 + jle L(999) ALIGN_4 -.L61: +L(61): FLD 0 * SIZE(X) #ifdef USE_ABS fabs @@ -302,10 +302,10 @@ addq INCX, X decq I - jg .L61 + jg L(61) ALIGN_4 -.L999: +L(999): ret EPILOGUE diff --git a/kernel/x86_64/amax_atom.S b/kernel/x86_64/amax_atom.S index 6164cb3e33..d2c128b0fe 100644 --- a/kernel/x86_64/amax_atom.S +++ b/kernel/x86_64/amax_atom.S @@ -60,10 +60,10 @@ leaq (, INCX, SIZE), INCX testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) #ifdef USE_ABS pcmpeqb %xmm15, %xmm15 @@ -77,18 +77,18 @@ andps %xmm15, %xmm0 #endif decq M - jle .L999 + jle L(999) movaps %xmm0, %xmm1 movaps %xmm0, %xmm2 movaps %xmm0, %xmm3 cmpq $SIZE, INCX - jne .L20 + jne L(20) movq M, I sarq $3, I - jle .L15 + jle L(15) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -101,10 +101,10 @@ movsd 7 * SIZE(X), %xmm11 decq I - jle .L13 + jle L(13) ALIGN_4 -.L12: +L(12): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -159,10 +159,10 @@ addq $8 * SIZE, X decq I - jg .L12 + jg L(12) ALIGN_4 -.L13: +L(13): #ifdef USE_ABS andps %xmm15, %xmm4 #endif @@ -200,9 +200,9 @@ addq $8 * SIZE, X ALIGN_4 -.L15: +L(15): testq $4, M - jle .L17 + jle L(17) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -229,9 +229,9 @@ addq $4 * SIZE, X ALIGN_3 -.L17: +L(17): testq $2, M - jle .L18 + jle L(18) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -247,22 +247,22 @@ addq $2 * SIZE, X ALIGN_3 -.L18: +L(18): testq $1, M - jle .L998 + jle L(998) movsd 0 * SIZE(X), %xmm4 #ifdef USE_ABS andps %xmm15, %xmm4 #endif maxsd %xmm4, %xmm3 - jmp .L998 + jmp L(998) ALIGN_3 -.L20: +L(20): movq M, I sarq $3, I - jle .L25 + jle L(25) movsd (X), %xmm4 addq INCX, X @@ -282,10 +282,10 @@ movsd (X), %xmm11 decq I - jle .L23 + jle L(23) ALIGN_4 -.L22: +L(22): #ifdef USE_ABS andps %xmm15, %xmm4 #endif @@ -343,10 +343,10 @@ movsd (X), %xmm11 decq I - jg .L22 + jg L(22) ALIGN_4 -.L23: +L(23): #ifdef USE_ABS andps %xmm15, %xmm4 #endif @@ -383,9 +383,9 @@ maxsd %xmm11, %xmm3 ALIGN_4 -.L25: +L(25): testq $4, M - jle .L27 + jle L(27) movsd (X), %xmm4 addq INCX, X @@ -414,9 +414,9 @@ maxsd %xmm7, %xmm3 ALIGN_3 -.L27: +L(27): testq $2, M - jle .L28 + jle L(28) movsd (X), %xmm4 addq INCX, X @@ -433,9 +433,9 @@ maxsd %xmm5, %xmm2 ALIGN_3 -.L28: +L(28): testq $1, M - jle .L998 + jle L(998) movsd (X), %xmm4 addq INCX, X @@ -445,13 +445,13 @@ maxsd %xmm4, %xmm3 ALIGN_3 -.L998: +L(998): maxsd %xmm1, %xmm0 maxsd %xmm3, %xmm2 maxsd %xmm2, %xmm0 ALIGN_4 -.L999: +L(999): RESTOREREGISTERS diff --git a/kernel/x86_64/amax_sse.S b/kernel/x86_64/amax_sse.S index 2349905d39..c4e7471711 100644 --- a/kernel/x86_64/amax_sse.S +++ b/kernel/x86_64/amax_sse.S @@ -61,7 +61,7 @@ leaq (, INCX, SIZE), INCX testq M, M - jle .L999 + jle L(999) #ifdef USE_ABS pcmpeqb %xmm15, %xmm15 @@ -78,18 +78,18 @@ movaps %xmm0, %xmm3 addq INCX, X decq M - jle .L999 + jle L(999) cmpq $SIZE, INCX - jne .L40 + jne L(40) subq $-32 * SIZE, X cmpq $3, M - jle .L17 + jle L(17) testq $SIZE, X - je .L05 + je L(05) movss -32 * SIZE(X), %xmm1 shufps $0, %xmm1, %xmm1 @@ -100,9 +100,9 @@ addq $SIZE, X ALIGN_3 -.L05: +L(05): testq $2 * SIZE, X - je .L06 + je L(06) movsd -32 * SIZE(X), %xmm2 unpcklps %xmm2, %xmm2 @@ -113,10 +113,10 @@ addq $2 * SIZE, X ALIGN_3 -.L06: +L(06): movq M, I sarq $5, I - jle .L15 + jle L(15) movaps -32 * SIZE(X), %xmm4 movaps -28 * SIZE(X), %xmm5 @@ -124,10 +124,10 @@ movaps -20 * SIZE(X), %xmm7 decq I - jle .L12 + jle L(12) ALIGN_4 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -186,10 +186,10 @@ subq $-32 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_4 -.L12: +L(12): #ifdef USE_ABS andps %xmm15, %xmm4 #endif @@ -238,9 +238,9 @@ ALIGN_3 -.L15: +L(15): testq $16, M - je .L16 + je L(16) movaps -32 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -269,9 +269,9 @@ addq $16 * SIZE, X ALIGN_3 -.L16: +L(16): testq $8, M - je .L17 + je L(17) movaps -32 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -287,9 +287,9 @@ addq $8 * SIZE, X ALIGN_3 -.L17: +L(17): testq $4, M - je .L18 + je L(18) movaps -32 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -299,9 +299,9 @@ addq $4 * SIZE, X ALIGN_3 -.L18: +L(18): testq $2, M - je .L19 + je L(19) movsd -32 * SIZE(X), %xmm4 unpcklps %xmm4, %xmm4 @@ -312,25 +312,25 @@ addq $2 * SIZE, X ALIGN_3 -.L19: +L(19): testq $1, M - je .L998 + je L(998) movss -32 * SIZE(X), %xmm4 #ifdef USE_ABS andps %xmm15, %xmm4 #endif maxss %xmm4, %xmm0 - jmp .L998 + jmp L(998) ALIGN_3 -.L40: +L(40): movq M, I sarq $3, I - jle .L45 + jle L(45) ALIGN_4 -.L41: +L(41): movss (X), %xmm4 addq INCX, X #ifdef USE_ABS @@ -388,12 +388,12 @@ maxss %xmm7, %xmm3 decq I - jg .L41 + jg L(41) ALIGN_4 -.L45: +L(45): testq $4, M - je .L46 + je L(46) movss (X), %xmm4 addq INCX, X @@ -424,9 +424,9 @@ maxss %xmm7, %xmm3 ALIGN_3 -.L46: +L(46): testq $2, M - je .L47 + je L(47) movss (X), %xmm4 addq INCX, X @@ -443,9 +443,9 @@ maxss %xmm5, %xmm1 ALIGN_3 -.L47: +L(47): testq $1, M - je .L998 + je L(998) movss (X), %xmm4 addq INCX, X @@ -455,7 +455,7 @@ maxss %xmm4, %xmm2 ALIGN_4 -.L998: +L(998): maxps %xmm1, %xmm0 maxps %xmm3, %xmm2 maxps %xmm2, %xmm0 @@ -467,7 +467,7 @@ maxss %xmm1, %xmm0 ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/amax_sse2.S b/kernel/x86_64/amax_sse2.S index 44ddaba778..5893b6a292 100644 --- a/kernel/x86_64/amax_sse2.S +++ b/kernel/x86_64/amax_sse2.S @@ -61,7 +61,7 @@ leaq (, INCX, SIZE), INCX testq M, M - jle .L999 + jle L(999) #ifdef USE_ABS pcmpeqb %xmm15, %xmm15 @@ -78,15 +78,15 @@ movaps %xmm0, %xmm2 movaps %xmm0, %xmm3 decq M - jle .L999 + jle L(999) cmpq $SIZE, INCX - jne .L40 + jne L(40) subq $-16 * SIZE, X testq $SIZE, X - je .L05 + je L(05) movsd -16 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -96,13 +96,13 @@ maxpd %xmm4, %xmm3 addq $SIZE, X decq M - jle .L998 + jle L(998) ALIGN_3 -.L05: +L(05): movq M, I sarq $4, I - jle .L15 + jle L(15) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -110,10 +110,10 @@ movaps -10 * SIZE(X), %xmm7 decq I - jle .L12 + jle L(12) ALIGN_4 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -172,10 +172,10 @@ subq $-16 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_4 -.L12: +L(12): #ifdef USE_ABS andps %xmm15, %xmm4 #endif @@ -223,9 +223,9 @@ subq $-16 * SIZE, X ALIGN_4 -.L15: +L(15): testq $8, M - jle .L16 + jle L(16) movaps -16 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -253,9 +253,9 @@ addq $8 * SIZE, X ALIGN_3 -.L16: +L(16): testq $4, M - jle .L17 + jle L(17) movaps -16 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -272,9 +272,9 @@ addq $4 * SIZE, X ALIGN_3 -.L17: +L(17): testq $2, M - jle .L18 + jle L(18) movaps -16 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -284,9 +284,9 @@ addq $2 * SIZE, X ALIGN_3 -.L18: +L(18): testq $1, M - jle .L998 + jle L(998) movsd -16 * SIZE(X), %xmm4 unpcklpd %xmm4, %xmm4 @@ -294,16 +294,16 @@ andps %xmm15, %xmm4 #endif maxpd %xmm4, %xmm3 - jmp .L998 + jmp L(998) ALIGN_3 -.L40: +L(40): movq M, I sarq $4, I - jle .L45 + jle L(45) ALIGN_4 -.L41: +L(41): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -385,15 +385,15 @@ maxpd %xmm7, %xmm3 decq I - jg .L41 + jg L(41) ALIGN_4 -.L45: +L(45): andq $15, M - jle .L998 + jle L(998) testq $8, M - je .L46 + je L(46) movsd (X), %xmm4 addq INCX, X @@ -432,9 +432,9 @@ maxpd %xmm7, %xmm3 ALIGN_3 -.L46: +L(46): testq $4, M - je .L47 + je L(47) movsd (X), %xmm4 addq INCX, X @@ -455,9 +455,9 @@ maxpd %xmm5, %xmm1 ALIGN_3 -.L47: +L(47): testq $2, M - je .L48 + je L(48) movsd (X), %xmm6 addq INCX, X @@ -469,9 +469,9 @@ maxpd %xmm6, %xmm2 ALIGN_3 -.L48: +L(48): testq $1, M - je .L998 + je L(998) movsd (X), %xmm7 unpcklpd %xmm7, %xmm7 @@ -481,7 +481,7 @@ maxpd %xmm7, %xmm3 ALIGN_4 -.L998: +L(998): maxpd %xmm1, %xmm0 maxpd %xmm3, %xmm2 maxpd %xmm2, %xmm0 @@ -490,7 +490,7 @@ maxsd %xmm1, %xmm0 ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/asum.S b/kernel/x86_64/asum.S index a2cbfd4804..11d5239c45 100644 --- a/kernel/x86_64/asum.S +++ b/kernel/x86_64/asum.S @@ -56,9 +56,9 @@ fldz testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) salq $BASE_SHIFT, INCX @@ -66,14 +66,14 @@ fldz fldz cmpq $SIZE, INCX - jne .L40 + jne L(40) movq M, I sarq $3, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -109,31 +109,31 @@ faddp %st, %st(1) decq I - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): andq $7, M - jle .L998 + jle L(998) ALIGN_4 -.L21: +L(21): FLD (X) fabs faddp %st,%st(1) addq $1 * SIZE, X decq M - jg .L21 - jmp .L998 + jg L(21) + jmp L(998) ALIGN_4 -.L40: +L(40): movq M, I sarq $3, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): FLD (X) addq INCX, X fabs @@ -171,31 +171,31 @@ faddp %st, %st(1) decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): andq $7, M - jle .L998 + jle L(998) ALIGN_4 -.L61: +L(61): FLD (X) addq INCX, X fabs faddp %st,%st(1) decq M - jg .L61 + jg L(61) ALIGN_4 -.L998: +L(998): faddp %st,%st(2) faddp %st,%st(1) faddp %st,%st(1) ALIGN_4 -.L999: +L(999): ret EPILOGUE diff --git a/kernel/x86_64/asum_atom.S b/kernel/x86_64/asum_atom.S index 910a48f09e..7dde619511 100644 --- a/kernel/x86_64/asum_atom.S +++ b/kernel/x86_64/asum_atom.S @@ -54,9 +54,9 @@ xorps %xmm0, %xmm0 testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) xorps %xmm1, %xmm1 xorps %xmm2, %xmm2 @@ -69,24 +69,24 @@ xorps %xmm13, %xmm13 cmpq $SIZE, INCX - jne .L20 + jne L(20) testq $SIZE, X - je .L05 + je L(05) movsd (X), %xmm0 addq $SIZE, X andps %xmm15, %xmm0 decq M - jle .L999 + jle L(999) ALIGN_3 -.L05: +L(05): subq $-16 * SIZE, X movq M, I sarq $4, I - jle .L12 + jle L(12) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -99,10 +99,10 @@ movaps -2 * SIZE(X), %xmm11 decq I - jle .L11 + jle L(11) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -161,10 +161,10 @@ subq $-16 * SIZE, X decq I - jg .L10 + jg L(10) ALIGN_4 -.L11: +L(11): andps %xmm15, %xmm4 addsd %xmm13, %xmm3 pshufd $0x4e, %xmm4, %xmm12 @@ -209,12 +209,12 @@ subq $-16 * SIZE, X ALIGN_3 -.L12: +L(12): andq $15, M - jle .L998 + jle L(998) testq $8, M - je .L13 + je L(13) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -240,9 +240,9 @@ addsd %xmm13, %xmm3 ALIGN_3 -.L13: +L(13): testq $4, M - je .L14 + je L(14) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -258,9 +258,9 @@ addsd %xmm13, %xmm3 ALIGN_3 -.L14: +L(14): testq $2, M - je .L15 + je L(15) movaps -16 * SIZE(X), %xmm4 addq $2 * SIZE, X @@ -271,20 +271,20 @@ addsd %xmm5, %xmm3 ALIGN_3 -.L15: +L(15): testq $1, M - je .L998 + je L(998) movsd -16 * SIZE(X), %xmm4 andps %xmm15, %xmm4 addsd %xmm4, %xmm0 - jmp .L998 + jmp L(998) ALIGN_3 -.L20: +L(20): movq M, I sarq $3, I - jle .L25 + jle L(25) movsd (X), %xmm4 addq INCX, X @@ -304,10 +304,10 @@ movsd (X), %xmm11 decq I - jle .L23 + jle L(23) ALIGN_4 -.L22: +L(22): andps %xmm15, %xmm4 addq INCX, X addsd %xmm4, %xmm0 @@ -343,10 +343,10 @@ movsd (X), %xmm11 decq I - jg .L22 + jg L(22) ALIGN_4 -.L23: +L(23): andps %xmm15, %xmm4 addq INCX, X addsd %xmm4, %xmm0 @@ -367,12 +367,12 @@ addsd %xmm11, %xmm3 ALIGN_3 -.L25: +L(25): andq $7, M - jle .L998 + jle L(998) testq $4, M - je .L26 + je L(26) movsd (X), %xmm4 addq INCX, X @@ -393,9 +393,9 @@ addsd %xmm7, %xmm3 ALIGN_3 -.L26: +L(26): testq $2, M - je .L27 + je L(27) movsd (X), %xmm4 addq INCX, X @@ -409,22 +409,22 @@ addsd %xmm5, %xmm1 ALIGN_3 -.L27: +L(27): testq $1, M - je .L998 + je L(998) movsd (X), %xmm4 andps %xmm15, %xmm4 addsd %xmm4, %xmm0 ALIGN_3 -.L998: +L(998): addsd %xmm1, %xmm0 addsd %xmm3, %xmm2 addsd %xmm2, %xmm0 ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/asum_sse.S b/kernel/x86_64/asum_sse.S index 7d7004d486..5519db6183 100644 --- a/kernel/x86_64/asum_sse.S +++ b/kernel/x86_64/asum_sse.S @@ -54,9 +54,9 @@ xorps %xmm0, %xmm0 testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) xorps %xmm1, %xmm1 xorps %xmm2, %xmm2 @@ -68,37 +68,37 @@ leaq (, INCX, SIZE), INCX cmpq $SIZE, INCX - jne .L100 + jne L(100) subq $-32 * SIZE, X cmpq $3, M - jle .L18 + jle L(18) testq $4, X - je .L05 + je L(05) movss -32 * SIZE(X), %xmm0 andps %xmm15, %xmm0 addq $SIZE, X decq M - jle .L998 + jle L(998) ALIGN_3 -.L05: +L(05): testq $8, X - je .L10 + je L(10) movsd -32 * SIZE(X), %xmm1 andps %xmm15, %xmm1 addq $2 * SIZE, X subq $2, M - jle .L998 + jle L(998) ALIGN_3 -.L10: +L(10): movq M, I sarq $5, I - jle .L14 + jle L(14) movaps -32 * SIZE(X), %xmm4 movaps -28 * SIZE(X), %xmm5 @@ -110,10 +110,10 @@ movaps -8 * SIZE(X), %xmm10 movaps -4 * SIZE(X), %xmm11 decq I - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -156,10 +156,10 @@ subq $-32 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): andps %xmm15, %xmm4 addps %xmm4, %xmm0 andps %xmm15, %xmm5 @@ -183,9 +183,9 @@ subq $-32 * SIZE, X ALIGN_3 -.L14: +L(14): testq $16, M - je .L16 + je L(16) movaps -32 * SIZE(X), %xmm4 andps %xmm15, %xmm4 @@ -206,9 +206,9 @@ addq $16 * SIZE, X ALIGN_3 -.L16: +L(16): testq $8, M - je .L17 + je L(17) movaps -32 * SIZE(X), %xmm4 andps %xmm15, %xmm4 @@ -221,9 +221,9 @@ addq $8 * SIZE, X ALIGN_3 -.L17: +L(17): testq $4, M - je .L18 + je L(18) movaps -32 * SIZE(X), %xmm6 andps %xmm15, %xmm6 @@ -231,9 +231,9 @@ addq $4 * SIZE, X ALIGN_3 -.L18: +L(18): testq $2, M - je .L19 + je L(19) #ifdef movsd xorps %xmm7, %xmm7 @@ -244,23 +244,23 @@ addq $2 * SIZE, X ALIGN_3 -.L19: +L(19): testq $1, M - je .L998 + je L(998) movss -32 * SIZE(X), %xmm6 andps %xmm15, %xmm6 addps %xmm6, %xmm2 - jmp .L998 + jmp L(998) ALIGN_4 -.L100: +L(100): movq M, I sarq $3, I - jle .L105 + jle L(105) ALIGN_4 -.L101: +L(101): movss 0 * SIZE(X), %xmm4 addq INCX, X andps %xmm15, %xmm4 @@ -302,24 +302,24 @@ addss %xmm6, %xmm3 decq I - jg .L101 + jg L(101) ALIGN_4 -.L105: +L(105): andq $7, M - jle .L998 + jle L(998) ALIGN_4 -.L106: +L(106): movss 0 * SIZE(X), %xmm4 andps %xmm15, %xmm4 addps %xmm4, %xmm0 addq INCX, X decq M - jg .L106 + jg L(106) ALIGN_4 -.L998: +L(998): addps %xmm1, %xmm0 addps %xmm3, %xmm2 addps %xmm2, %xmm0 @@ -337,7 +337,7 @@ #endif ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/asum_sse2.S b/kernel/x86_64/asum_sse2.S index e75ebde137..35bcc1d8f7 100644 --- a/kernel/x86_64/asum_sse2.S +++ b/kernel/x86_64/asum_sse2.S @@ -54,9 +54,9 @@ xorps %xmm0, %xmm0 testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) xorps %xmm1, %xmm1 xorps %xmm2, %xmm2 @@ -70,23 +70,23 @@ subq $-16 * SIZE, X cmpq $SIZE, INCX - jne .L40 + jne L(40) testq $SIZE, X - je .L05 + je L(05) movsd -16 * SIZE(X), %xmm0 addq $SIZE, X andps %xmm15, %xmm0 subq $1, M - jle .L999 + jle L(999) ALIGN_3 -.L05: +L(05): movq M, I sarq $4, I - jle .L20 + jle L(20) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -99,10 +99,10 @@ movaps -2 * SIZE(X), %xmm11 decq I - jle .L11 + jle L(11) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -145,10 +145,10 @@ subq $-16 * SIZE, X decq I - jg .L10 + jg L(10) ALIGN_4 -.L11: +L(11): andps %xmm15, %xmm4 andps %xmm15, %xmm5 andps %xmm15, %xmm6 @@ -172,12 +172,12 @@ subq $-16 * SIZE, X ALIGN_3 -.L20: +L(20): andq $15, M - jle .L998 + jle L(998) testq $8, M - je .L21 + je L(21) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -196,9 +196,9 @@ addq $8 * SIZE, X ALIGN_3 -.L21: +L(21): testq $4, M - je .L22 + je L(22) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -211,18 +211,18 @@ addq $4 * SIZE, X ALIGN_3 -.L22: +L(22): testq $2, M - je .L23 + je L(23) movaps -16 * SIZE(X), %xmm6 andps %xmm15, %xmm6 addpd %xmm6, %xmm3 addq $2 * SIZE, X -.L23: +L(23): testq $1, M - je .L998 + je L(998) #ifdef movsd xorps %xmm4, %xmm4 @@ -230,16 +230,16 @@ movsd -16 * SIZE(X), %xmm4 andps %xmm15, %xmm4 addsd %xmm4, %xmm0 - jmp .L998 + jmp L(998) ALIGN_3 -.L40: +L(40): movq M, I sarq $3, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): movsd -16 * SIZE(X), %xmm4 addq INCX, X movhpd -16 * SIZE(X), %xmm4 @@ -269,33 +269,33 @@ addpd %xmm7, %xmm3 decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): #ifdef movsd xorps %xmm4, %xmm4 #endif andq $7, M - jle .L998 + jle L(998) ALIGN_4 -.L61: +L(61): movsd -16 * SIZE(X), %xmm4 andps %xmm15, %xmm4 addpd %xmm4, %xmm0 addq INCX, X decq M - jg .L61 + jg L(61) ALIGN_4 -.L998: +L(998): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 ALIGN_4 -.L999: +L(999): #ifndef HAVE_SSE3 movhlps %xmm0, %xmm1 addsd %xmm1, %xmm0 diff --git a/kernel/x86_64/axpy.S b/kernel/x86_64/axpy.S index 0ad6591b0f..6884122513 100644 --- a/kernel/x86_64/axpy.S +++ b/kernel/x86_64/axpy.S @@ -60,21 +60,21 @@ salq $BASE_SHIFT, INCY testq M, M - jle .L40 + jle L(40) cmpq $SIZE, INCX - jne .L14 + jne L(14) cmpq $SIZE, INCY - jne .L14 + jne L(14) movq M, %rax sarq $3, %rax - jle .L15 + jle L(15) ALIGN_3 #define PRESIZE 33 -.L16: +L(16): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -134,16 +134,16 @@ addq $8 * SIZE, X addq $8 * SIZE, Y decq %rax - jg .L16 + jg L(16) ALIGN_3 -.L15: +L(15): movq M, %rax andq $7, %rax - jle .L40 + jle L(40) ALIGN_3 -.L22: +L(22): FLD 0 * SIZE(X) fmul %st(1),%st FLD 0 * SIZE(Y) @@ -152,17 +152,17 @@ addq $SIZE, X addq $SIZE, Y decq %rax - jg .L22 - jmp .L40 + jg L(22) + jmp L(40) ALIGN_3 -.L14: +L(14): movq M, %rax sarq $2, %rax - jle .L28 + jle L(28) ALIGN_3 -.L29: +L(29): FLD (X) fmul %st(1),%st FLD (Y) @@ -196,16 +196,16 @@ addq INCY, Y decq %rax - jg .L29 + jg L(29) ALIGN_3 -.L28: +L(28): movq M, %rax andq $3, %rax - jle .L40 + jle L(40) ALIGN_3 -.L35: +L(35): FLD (X) fmul %st(1),%st FLD (Y) @@ -215,9 +215,9 @@ addq INCY, Y decq %rax - jg .L35 + jg L(35) -.L40: +L(40): ffreep %st(0) ret diff --git a/kernel/x86_64/axpy_atom.S b/kernel/x86_64/axpy_atom.S index adfd691641..565b19adc9 100644 --- a/kernel/x86_64/axpy_atom.S +++ b/kernel/x86_64/axpy_atom.S @@ -83,16 +83,16 @@ leaq (, INCY, SIZE), INCY testq M, M - jle .L29 + jle L(29) cmpq $SIZE, INCX - jne .L20 + jne L(20) cmpq $SIZE, INCY - jne .L20 + jne L(20) movq M, %rax sarq $3, %rax - jle .L13 + jle L(13) movsd 0 * SIZE(X), %xmm0 movsd 1 * SIZE(X), %xmm1 @@ -114,10 +114,10 @@ mulsd ALPHA, %xmm3 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -179,10 +179,10 @@ addq $8 * SIZE, Y addq $8 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): addsd %xmm4, %xmm0 movsd 4 * SIZE(Y), %xmm4 addsd %xmm5, %xmm1 @@ -215,10 +215,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L13: +L(13): movq M, %rax andq $4, %rax - jle .L15 + jle L(15) ALIGN_3 movsd 0 * SIZE(X), %xmm0 @@ -249,10 +249,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L15: +L(15): movq M, %rax andq $2, %rax - jle .L16 + jle L(16) ALIGN_3 movsd 0 * SIZE(X), %xmm0 @@ -272,10 +272,10 @@ addq $2 * SIZE, Y ALIGN_3 -.L16: +L(16): movq M, %rax andq $1, %rax - jle .L19 + jle L(19) ALIGN_3 movsd 0 * SIZE(X), %xmm0 @@ -286,7 +286,7 @@ addq $SIZE, Y ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -294,12 +294,12 @@ ret ALIGN_3 -.L20: +L(20): movq Y, YY movq M, %rax sarq $3, %rax - jle .L23 + jle L(23) movsd (X), %xmm0 addq INCX, X @@ -333,10 +333,10 @@ mulsd ALPHA, %xmm3 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): addsd %xmm4, %xmm0 movsd (Y), %xmm4 addq INCY, Y @@ -414,10 +414,10 @@ mulsd ALPHA, %xmm3 decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): addsd %xmm4, %xmm0 movsd (Y), %xmm4 addq INCY, Y @@ -462,10 +462,10 @@ addq INCY, YY ALIGN_3 -.L23: +L(23): movq M, %rax andq $4, %rax - jle .L25 + jle L(25) ALIGN_3 movsd (X), %xmm0 @@ -505,10 +505,10 @@ addq INCY, YY ALIGN_3 -.L25: +L(25): movq M, %rax andq $2, %rax - jle .L26 + jle L(26) ALIGN_3 movsd (X), %xmm0 @@ -531,10 +531,10 @@ addq INCY, YY ALIGN_3 -.L26: +L(26): movq M, %rax andq $1, %rax - jle .L29 + jle L(29) ALIGN_3 movsd (X), %xmm0 @@ -545,7 +545,7 @@ addq $SIZE, Y ALIGN_3 -.L29: +L(29): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/axpy_sse.S b/kernel/x86_64/axpy_sse.S index dd52a7c830..52367954ab 100644 --- a/kernel/x86_64/axpy_sse.S +++ b/kernel/x86_64/axpy_sse.S @@ -89,21 +89,21 @@ leaq (, INCY, SIZE), INCY testq M, M - jle .L19 + jle L(19) cmpq $SIZE, INCX - jne .L50 + jne L(50) cmpq $SIZE, INCY - jne .L50 + jne L(50) subq $-32 * SIZE, X subq $-32 * SIZE, Y cmpq $3, M - jle .L16 + jle L(16) testq $SIZE, Y - je .L00 + je L(00) movss -32 * SIZE(X), %xmm0 mulss ALPHA, %xmm0 @@ -112,12 +112,12 @@ addq $1 * SIZE, X addq $1 * SIZE, Y decq M - jle .L19 + jle L(19) ALIGN_3 -.L00: +L(00): testq $SIZE * 2, Y - je .L10 + je L(10) movsd -32 * SIZE(X), %xmm0 movsd -32 * SIZE(Y), %xmm4 @@ -128,16 +128,16 @@ addq $2 * SIZE, X addq $2 * SIZE, Y subq $2, M - jle .L19 + jle L(19) ALIGN_3 -.L10: +L(10): testq $SIZE * 3, X - jne .L20 + jne L(20) movq M, %rax sarq $5, %rax - jle .L13 + jle L(13) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -145,10 +145,10 @@ movaps -20 * SIZE(X), %xmm3 decq %rax - jle .L12 + jle L(12) ALIGN_4 -.L11: +L(11): movaps -16 * SIZE(X), %xmm4 movaps -12 * SIZE(X), %xmm5 @@ -212,10 +212,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movaps -16 * SIZE(X), %xmm4 movaps -12 * SIZE(X), %xmm5 @@ -258,10 +258,10 @@ subq $-32 * SIZE, Y ALIGN_3 -.L13: +L(13): movq M, %rax andq $16, %rax - jle .L14 + jle L(14) ALIGN_3 movaps -32 * SIZE(X), %xmm0 @@ -287,10 +287,10 @@ addq $16 * SIZE, Y ALIGN_3 -.L14: +L(14): movq M, %rax andq $8, %rax - jle .L15 + jle L(15) ALIGN_3 movaps -32 * SIZE(X), %xmm0 @@ -308,10 +308,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L15: +L(15): movq M, %rax andq $4, %rax - jle .L16 + jle L(16) ALIGN_3 movaps -32 * SIZE(X), %xmm0 @@ -326,10 +326,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L16: +L(16): movq M, %rax andq $2, %rax - jle .L17 + jle L(17) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -344,10 +344,10 @@ addq $2 * SIZE, Y ALIGN_3 -.L17: +L(17): movq M, %rax andq $1, %rax - jle .L19 + jle L(19) ALIGN_3 movss -32 * SIZE(X), %xmm0 @@ -357,7 +357,7 @@ movss %xmm0, -32 * SIZE(Y) ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -365,18 +365,18 @@ ret ALIGN_3 -.L20: +L(20): #ifdef ALIGNED_ACCESS testq $SIZE, X - jne .L30 + jne L(30) movhps -32 * SIZE(X), %xmm0 movq M, %rax sarq $5, %rax - jle .L23 + jle L(23) movaps -30 * SIZE(X), %xmm1 movaps -26 * SIZE(X), %xmm2 @@ -384,10 +384,10 @@ movaps -18 * SIZE(X), %xmm4 decq %rax - jle .L22 + jle L(22) ALIGN_4 -.L21: +L(21): movaps -14 * SIZE(X), %xmm5 movaps -10 * SIZE(X), %xmm6 @@ -459,10 +459,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): movaps -14 * SIZE(X), %xmm5 movaps -10 * SIZE(X), %xmm6 @@ -513,10 +513,10 @@ subq $-32 * SIZE, Y ALIGN_3 -.L23: +L(23): movq M, %rax andq $16, %rax - jle .L24 + jle L(24) ALIGN_3 movaps -30 * SIZE(X), %xmm1 @@ -549,10 +549,10 @@ addq $16 * SIZE, Y ALIGN_3 -.L24: +L(24): movq M, %rax andq $8, %rax - jle .L25 + jle L(25) ALIGN_3 movaps -30 * SIZE(X), %xmm1 @@ -573,10 +573,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L25: +L(25): movq M, %rax andq $4, %rax - jle .L26 + jle L(26) ALIGN_3 movaps -30 * SIZE(X), %xmm1 @@ -592,10 +592,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L26: +L(26): movq M, %rax andq $2, %rax - jle .L27 + jle L(27) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -610,10 +610,10 @@ addq $2 * SIZE, Y ALIGN_3 -.L27: +L(27): movq M, %rax andq $1, %rax - jle .L29 + jle L(29) ALIGN_3 movss -32 * SIZE(X), %xmm0 @@ -624,7 +624,7 @@ addq $SIZE, Y ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -632,15 +632,15 @@ ret ALIGN_3 -.L30: +L(30): testq $2 * SIZE, X - jne .L40 + jne L(40) movaps -33 * SIZE(X), %xmm0 movq M, %rax sarq $5, %rax - jle .L33 + jle L(33) movaps -29 * SIZE(X), %xmm1 movaps -25 * SIZE(X), %xmm2 @@ -648,10 +648,10 @@ movaps -17 * SIZE(X), %xmm4 decq %rax - jle .L32 + jle L(32) ALIGN_4 -.L31: +L(31): movaps -13 * SIZE(X), %xmm5 movaps -9 * SIZE(X), %xmm6 @@ -731,10 +731,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L32: +L(32): movaps -13 * SIZE(X), %xmm5 movaps -9 * SIZE(X), %xmm6 @@ -793,10 +793,10 @@ subq $-32 * SIZE, Y ALIGN_3 -.L33: +L(33): movq M, %rax andq $16, %rax - jle .L34 + jle L(34) ALIGN_3 movaps -29 * SIZE(X), %xmm1 @@ -835,10 +835,10 @@ addq $16 * SIZE, Y ALIGN_3 -.L34: +L(34): movq M, %rax andq $8, %rax - jle .L35 + jle L(35) ALIGN_3 movaps -29 * SIZE(X), %xmm1 @@ -862,10 +862,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L35: +L(35): movq M, %rax andq $4, %rax - jle .L36 + jle L(36) ALIGN_3 movaps -29 * SIZE(X), %xmm1 @@ -882,10 +882,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L36: +L(36): movq M, %rax andq $2, %rax - jle .L37 + jle L(37) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -900,10 +900,10 @@ addq $2 * SIZE, Y ALIGN_3 -.L37: +L(37): movq M, %rax andq $1, %rax - jle .L39 + jle L(39) ALIGN_3 movss -32 * SIZE(X), %xmm0 @@ -914,7 +914,7 @@ addq $SIZE, Y ALIGN_3 -.L39: +L(39): xorq %rax,%rax RESTOREREGISTERS @@ -922,12 +922,12 @@ ret ALIGN_3 -.L40: +L(40): movaps -35 * SIZE(X), %xmm0 movq M, %rax sarq $5, %rax - jle .L43 + jle L(43) movaps -31 * SIZE(X), %xmm1 movaps -27 * SIZE(X), %xmm2 @@ -935,10 +935,10 @@ movaps -19 * SIZE(X), %xmm4 decq %rax - jle .L42 + jle L(42) ALIGN_4 -.L41: +L(41): movaps -15 * SIZE(X), %xmm5 movaps -11 * SIZE(X), %xmm6 @@ -1018,10 +1018,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L42: +L(42): movaps -15 * SIZE(X), %xmm5 movaps -11 * SIZE(X), %xmm6 @@ -1080,10 +1080,10 @@ subq $-32 * SIZE, Y ALIGN_3 -.L43: +L(43): movq M, %rax andq $16, %rax - jle .L44 + jle L(44) ALIGN_3 movaps -31 * SIZE(X), %xmm1 @@ -1120,10 +1120,10 @@ addq $16 * SIZE, Y ALIGN_3 -.L44: +L(44): movq M, %rax andq $8, %rax - jle .L45 + jle L(45) ALIGN_3 movaps -31 * SIZE(X), %xmm1 @@ -1146,10 +1146,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L45: +L(45): movq M, %rax andq $4, %rax - jle .L46 + jle L(46) ALIGN_3 movaps -31 * SIZE(X), %xmm1 @@ -1166,10 +1166,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L46: +L(46): movq M, %rax andq $2, %rax - jle .L47 + jle L(47) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -1184,10 +1184,10 @@ addq $2 * SIZE, Y ALIGN_3 -.L47: +L(47): movq M, %rax andq $1, %rax - jle .L49 + jle L(49) ALIGN_3 movss -32 * SIZE(X), %xmm0 @@ -1198,7 +1198,7 @@ addq $SIZE, Y ALIGN_3 -.L49: +L(49): xorq %rax,%rax RESTOREREGISTERS @@ -1209,7 +1209,7 @@ movq M, %rax sarq $5, %rax - jle .L23 + jle L(23) movsd -32 * SIZE(X), %xmm0 movhps -30 * SIZE(X), %xmm0 @@ -1221,10 +1221,10 @@ movhps -18 * SIZE(X), %xmm3 decq %rax - jle .L22 + jle L(22) ALIGN_4 -.L21: +L(21): movsd -16 * SIZE(X), %xmm4 movhps -14 * SIZE(X), %xmm4 movsd -12 * SIZE(X), %xmm5 @@ -1296,10 +1296,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): movsd -16 * SIZE(X), %xmm4 movhps -14 * SIZE(X), %xmm4 movsd -12 * SIZE(X), %xmm5 @@ -1346,10 +1346,10 @@ subq $-32 * SIZE, Y ALIGN_3 -.L23: +L(23): movq M, %rax andq $16, %rax - jle .L24 + jle L(24) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -1380,10 +1380,10 @@ addq $16 * SIZE, Y ALIGN_3 -.L24: +L(24): movq M, %rax andq $8, %rax - jle .L25 + jle L(25) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -1403,10 +1403,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L25: +L(25): movq M, %rax andq $4, %rax - jle .L26 + jle L(26) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -1422,10 +1422,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L26: +L(26): movq M, %rax andq $2, %rax - jle .L27 + jle L(27) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -1440,10 +1440,10 @@ addq $2 * SIZE, Y ALIGN_3 -.L27: +L(27): movq M, %rax andq $1, %rax - jle .L29 + jle L(29) ALIGN_3 movss -32 * SIZE(X), %xmm0 @@ -1454,7 +1454,7 @@ addq $SIZE, Y ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -1464,20 +1464,20 @@ ALIGN_3 -.L50: +L(50): movq M, %rax movq Y, YY //If incx==0 || incy==0, avoid unloop. cmpq $0, INCX - je .L56 + je L(56) cmpq $0, INCY - je .L56 + je L(56) sarq $3, %rax - jle .L55 + jle L(55) ALIGN_3 -.L51: +L(51): movss (X), %xmm0 addq INCX, X mulss ALPHA, %xmm0 @@ -1553,16 +1553,16 @@ addq INCY, Y decq %rax - jg .L51 + jg L(51) ALIGN_3 -.L55: +L(55): movq M, %rax andq $7, %rax - jle .L59 + jle L(59) ALIGN_3 -.L56: +L(56): movss (X), %xmm0 addq INCX, X mulss ALPHA, %xmm0 @@ -1571,10 +1571,10 @@ movss %xmm0, (Y) addq INCY, Y decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L59: +L(59): xorq %rax,%rax RESTOREREGISTERS diff --git a/kernel/x86_64/axpy_sse2.S b/kernel/x86_64/axpy_sse2.S index 9b07b90204..19c6980d4a 100644 --- a/kernel/x86_64/axpy_sse2.S +++ b/kernel/x86_64/axpy_sse2.S @@ -88,15 +88,15 @@ leaq (, INCY, SIZE), INCY testq M, M - jle .L47 + jle L(47) cmpq $SIZE, INCX - jne .L40 + jne L(40) cmpq $SIZE, INCY - jne .L40 + jne L(40) testq $SIZE, Y - je .L10 + je L(10) movsd (X), %xmm0 mulsd ALPHA, %xmm0 @@ -105,19 +105,19 @@ addq $1 * SIZE, X addq $1 * SIZE, Y decq M - jle .L19 + jle L(19) ALIGN_4 -.L10: +L(10): subq $-16 * SIZE, X subq $-16 * SIZE, Y testq $SIZE, X - jne .L20 + jne L(20) movq M, %rax sarq $4, %rax - jle .L13 + jle L(13) movaps -16 * SIZE(X), %xmm0 movaps -14 * SIZE(X), %xmm1 @@ -125,10 +125,10 @@ movaps -10 * SIZE(X), %xmm3 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): movaps -8 * SIZE(X), %xmm4 movaps -6 * SIZE(X), %xmm5 @@ -192,10 +192,10 @@ subq $-16 * SIZE, Y subq $-16 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movaps -8 * SIZE(X), %xmm4 movaps -6 * SIZE(X), %xmm5 @@ -238,10 +238,10 @@ subq $-16 * SIZE, X ALIGN_3 -.L13: +L(13): movq M, %rax andq $8, %rax - jle .L14 + jle L(14) ALIGN_3 movaps -16 * SIZE(X), %xmm0 @@ -267,10 +267,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L14: +L(14): movq M, %rax andq $4, %rax - jle .L15 + jle L(15) ALIGN_3 movaps -16 * SIZE(X), %xmm0 @@ -289,10 +289,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L15: +L(15): movq M, %rax andq $2, %rax - jle .L16 + jle L(16) ALIGN_3 movaps -16 * SIZE(X), %xmm0 @@ -304,10 +304,10 @@ addq $2 * SIZE, Y ALIGN_3 -.L16: +L(16): movq M, %rax andq $1, %rax - jle .L19 + jle L(19) ALIGN_3 movsd -16 * SIZE(X), %xmm0 @@ -317,7 +317,7 @@ movsd %xmm0, -16 * SIZE(Y) ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -325,24 +325,24 @@ ret ALIGN_3 -.L20: +L(20): #ifdef ALIGNED_ACCESS movhps -16 * SIZE(X), %xmm0 movq M, %rax sarq $4, %rax - jle .L23 + jle L(23) movaps -15 * SIZE(X), %xmm1 movaps -13 * SIZE(X), %xmm2 movaps -11 * SIZE(X), %xmm3 decq %rax - jle .L22 + jle L(22) ALIGN_4 -.L21: +L(21): movaps -9 * SIZE(X), %xmm4 movaps -7 * SIZE(X), %xmm5 @@ -414,10 +414,10 @@ subq $-16 * SIZE, X subq $-16 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): movaps -9 * SIZE(X), %xmm4 movaps -7 * SIZE(X), %xmm5 @@ -469,10 +469,10 @@ subq $-16 * SIZE, Y ALIGN_3 -.L23: +L(23): movq M, %rax andq $8, %rax - jle .L24 + jle L(24) ALIGN_3 movaps -15 * SIZE(X), %xmm1 @@ -506,10 +506,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L24: +L(24): movq M, %rax andq $4, %rax - jle .L25 + jle L(25) ALIGN_3 movaps -15 * SIZE(X), %xmm1 @@ -532,10 +532,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L25: +L(25): movq M, %rax andq $2, %rax - jle .L26 + jle L(26) ALIGN_3 movaps -15 * SIZE(X), %xmm1 @@ -549,10 +549,10 @@ addq $2 * SIZE, Y ALIGN_3 -.L26: +L(26): movq M, %rax andq $1, %rax - jle .L29 + jle L(29) ALIGN_3 movsd -16 * SIZE(X), %xmm0 @@ -562,7 +562,7 @@ movsd %xmm0, -16 * SIZE(Y) ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -573,7 +573,7 @@ #else movq M, %rax sarq $4, %rax - jle .L23 + jle L(23) movsd -16 * SIZE(X), %xmm0 movhps -15 * SIZE(X), %xmm0 @@ -585,10 +585,10 @@ movhps -9 * SIZE(X), %xmm3 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): movsd -8 * SIZE(X), %xmm4 movhps -7 * SIZE(X), %xmm4 movsd -6 * SIZE(X), %xmm5 @@ -660,10 +660,10 @@ subq $-16 * SIZE, Y subq $-16 * SIZE, X decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): movsd -8 * SIZE(X), %xmm4 movhps -7 * SIZE(X), %xmm4 movsd -6 * SIZE(X), %xmm5 @@ -710,10 +710,10 @@ subq $-16 * SIZE, X ALIGN_3 -.L23: +L(23): movq M, %rax andq $8, %rax - jle .L24 + jle L(24) ALIGN_3 movsd -16 * SIZE(X), %xmm0 @@ -743,10 +743,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L24: +L(24): movq M, %rax andq $4, %rax - jle .L25 + jle L(25) ALIGN_3 movsd -16 * SIZE(X), %xmm0 @@ -767,10 +767,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L25: +L(25): movq M, %rax andq $2, %rax - jle .L26 + jle L(26) ALIGN_3 movsd -16 * SIZE(X), %xmm0 @@ -783,10 +783,10 @@ addq $2 * SIZE, Y ALIGN_3 -.L26: +L(26): movq M, %rax andq $1, %rax - jle .L29 + jle L(29) ALIGN_3 movsd -16 * SIZE(X), %xmm0 @@ -796,7 +796,7 @@ movsd %xmm0, -16 * SIZE(Y) ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -805,20 +805,20 @@ ALIGN_3 #endif -.L40: +L(40): movq Y, YY movq M, %rax //If incx==0 || incy==0, avoid unloop. cmpq $0, INCX - je .L46 + je L(46) cmpq $0, INCY - je .L46 + je L(46) sarq $3, %rax - jle .L45 + jle L(45) ALIGN_3 -.L41: +L(41): movsd 0 * SIZE(X), %xmm0 addq INCX, X movhpd 0 * SIZE(X), %xmm0 @@ -885,16 +885,16 @@ addq INCY, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L45: +L(45): movq M, %rax andq $7, %rax - jle .L47 + jle L(47) ALIGN_3 -.L46: +L(46): movsd (X), %xmm0 addq INCX, X mulsd %xmm15, %xmm0 @@ -902,10 +902,10 @@ movsd %xmm0, (Y) addq INCY, Y decq %rax - jg .L46 + jg L(46) ALIGN_3 -.L47: +L(47): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/builtin_stinit.S b/kernel/x86_64/builtin_stinit.S index cb3a288878..b33c04d64f 100644 --- a/kernel/x86_64/builtin_stinit.S +++ b/kernel/x86_64/builtin_stinit.S @@ -43,18 +43,18 @@ PROFCODE cmpq $4096, %rax - jle .L999 + jle L(999) ALIGN_3 -.L01: +L(01): subq $4096, %rax subq $4096, %rsp movq $0, (%rsp) cmpq $4096, %rax - jg .L01 + jg L(01) ALIGN_3 -.L999: +L(999): subq %rax, %rsp ret diff --git a/kernel/x86_64/cgemm_kernel_4x2_bulldozer.S b/kernel/x86_64/cgemm_kernel_4x2_bulldozer.S index 2675f71fb9..00ddb33135 100644 --- a/kernel/x86_64/cgemm_kernel_4x2_bulldozer.S +++ b/kernel/x86_64/cgemm_kernel_4x2_bulldozer.S @@ -561,13 +561,13 @@ STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -595,36 +595,36 @@ #endif #endif -.L2_0: +L(2_0): movq Ndiv6, J cmpq $0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $4*SIZE,BO1 addq $4*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -638,11 +638,11 @@ movq M, I sarq $2, I // i = (m >> 2) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -680,7 +680,7 @@ andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -691,7 +691,7 @@ negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL4x2_1(xxx) @@ -705,7 +705,7 @@ KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL4x2_1(xxx) @@ -719,12 +719,12 @@ KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -732,7 +732,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -744,14 +744,14 @@ negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL4x2_SUB(xxx) - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -847,7 +847,7 @@ addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 @@ -856,15 +856,15 @@ * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): testq $3, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -902,7 +902,7 @@ andq $-8, %rax // K = K - ( K % 8 ) - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -913,7 +913,7 @@ negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -927,7 +927,7 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -941,12 +941,12 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -954,7 +954,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -966,14 +966,14 @@ negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL2x2_SUB(xxx) - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -1053,13 +1053,13 @@ /**************************************************************************/ -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1097,7 +1097,7 @@ andq $-8, %rax // K = K - ( K % 8 ) - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1108,7 +1108,7 @@ negq %rax ALIGN_4 -.L2_42: +L(2_42): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x2_1(xxx) @@ -1122,7 +1122,7 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x2_1(xxx) @@ -1136,12 +1136,12 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -1149,7 +1149,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1161,14 +1161,14 @@ negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -1250,17 +1250,17 @@ -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -1268,30 +1268,30 @@ movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -1305,11 +1305,11 @@ movq M, I sarq $2, I // i = (m >> 2) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1347,7 +1347,7 @@ andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1358,7 +1358,7 @@ negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL4x1_1(xxx) @@ -1371,7 +1371,7 @@ KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_16 + je L(1_16) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL4x1_1(xxx) @@ -1384,12 +1384,12 @@ KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -1397,7 +1397,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 4 ; number of values @@ -1409,14 +1409,14 @@ negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL4x1_SUB(xxx) - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -1489,7 +1489,7 @@ addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 @@ -1498,15 +1498,15 @@ * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): testq $3, M - jz .L999 + jz L(999) testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1544,7 +1544,7 @@ andq $-8, %rax // K = K - ( K % 8 ) - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1555,7 +1555,7 @@ negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -1568,7 +1568,7 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_26 + je L(1_26) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -1581,12 +1581,12 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -1594,7 +1594,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2; number of values @@ -1606,14 +1606,14 @@ negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL2x1_SUB(xxx) - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -1680,13 +1680,13 @@ /**************************************************************************/ -.L1_40: +L(1_40): testq $1, M - jz .L999 // to next 2 lines of N + jz L(999) // to next 2 lines of N ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1724,7 +1724,7 @@ andq $-8, %rax // K = K - ( K % 8 ) - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1735,7 +1735,7 @@ negq %rax ALIGN_4 -.L1_42: +L(1_42): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x1_1(xxx) @@ -1748,7 +1748,7 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x1_1(xxx) @@ -1761,12 +1761,12 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -1774,7 +1774,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1786,14 +1786,14 @@ negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -1865,7 +1865,7 @@ -.L999: +L(999): vzeroupper movq SP, %rsp diff --git a/kernel/x86_64/cgemm_kernel_4x2_piledriver.S b/kernel/x86_64/cgemm_kernel_4x2_piledriver.S index bf7f91ee9e..9d1ea86cab 100644 --- a/kernel/x86_64/cgemm_kernel_4x2_piledriver.S +++ b/kernel/x86_64/cgemm_kernel_4x2_piledriver.S @@ -585,13 +585,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -619,36 +619,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif #endif -.L2_0: +L(2_0): movq Ndiv6, J cmpq $0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $4*SIZE,BO1 addq $4*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -662,11 +662,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = (m >> 2) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -704,7 +704,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -715,7 +715,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL4x2_1(xxx) @@ -729,7 +729,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL4x2_1(xxx) @@ -743,12 +743,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -756,7 +756,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -768,14 +768,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL4x2_SUB(xxx) - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -871,7 +871,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 @@ -880,15 +880,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): testq $3, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -926,7 +926,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax // K = K - ( K % 8 ) - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -937,7 +937,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -951,7 +951,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -965,12 +965,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -978,7 +978,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -990,14 +990,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL2x2_SUB(xxx) - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -1077,13 +1077,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1121,7 +1121,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax // K = K - ( K % 8 ) - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1132,7 +1132,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_42: +L(2_42): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x2_1(xxx) @@ -1146,7 +1146,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x2_1(xxx) @@ -1160,12 +1160,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -1173,7 +1173,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1185,14 +1185,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -1274,17 +1274,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -1292,30 +1292,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -1329,11 +1329,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = (m >> 2) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1371,7 +1371,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1382,7 +1382,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL4x1_1(xxx) @@ -1395,7 +1395,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_16 + je L(1_16) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL4x1_1(xxx) @@ -1408,12 +1408,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -1421,7 +1421,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 4 ; number of values @@ -1433,14 +1433,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL4x1_SUB(xxx) - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -1513,7 +1513,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 @@ -1522,15 +1522,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): testq $3, M - jz .L999 + jz L(999) testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1568,7 +1568,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax // K = K - ( K % 8 ) - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1579,7 +1579,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -1592,7 +1592,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_26 + je L(1_26) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -1605,12 +1605,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -1618,7 +1618,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2; number of values @@ -1630,14 +1630,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL2x1_SUB(xxx) - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -1704,13 +1704,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L1_40: +L(1_40): testq $1, M - jz .L999 // to next 2 lines of N + jz L(999) // to next 2 lines of N ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1748,7 +1748,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax // K = K - ( K % 8 ) - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1759,7 +1759,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_42: +L(1_42): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x1_1(xxx) @@ -1772,7 +1772,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x1_1(xxx) @@ -1785,12 +1785,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -1798,7 +1798,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1810,14 +1810,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -1889,7 +1889,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L999: +L(999): vzeroupper movq SP, %rsp diff --git a/kernel/x86_64/cgemm_kernel_4x8_sandy.S b/kernel/x86_64/cgemm_kernel_4x8_sandy.S index f64f199b7e..810d1d5946 100644 --- a/kernel/x86_64/cgemm_kernel_4x8_sandy.S +++ b/kernel/x86_64/cgemm_kernel_4x8_sandy.S @@ -283,9 +283,9 @@ movq %r11, kk; MOVQ bn,j; SARQ $2,j; # Rn = 4 -JLE .L0_loopE; +JLE L(0_loopE); ALIGN_5; -.L0_bodyB:; +L(0_bodyB):; #if defined(TRMMKERNEL) && defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk; @@ -298,9 +298,9 @@ LEAQ (bb, k, 1), prebb; # Rn=4, SIZE=4 COMPLEX=2 MOVQ ba,ptrba; MOVQ bm,i; SARQ $3,i; # Rm = 8 -JLE .L1_loopE; +JLE L(1_loopE); ALIGN_5; -.L1_bodyB:; +L(1_bodyB):; #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -345,9 +345,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2,k; # Unroll 4 times -JLE .L2_loopE; +JLE L(2_loopE); ALIGN_5; -.L2_bodyB:; +L(2_bodyB):; # Computing kernel ######### Unroll 1 ################## @@ -567,19 +567,19 @@ MUL_SY yvec1, yvec4, yvec6; MUL_SY yvec1, yvec5, yvec7; ADD2_SY yvec6, yvec10, yvec10; ADD2_SY yvec7, yvec8, yvec8; -.L2_bodyE:; +L(2_bodyE):; DECQ k; -JG .L2_bodyB; +JG L(2_bodyB); ALIGN_5 -.L2_loopE:; +L(2_loopE):; #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L3_loopE; +JLE L(3_loopE); ALIGN_5 -.L3_loopB: +L(3_loopB): ######### Unroll 1 ################## PREFETCH0 PRESIZE*SIZE(ptrba) LD_SY 8*SIZE(ptrba), yvec1; # Ar4, Ai4, Ar5, Ai5.. @@ -689,15 +689,15 @@ MUL_SY yvec1, yvec4, yvec6; MUL_SY yvec1, yvec5, yvec7; ADD2_SY yvec6, yvec10, yvec10; ADD2_SY yvec7, yvec8, yvec8; -.L3_loopE: +L(3_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L4_loopE; +JLE L(4_loopE); ALIGN_5 -.L4_loopB:; +L(4_loopB):; ######### Unroll 1 ################## PREFETCH0 PRESIZE*SIZE(ptrba) LD_SY 8*SIZE(ptrba), yvec1; # Ar4, Ai4, Ar5, Ai5.. @@ -752,7 +752,7 @@ MUL_SY yvec1, yvec5, yvec7; ADD2_SY yvec6, yvec10, yvec10; ADD2_SY yvec7, yvec8, yvec8; -.L4_loopE:; +L(4_loopE):; #### Handle #### XOR_SY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -853,7 +853,7 @@ REVS_SY $0xe4,yvec7,yvec8,yvec8; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L4_loopEx; +JNE L(4_loopEx); ALIGN_5 EXTRA_SY $1,yvec15,xvec7; EXTRA_SY $1,yvec14,xvec6; @@ -909,12 +909,12 @@ ADDQ $8, kk; #endif ADDQ $16*SIZE,C0; ADDQ $16*SIZE,C1; -.L1_bodyE:; +L(1_bodyE):; DECQ i; -JG .L1_bodyB; -JMP .L1_loopE; +JG L(1_bodyB); +JMP L(1_loopE); ALIGN_5 -.L4_loopEx: +L(4_loopEx): EXTRA_SY $1, yvec15, xvec7; #ifndef TRMMKERNEL LDL_SY 0*SIZE(C0), xvec6, xvec6; @@ -1055,13 +1055,13 @@ ADDQ $8, kk; ADDQ $16*SIZE, C0; ADDQ $16*SIZE, C1; DECQ i; -JG .L1_bodyB; +JG L(1_bodyB); ALIGN_5; -.L1_loopE:; +L(1_loopE):; TEST $4, bm; -JLE .L5_loopE; +JLE L(5_loopE); ALIGN_5 -.L5_bodyB: +L(5_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -1091,9 +1091,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L8_loopE; +JLE L(8_loopE); ALIGN_5 -.L8_bodyB: +L(8_bodyB): #### Unroll times 1 #### LD_SY 0*SIZE(ptrba), yvec0; VPERMILP_SY $0xb1, yvec0, yvec1; @@ -1220,17 +1220,17 @@ ADD2_SY yvec7, yvec9, yvec9; ADDQ $32*SIZE, ptrba; ADDQ $32*SIZE, ptrbb; DECQ k; -JG .L8_bodyB; +JG L(8_bodyB); ALIGN_5 -.L8_loopE: +L(8_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L9_loopE; +JLE L(9_loopE); ALIGN_5 -.L9_bodyB: +L(9_bodyB): #### Unroll times 1 #### LD_SY 0*SIZE(ptrba), yvec0; VPERMILP_SY $0xb1, yvec0, yvec1; @@ -1295,15 +1295,15 @@ ADD2_SY yvec7, yvec9, yvec9; ADDQ $16*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; -.L9_loopE: +L(9_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L10_loopE; +JLE L(10_loopE); ALIGN_5 -.L10_bodyB: +L(10_bodyB): #### Unroll times 1 #### LD_SY 0*SIZE(ptrba), yvec0; VPERMILP_SY $0xb1, yvec0, yvec1; @@ -1337,7 +1337,7 @@ ADD2_SY yvec7, yvec9, yvec9; ADDQ $8*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L10_loopE: +L(10_loopE): #### Handle #### XOR_SY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -1470,11 +1470,11 @@ ADDQ $4, kk; ADDQ $8*SIZE, C0; ADDQ $8*SIZE, C1; -.L5_loopE: +L(5_loopE): TEST $2, bm; -JLE .L6_loopE; +JLE L(6_loopE); ALIGN_5 -.L6_bodyB: +L(6_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -1505,9 +1505,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L11_loopE; +JLE L(11_loopE); ALIGN_5 -.L11_bodyB: +L(11_bodyB): LD_SX 0*SIZE(ptrba), xvec0; # ar1, ai1, ar2, ai2 EDUP_SX 0*SIZE(ptrbb), xvec2; # br1, br1, br2, br2 SHUF_SX $0x4e, xvec2, xvec3; # br3, br3, br4, br4 @@ -1630,17 +1630,17 @@ ADD2_SX xvec5, xvec12, xvec12; ADDQ $16*SIZE, ptrba; ADDQ $32*SIZE, ptrbb; DECQ k; -JG .L11_bodyB; +JG L(11_bodyB); ALIGN_5 -.L11_loopE: +L(11_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L12_loopE; +JLE L(12_loopE); ALIGN_5 -.L12_bodyB: +L(12_bodyB): LD_SX 0*SIZE(ptrba), xvec0; # ar1, ai1, ar2, ai2 EDUP_SX 0*SIZE(ptrbb), xvec2; # br1, br1, br2, br2 SHUF_SX $0x4e, xvec2, xvec3; # br3, br3, br4, br4 @@ -1703,15 +1703,15 @@ ADD2_SX xvec5, xvec12, xvec12; ADDQ $8*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; -.L12_loopE: +L(12_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L13_loopE; +JLE L(13_loopE); ALIGN_5 -.L13_bodyB: +L(13_bodyB): LD_SX 0*SIZE(ptrba), xvec0; # ar1, ai1, ar2, ai2 EDUP_SX 0*SIZE(ptrbb), xvec2; # br1, br1, br2, br2 SHUF_SX $0x4e, xvec2, xvec3; # br3, br3, br4, br4 @@ -1744,7 +1744,7 @@ ADD2_SX xvec5, xvec12, xvec12; ADDQ $4*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L13_loopE: +L(13_loopE): #### Handle #### #if defined(RN) || defined(RT) || defined(CN) || defined(CT) XOR_SY yvec7, yvec7, yvec7; @@ -1851,11 +1851,11 @@ ADDQ $2, kk; ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -.L6_loopE: +L(6_loopE): TEST $1, bm; -JLE .L7_loopE; +JLE L(7_loopE); ALIGN_5 -.L7_bodyB: +L(7_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -1883,9 +1883,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L14_loopE; +JLE L(14_loopE); ALIGN_5 -.L14_bodyB: +L(14_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0xb1, xvec2, xvec3; @@ -1956,17 +1956,17 @@ ADD2_SX xvec5, xvec14, xvec14; ADDQ $8*SIZE, ptrba; ADDQ $32*SIZE, ptrbb; DECQ k; -JG .L14_bodyB; +JG L(14_bodyB); ALIGN_5 -.L14_loopE: +L(14_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L15_loopE; +JLE L(15_loopE); ALIGN_5 -.L15_bodyB: +L(15_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0xb1, xvec2, xvec3; @@ -2003,15 +2003,15 @@ ADD2_SX xvec5, xvec14, xvec14; ADDQ $4*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; -.L15_loopE: +L(15_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L16_loopE; +JLE L(16_loopE); ALIGN_5 -.L16_bodyB: +L(16_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0xb1, xvec2, xvec3; @@ -2031,7 +2031,7 @@ ADD2_SX xvec5, xvec14, xvec14; ADDQ $2*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L16_loopE: +L(16_loopE): #### Handle #### #if defined(NR) || defined(NC) || defined(TR) || defined(TC) XOR_SY yvec7, yvec7, yvec7; @@ -2097,7 +2097,7 @@ ADDQ $1, kk; ADDQ $2*SIZE, C0; ADDQ $2*SIZE, C1; -.L7_loopE: +L(7_loopE): #if defined(TRMMKERNEL) && !defined(LEFT) ADDQ $4, kk; #endif @@ -2105,15 +2105,15 @@ MOVQ bk,k; SALQ $5,k; ADDQ k,bb; LEAQ (C,ldc,4),C; -.L0_bodyE:; +L(0_bodyE):; DECQ j; -JG .L0_bodyB; +JG L(0_bodyB); ALIGN_5; -.L0_loopE:; +L(0_loopE):; TEST $2, bn; -JLE .L20_loopE; +JLE L(20_loopE); ALIGN_5 -.L20_bodyB: +L(20_bodyB): #if defined(TRMMKERNEL) && defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk; @@ -2123,9 +2123,9 @@ LEAQ (C, ldc, 1), C1; MOVQ ba, ptrba; MOVQ bm, i; SARQ $3, i; -JLE .L21_loopE; +JLE L(21_loopE); ALIGN_5 -.L21_bodyB: +L(21_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -2159,9 +2159,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L211_loopE; +JLE L(211_loopE); ALIGN_5 -.L211_bodyB: +L(211_bodyB): EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; SHUF_SX $0x4e, xvec4, xvec6; @@ -2408,17 +2408,17 @@ ADD2_SX xvec3, xvec8, xvec8; ADDQ $64*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; DECQ k; -JG .L211_bodyB; +JG L(211_bodyB); ALIGN_5 -.L211_loopE: +L(211_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L212_loopE; +JLE L(212_loopE); ALIGN_5 -.L212_bodyB: +L(212_bodyB): EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; SHUF_SX $0x4e, xvec4, xvec6; @@ -2543,15 +2543,15 @@ ADD2_SX xvec3, xvec8, xvec8; ADDQ $32*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L212_loopE: +L(212_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L213_loopE; +JLE L(213_loopE); ALIGN_5 -.L213_bodyB: +L(213_bodyB): EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; SHUF_SX $0x4e, xvec4, xvec6; @@ -2615,7 +2615,7 @@ ADD2_SX xvec3, xvec8, xvec8; ADDQ $16*SIZE, ptrba; ADDQ $4*SIZE, ptrbb -.L213_loopE: +L(213_loopE): #### Handle #### #if defined(RN) || defined(RT) || defined(CN) || defined(CT) XOR_SY yvec7, yvec7, yvec7; @@ -2803,13 +2803,13 @@ ADDQ $8, kk; ADDQ $16*SIZE, C0; ADDQ $16*SIZE, C1; DECQ i; -JG .L21_bodyB; +JG L(21_bodyB); ALIGN_5 -.L21_loopE: +L(21_loopE): TEST $4, bm; -JLE .L22_loopE; +JLE L(22_loopE); ALIGN_5 -.L22_bodyB: +L(22_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -2840,9 +2840,9 @@ MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L221_loopE; +JLE L(221_loopE); ALIGN_5 -.L221_bodyB: +L(221_bodyB): EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; SHUF_SX $0x4e, xvec4, xvec6; @@ -2980,17 +2980,17 @@ ADD2_SX xvec3, xvec10, xvec10; ADDQ $32*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; DECQ k; -JG .L221_bodyB; +JG L(221_bodyB); ALIGN_5 -.L221_loopE: +L(221_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L222_loopE; +JLE L(222_loopE); ALIGN_5 -.L222_bodyB: +L(222_bodyB): EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; SHUF_SX $0x4e, xvec4, xvec6; @@ -3061,15 +3061,15 @@ ADDQ $16*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L222_loopE: +L(222_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L223_loopE; +JLE L(223_loopE); ALIGN_5 -.L223_bodyB: +L(223_bodyB): EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; SHUF_SX $0x4e, xvec4, xvec6; @@ -3105,7 +3105,7 @@ ADD2_SX xvec3, xvec10, xvec10; ADDQ $8*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L223_loopE: +L(223_loopE): #### Handle #### #if defined(RN) || defined(RT) || defined(CN) || defined(CT) XOR_SY yvec7, yvec7, yvec7; @@ -3213,11 +3213,11 @@ ADDQ $4, kk; ADDQ $8*SIZE, C0; ADDQ $8*SIZE, C1; -.L22_loopE: +L(22_loopE): TEST $2, bm; -JLE .L23_loopE; +JLE L(23_loopE); ALIGN_5 -.L23_bodyB: +L(23_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -3245,9 +3245,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L231_loopE; +JLE L(231_loopE); ALIGN_5 -.L231_bodyB: +L(231_bodyB): EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; SHUF_SX $0x4e, xvec4, xvec6; @@ -3329,17 +3329,17 @@ ADD2_SX xvec3, xvec11, xvec11; ADDQ $16*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; DECQ k; -JG .L231_bodyB; +JG L(231_bodyB); ALIGN_5 -.L231_loopE: +L(231_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L232_loopE; +JLE L(232_loopE); ALIGN_5 -.L232_bodyB: +L(232_bodyB): EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; SHUF_SX $0x4e, xvec4, xvec6; @@ -3381,15 +3381,15 @@ ADD2_SX xvec3, xvec11, xvec11; ADDQ $8*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L232_loopE: +L(232_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L233_loopE; +JLE L(233_loopE); ALIGN_5 -.L233_bodyB: +L(233_bodyB): EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; SHUF_SX $0x4e, xvec4, xvec6; @@ -3411,7 +3411,7 @@ ADD2_SX xvec3, xvec11, xvec11; ADDQ $4*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L233_loopE: +L(233_loopE): #### Handle #### #if defined(RN) || defined(RT) || defined(CN) || defined(CT) XOR_SY yvec7, yvec7, yvec7; @@ -3479,11 +3479,11 @@ ADDQ $2, kk; ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -.L23_loopE: +L(23_loopE): TEST $1, bm; -JLE .L24_loopE; +JLE L(24_loopE); ALIGN_5 -.L24_bodyB: +L(24_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -3510,9 +3510,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L241_loopE; +JLE L(241_loopE); ALIGN_5 -.L241_bodyB: +L(241_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0xb1, xvec2, xvec3; @@ -3555,17 +3555,17 @@ ADD2_SX xvec3, xvec15, xvec15; ADDQ $8*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; DECQ k; -JG .L241_bodyB; +JG L(241_bodyB); ALIGN_5 -.L241_loopE: +L(241_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L242_loopE; +JLE L(242_loopE); ALIGN_5 -.L242_bodyB: +L(242_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0xb1, xvec2, xvec3; @@ -3588,15 +3588,15 @@ ADD2_SX xvec3, xvec15, xvec15; ADDQ $4*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L242_loopE: +L(242_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L243_loopE; +JLE L(243_loopE); ALIGN_5 -.L243_bodyB: +L(243_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0xb1, xvec2, xvec3; @@ -3609,7 +3609,7 @@ ADD2_SX xvec3, xvec15, xvec15; ADDQ $2*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L243_loopE: +L(243_loopE): #### Handle #### XOR_SY yvec7, yvec7, yvec7; #if defined(NR) || defined(NC) || defined(TR) || defined(TC) @@ -3652,7 +3652,7 @@ ADDQ $1, kk; #endif ADDQ $2*SIZE, C0; ADDQ $2*SIZE, C1; -.L24_loopE: +L(24_loopE): #if defined(TRMMKERNEL) && !defined(LEFT) ADDQ $2, kk; #endif @@ -3660,11 +3660,11 @@ MOVQ bk, k; SALQ $4, k; ADDQ k, bb; LEAQ (C, ldc, 2), C; -.L20_loopE: +L(20_loopE): TEST $1, bn; -JLE .L30_loopE; +JLE L(30_loopE); ALIGN_5 -.L30_bodyB: +L(30_bodyB): #if defined(TRMMKERNEL) && defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk; @@ -3673,9 +3673,9 @@ MOVQ C, C0; MOVQ ba, ptrba; MOVQ bm, i; SARQ $3, i; -JLE .L31_loopE; +JLE L(31_loopE); ALIGN_5 -.L31_bodyB: +L(31_bodyB): MOVQ bb, ptrbb; #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; @@ -3705,9 +3705,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L311_loopE; +JLE L(311_loopE); ALIGN_5 -.L311_bodyB: +L(311_bodyB): #### Unroll 1 #### LD_SY 0*SIZE(ptrba), yvec0; LD_SY 8*SIZE(ptrba), yvec1; @@ -3778,17 +3778,17 @@ ADD2_SY yvec7, yvec14, yvec14; ADDQ $64*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L311_bodyB; +JG L(311_bodyB); ALIGN_5 -.L311_loopE: +L(311_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L312_loopE; +JLE L(312_loopE); ALIGN_5 -.L312_bodyB: +L(312_bodyB): #### Unroll 1 #### LD_SY 0*SIZE(ptrba), yvec0; LD_SY 8*SIZE(ptrba), yvec1; @@ -3825,15 +3825,15 @@ ADD2_SY yvec7, yvec14, yvec14; ADDQ $32*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L312_loopE: +L(312_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L313_loopE; +JLE L(313_loopE); ALIGN_5 -.L313_bodyB: +L(313_bodyB): #### Unroll 1 #### LD_SY 0*SIZE(ptrba), yvec0; LD_SY 8*SIZE(ptrba), yvec1; @@ -3853,7 +3853,7 @@ ADD2_SY yvec7, yvec14, yvec14; ADDQ $16*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L313_loopE: +L(313_loopE): #### Handle #### XOR_SY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -3919,13 +3919,13 @@ ADDQ $8, kk; #endif ADDQ $16*SIZE, C0; DECQ i; -JG .L31_bodyB; +JG L(31_bodyB); ALIGN_5 -.L31_loopE: +L(31_loopE): TEST $4, bm; -JLE .L32_loopE; +JLE L(32_loopE); ALIGN_5 -.L32_bodyB: +L(32_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -3952,9 +3952,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L321_loopE; +JLE L(321_loopE); ALIGN_5 -.L321_bodyB: +L(321_bodyB): #### Unroll 1 #### LD_SY 0*SIZE(ptrba), yvec0; BROAD_SY 0*SIZE(ptrbb), yvec2; @@ -4001,17 +4001,17 @@ ADD2_SY yvec6, yvec15, yvec15; ADDQ $32*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L321_bodyB; +JG L(321_bodyB); ALIGN_5 -.L321_loopE: +L(321_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L322_loopE; +JLE L(322_loopE); ALIGN_5 -.L322_bodyB: +L(322_bodyB): #### Unroll 1 #### LD_SY 0*SIZE(ptrba), yvec0; BROAD_SY 0*SIZE(ptrbb), yvec2; @@ -4036,15 +4036,15 @@ ADD2_SY yvec6, yvec15, yvec15; ADDQ $16*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L322_loopE: +L(322_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L323_loopE; +JLE L(323_loopE); ALIGN_5 -.L323_bodyB: +L(323_bodyB): #### Unroll 1 #### LD_SY 0*SIZE(ptrba), yvec0; BROAD_SY 0*SIZE(ptrbb), yvec2; @@ -4058,7 +4058,7 @@ ADD2_SY yvec6, yvec15, yvec15; ADDQ $8*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L323_loopE: +L(323_loopE): #### Handle #### XOR_SY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -4104,11 +4104,11 @@ ADDQ $4, kk; #endif ADDQ $8*SIZE, C0; -.L32_loopE: +L(32_loopE): TEST $2, bm; -JLE .L33_loopE; +JLE L(33_loopE); ALIGN_5 -.L33_bodyB: +L(33_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -4135,9 +4135,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L331_loopE; +JLE L(331_loopE); ALIGN_5 -.L331_bodyB: +L(331_bodyB): #### Unroll 1 #### LD_SX 0*SIZE(ptrba), xvec0; BROAD_SX 0*SIZE(ptrbb), xvec2; @@ -4180,17 +4180,17 @@ ADD2_SX xvec3, xvec15, xvec15; ADDQ $16*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L331_bodyB; +JG L(331_bodyB); ALIGN_5 -.L331_loopE: +L(331_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L332_loopE; +JLE L(332_loopE); ALIGN_5 -.L332_bodyB: +L(332_bodyB): #### Unroll 1 #### LD_SX 0*SIZE(ptrba), xvec0; BROAD_SX 0*SIZE(ptrbb), xvec2; @@ -4213,15 +4213,15 @@ ADD2_SX xvec3, xvec15, xvec15; ADDQ $8*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L332_loopE: +L(332_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L333_loopE; +JLE L(333_loopE); ALIGN_5 -.L333_bodyB: +L(333_bodyB): #### Unroll 1 #### LD_SX 0*SIZE(ptrba), xvec0; BROAD_SX 0*SIZE(ptrbb), xvec2; @@ -4234,7 +4234,7 @@ ADD2_SX xvec3, xvec15, xvec15; ADDQ $4*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L333_loopE: +L(333_loopE): #### Handle #### XOR_SY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -4276,11 +4276,11 @@ ADDQ $2, kk; #endif ADDQ $4*SIZE, C0; -.L33_loopE: +L(33_loopE): TEST $1, bm; -JLE .L34_loopE; +JLE L(34_loopE); ALIGN_5 -.L34_bodyB: +L(34_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -4307,9 +4307,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L341_loopE; +JLE L(341_loopE); ALIGN_5 -.L341_bodyB: +L(341_bodyB): LD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0xa0, xvec2, xvec3; @@ -4332,17 +4332,17 @@ ADD2_SX xvec4, xvec15, xvec15; ADDQ $8*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L341_bodyB; +JG L(341_bodyB); ALIGN_5 -.L341_loopE: +L(341_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L342_loopE; +JLE L(342_loopE); ALIGN_5 -.L342_bodyB: +L(342_bodyB): LD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0xa0, xvec2, xvec3; @@ -4355,15 +4355,15 @@ ADD2_SX xvec4, xvec15, xvec15; ADDQ $4*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L342_loopE: +L(342_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L343_loopE; +JLE L(343_loopE); ALIGN_5 -.L343_bodyB: +L(343_bodyB): XOR_SY yvec0, yvec0, yvec0; XOR_SY yvec2, yvec2, yvec2; LDL_SX 0*SIZE(ptrba), xvec0, xvec0; @@ -4378,7 +4378,7 @@ ADD2_SX xvec4, xvec15, xvec15; ADDQ $2*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L343_loopE: +L(343_loopE): #### Handle #### XOR_SY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -4419,7 +4419,7 @@ ADDQ $1, kk; #endif ADDQ $2*SIZE, C0; -.L34_loopE: +L(34_loopE): #if defined(TRMMKERNEL) && !defined(LEFT) ADDQ $1, kk; #endif @@ -4428,7 +4428,7 @@ MOVQ bk, k; SALQ $3, k; ADDQ k, bb; ADDQ ldc, C; -.L30_loopE: +L(30_loopE): movq 0(%rsp), %rbx; movq 8(%rsp), %rbp; movq 16(%rsp), %r12; diff --git a/kernel/x86_64/cgemm_kernel_8x2_haswell.S b/kernel/x86_64/cgemm_kernel_8x2_haswell.S index a608071dbe..92243e10b0 100644 --- a/kernel/x86_64/cgemm_kernel_8x2_haswell.S +++ b/kernel/x86_64/cgemm_kernel_8x2_haswell.S @@ -1268,13 +1268,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $ 0, OLD_M - je .L999 + je L(999) cmpq $ 0, OLD_N - je .L999 + je L(999) cmpq $ 0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -1294,16 +1294,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /************************************************************************************************/ -.L6_0: +L(6_0): movq Ndiv6, J cmpq $ 0, J - je .L2_00 + je L(2_00) ALIGN_4 -.L6_01: +L(6_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO @@ -1315,7 +1315,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L6_02b: +L(6_02b): vmovups (BO1), %xmm0 vmovsd (BO2), %xmm1 @@ -1325,10 +1325,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4*SIZE,BO2 addq $ 6*SIZE,BO decq %rax - jnz .L6_02b + jnz L(6_02b) -.L6_10: +L(6_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc leaq (C, LDC, 1), C // c += 1 * ldc @@ -1338,12 +1338,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 3, I // i = (m >> 3) - je .L6_4_10 + je L(6_4_10) ALIGN_4 /**********************************************************************************************************/ -.L6_8_11: +L(6_8_11): leaq BUFFER1, BO // first buffer to BO addq $ 8 * SIZE, BO @@ -1353,11 +1353,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L6_8_16 + je L(6_8_16) ALIGN_4 -.L6_8_12: +L(6_8_12): KERNEL8x3_SUB KERNEL8x3_SUB @@ -1369,7 +1369,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_SUB KERNEL8x3_SUB - je .L6_8_16 + je L(6_8_16) KERNEL8x3_SUB KERNEL8x3_SUB @@ -1381,50 +1381,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_SUB KERNEL8x3_SUB - je .L6_8_16 + je L(6_8_16) - jmp .L6_8_12 + jmp L(6_8_12) ALIGN_4 -.L6_8_16: +L(6_8_16): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L6_8_19 + je L(6_8_19) ALIGN_4 -.L6_8_17: +L(6_8_17): KERNEL8x3_SUB - jnz .L6_8_17 + jnz L(6_8_17) ALIGN_4 -.L6_8_19: +L(6_8_19): SAVE8x3 addq $ 16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L6_8_11 + jg L(6_8_11) ALIGN_4 /**********************************************************************************************************/ -.L6_4_10: +L(6_4_10): testq $ 7, M - jz .L6_4_60 // to next 2 lines of N + jz L(6_4_60) // to next 2 lines of N testq $ 4, M - jz .L6_4_20 + jz L(6_4_20) ALIGN_4 -.L6_4_11: +L(6_4_11): leaq BUFFER1, BO // first buffer to BO addq $ 8 * SIZE, BO @@ -1434,11 +1434,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L6_4_16 + je L(6_4_16) ALIGN_4 -.L6_4_12: +L(6_4_12): prefetcht0 A_PR1(AO) KERNEL4x3_SUB @@ -1454,7 +1454,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_SUB KERNEL4x3_SUB - je .L6_4_16 + je L(6_4_16) prefetcht0 A_PR1(AO) KERNEL4x3_SUB @@ -1470,28 +1470,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_SUB KERNEL4x3_SUB - je .L6_4_16 + je L(6_4_16) - jmp .L6_4_12 + jmp L(6_4_12) ALIGN_4 -.L6_4_16: +L(6_4_16): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L6_4_19 + je L(6_4_19) ALIGN_4 -.L6_4_17: +L(6_4_17): KERNEL4x3_SUB - jnz .L6_4_17 + jnz L(6_4_17) ALIGN_4 -.L6_4_19: +L(6_4_19): SAVE4x3 @@ -1504,13 +1504,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Rest of M ***************************************************************************/ -.L6_4_20: +L(6_4_20): testq $ 2, M - jz .L6_4_40 + jz L(6_4_40) ALIGN_4 -.L6_4_21: +L(6_4_21): leaq BUFFER1, BO // first buffer to BO addq $ 8 * SIZE, BO @@ -1520,11 +1520,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L6_4_26 + je L(6_4_26) ALIGN_4 -.L6_4_22: +L(6_4_22): prefetcht0 A_PR1(AO) KERNEL2x3_SUB @@ -1538,7 +1538,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_SUB KERNEL2x3_SUB - je .L6_4_26 + je L(6_4_26) prefetcht0 A_PR1(AO) KERNEL2x3_SUB @@ -1552,46 +1552,46 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_SUB KERNEL2x3_SUB - je .L6_4_26 + je L(6_4_26) - jmp .L6_4_22 + jmp L(6_4_22) ALIGN_4 -.L6_4_26: +L(6_4_26): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L6_4_29 + je L(6_4_29) ALIGN_4 -.L6_4_27: +L(6_4_27): KERNEL2x3_SUB - jnz .L6_4_27 + jnz L(6_4_27) ALIGN_4 -.L6_4_29: +L(6_4_29): SAVE2x3 addq $ 4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L6_4_21 + jg L(6_4_21) ALIGN_4 /**************************************************************************/ -.L6_4_40: +L(6_4_40): testq $ 1, M - jz .L6_4_60 // to next 2 lines of N + jz L(6_4_60) // to next 2 lines of N ALIGN_4 -.L6_4_41: +L(6_4_41): leaq BUFFER1, BO // first buffer to BO addq $ 8 * SIZE, BO @@ -1601,11 +1601,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L6_4_46 + je L(6_4_46) ALIGN_4 -.L6_4_42: +L(6_4_42): prefetcht0 A_PR1(AO) KERNEL1x3_SUB @@ -1618,7 +1618,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_SUB KERNEL1x3_SUB - je .L6_4_46 + je L(6_4_46) prefetcht0 A_PR1(AO) KERNEL1x3_SUB @@ -1631,44 +1631,44 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_SUB KERNEL1x3_SUB - je .L6_4_46 + je L(6_4_46) - jmp .L6_4_42 + jmp L(6_4_42) ALIGN_4 -.L6_4_46: +L(6_4_46): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L6_4_49 + je L(6_4_49) ALIGN_4 -.L6_4_47: +L(6_4_47): KERNEL1x3_SUB - jnz .L6_4_47 + jnz L(6_4_47) ALIGN_4 -.L6_4_49: +L(6_4_49): SAVE1x3 addq $ 2 * SIZE, CO1 # coffset += 2 decq I # i -- - jg .L6_4_41 + jg L(6_4_41) ALIGN_4 -.L6_4_60: +L(6_4_60): /*******************************************************************************************/ -.L7_01: +L(7_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO @@ -1679,7 +1679,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L7_02b: +L(7_02b): vmovsd 2*SIZE(BO1), %xmm0 vmovups (BO2), %xmm1 @@ -1689,11 +1689,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4*SIZE,BO2 addq $ 6*SIZE,BO decq %rax - jnz .L7_02b + jnz L(7_02b) movq BO2, B // next offset of B -.L7_10: +L(7_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc leaq (C, LDC, 1), C // c += 1 * ldc @@ -1703,12 +1703,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 3, I // i = (m >> 3) - je .L7_4_10 + je L(7_4_10) ALIGN_4 /**********************************************************************************************************/ -.L7_8_11: +L(7_8_11): leaq BUFFER1, BO // first buffer to BO addq $ 8 * SIZE, BO @@ -1718,11 +1718,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L7_8_16 + je L(7_8_16) ALIGN_4 -.L7_8_12: +L(7_8_12): KERNEL8x3_SUB KERNEL8x3_SUB @@ -1734,7 +1734,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_SUB KERNEL8x3_SUB - je .L7_8_16 + je L(7_8_16) KERNEL8x3_SUB KERNEL8x3_SUB @@ -1746,50 +1746,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_SUB KERNEL8x3_SUB - je .L7_8_16 + je L(7_8_16) - jmp .L7_8_12 + jmp L(7_8_12) ALIGN_4 -.L7_8_16: +L(7_8_16): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L7_8_19 + je L(7_8_19) ALIGN_4 -.L7_8_17: +L(7_8_17): KERNEL8x3_SUB - jnz .L7_8_17 + jnz L(7_8_17) ALIGN_4 -.L7_8_19: +L(7_8_19): SAVE8x3 addq $ 16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L7_8_11 + jg L(7_8_11) ALIGN_4 /**********************************************************************************************************/ -.L7_4_10: +L(7_4_10): testq $ 7, M - jz .L7_4_60 // to next 2 lines of N + jz L(7_4_60) // to next 2 lines of N testq $ 4, M - jz .L7_4_20 + jz L(7_4_20) ALIGN_4 -.L7_4_11: +L(7_4_11): leaq BUFFER1, BO // first buffer to BO addq $ 8 * SIZE, BO @@ -1799,11 +1799,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L7_4_16 + je L(7_4_16) ALIGN_4 -.L7_4_12: +L(7_4_12): prefetcht0 A_PR1(AO) KERNEL4x3_SUB @@ -1819,7 +1819,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_SUB KERNEL4x3_SUB - je .L7_4_16 + je L(7_4_16) prefetcht0 A_PR1(AO) KERNEL4x3_SUB @@ -1835,28 +1835,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_SUB KERNEL4x3_SUB - je .L7_4_16 + je L(7_4_16) - jmp .L7_4_12 + jmp L(7_4_12) ALIGN_4 -.L7_4_16: +L(7_4_16): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L7_4_19 + je L(7_4_19) ALIGN_4 -.L7_4_17: +L(7_4_17): KERNEL4x3_SUB - jnz .L7_4_17 + jnz L(7_4_17) ALIGN_4 -.L7_4_19: +L(7_4_19): SAVE4x3 @@ -1869,13 +1869,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Rest of M ***************************************************************************/ -.L7_4_20: +L(7_4_20): testq $ 2, M - jz .L7_4_40 + jz L(7_4_40) ALIGN_4 -.L7_4_21: +L(7_4_21): leaq BUFFER1, BO // first buffer to BO addq $ 8 * SIZE, BO @@ -1885,11 +1885,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L7_4_26 + je L(7_4_26) ALIGN_4 -.L7_4_22: +L(7_4_22): prefetcht0 A_PR1(AO) KERNEL2x3_SUB @@ -1903,7 +1903,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_SUB KERNEL2x3_SUB - je .L7_4_26 + je L(7_4_26) prefetcht0 A_PR1(AO) KERNEL2x3_SUB @@ -1917,46 +1917,46 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_SUB KERNEL2x3_SUB - je .L7_4_26 + je L(7_4_26) - jmp .L7_4_22 + jmp L(7_4_22) ALIGN_4 -.L7_4_26: +L(7_4_26): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L7_4_29 + je L(7_4_29) ALIGN_4 -.L7_4_27: +L(7_4_27): KERNEL2x3_SUB - jnz .L7_4_27 + jnz L(7_4_27) ALIGN_4 -.L7_4_29: +L(7_4_29): SAVE2x3 addq $ 4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L7_4_21 + jg L(7_4_21) ALIGN_4 /**************************************************************************/ -.L7_4_40: +L(7_4_40): testq $ 1, M - jz .L7_4_60 // to next 2 lines of N + jz L(7_4_60) // to next 2 lines of N ALIGN_4 -.L7_4_41: +L(7_4_41): leaq BUFFER1, BO // first buffer to BO addq $ 8 * SIZE, BO @@ -1966,11 +1966,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L7_4_46 + je L(7_4_46) ALIGN_4 -.L7_4_42: +L(7_4_42): prefetcht0 A_PR1(AO) KERNEL1x3_SUB @@ -1983,7 +1983,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_SUB KERNEL1x3_SUB - je .L7_4_46 + je L(7_4_46) prefetcht0 A_PR1(AO) KERNEL1x3_SUB @@ -1996,78 +1996,78 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_SUB KERNEL1x3_SUB - je .L7_4_46 + je L(7_4_46) - jmp .L7_4_42 + jmp L(7_4_42) ALIGN_4 -.L7_4_46: +L(7_4_46): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L7_4_49 + je L(7_4_49) ALIGN_4 -.L7_4_47: +L(7_4_47): KERNEL1x3_SUB - jnz .L7_4_47 + jnz L(7_4_47) ALIGN_4 -.L7_4_49: +L(7_4_49): SAVE1x3 addq $ 2 * SIZE, CO1 # coffset += 2 decq I # i -- - jg .L7_4_41 + jg L(7_4_41) ALIGN_4 -.L7_4_60: +L(7_4_60): decq J // j -- - jg .L6_01 // next 6 lines of N + jg L(6_01) // next 6 lines of N /************************************************************************************************/ -.L2_00: +L(2_00): movq Nmod6, J sarq $1, J // j = j / 2 cmpq $ 0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $ 4*SIZE,BO1 addq $ 4*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -2081,12 +2081,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 3, I // i = (m >> 3) - je .L2_4_10 + je L(2_4_10) ALIGN_4 /**********************************************************************************************************/ -.L2_8_11: +L(2_8_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2124,7 +2124,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_8_16 + je L(2_8_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2135,7 +2135,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_8_12: +L(2_8_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2157,7 +2157,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL8x2_SUB - je .L2_8_16 + je L(2_8_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2179,12 +2179,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL8x2_SUB - je .L2_8_16 + je L(2_8_16) - jmp .L2_8_12 + jmp L(2_8_12) ALIGN_4 -.L2_8_16: +L(2_8_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -2192,7 +2192,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_8_19 + je L(2_8_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2204,15 +2204,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_8_17: +L(2_8_17): KERNEL8x2_SUB - jl .L2_8_17 + jl L(2_8_17) ALIGN_4 -.L2_8_19: +L(2_8_19): SAVE8x2 @@ -2235,7 +2235,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_8_11 + jg L(2_8_11) ALIGN_4 @@ -2244,16 +2244,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_4_10: +L(2_4_10): testq $ 7, M - jz .L2_4_60 // to next 2 lines of N + jz L(2_4_60) // to next 2 lines of N testq $ 4, M - jz .L2_4_20 + jz L(2_4_20) ALIGN_4 -.L2_4_11: +L(2_4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2291,7 +2291,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_4_16 + je L(2_4_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2302,7 +2302,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_12: +L(2_4_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2320,7 +2320,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_4_16 + je L(2_4_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2338,12 +2338,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_4_16 + je L(2_4_16) - jmp .L2_4_12 + jmp L(2_4_12) ALIGN_4 -.L2_4_16: +L(2_4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -2351,7 +2351,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_4_19 + je L(2_4_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2363,15 +2363,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_17: +L(2_4_17): KERNEL4x2_SUB - jl .L2_4_17 + jl L(2_4_17) ALIGN_4 -.L2_4_19: +L(2_4_19): SAVE4x2 @@ -2400,13 +2400,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Rest of M ***************************************************************************/ -.L2_4_20: +L(2_4_20): testq $ 2, M - jz .L2_4_40 + jz L(2_4_40) ALIGN_4 -.L2_4_21: +L(2_4_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2444,7 +2444,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_4_26 + je L(2_4_26) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2455,7 +2455,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_22: +L(2_4_22): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2471,7 +2471,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_4_26 + je L(2_4_26) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2487,12 +2487,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_4_26 + je L(2_4_26) - jmp .L2_4_22 + jmp L(2_4_22) ALIGN_4 -.L2_4_26: +L(2_4_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -2500,7 +2500,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_4_29 + je L(2_4_29) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2512,15 +2512,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_27: +L(2_4_27): KERNEL2x2_SUB - jl .L2_4_27 + jl L(2_4_27) ALIGN_4 -.L2_4_29: +L(2_4_29): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -2596,19 +2596,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L2_4_21 + jg L(2_4_21) ALIGN_4 /**************************************************************************/ -.L2_4_40: +L(2_4_40): testq $ 1, M - jz .L2_4_60 // to next 2 lines of N + jz L(2_4_60) // to next 2 lines of N ALIGN_4 -.L2_4_41: +L(2_4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2646,7 +2646,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_4_46 + je L(2_4_46) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2657,7 +2657,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_42: +L(2_4_42): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2672,7 +2672,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_4_46 + je L(2_4_46) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2687,12 +2687,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_4_46 + je L(2_4_46) - jmp .L2_4_42 + jmp L(2_4_42) ALIGN_4 -.L2_4_46: +L(2_4_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -2700,7 +2700,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_4_49 + je L(2_4_49) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2712,15 +2712,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_47: +L(2_4_47): KERNEL1x2_SUB - jl .L2_4_47 + jl L(2_4_47) ALIGN_4 -.L2_4_49: +L(2_4_49): SAVE1x2 @@ -2742,23 +2742,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 2 * SIZE, CO1 # coffset += 2 decq I # i -- - jg .L2_4_41 + jg L(2_4_41) ALIGN_4 -.L2_4_60: +L(2_4_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $ 2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -2766,30 +2766,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $ 1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $ 2*SIZE,BO1 addq $ 2*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -2803,13 +2803,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 3, I // i = (m >> 3) - je .L1_4_10 + je L(1_4_10) ALIGN_4 /**************************************************************************************************/ -.L1_8_11: +L(1_8_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2847,7 +2847,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_8_16 + je L(1_8_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -2858,7 +2858,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_8_12: +L(1_8_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2879,7 +2879,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL8x1_SUB - je .L1_8_16 + je L(1_8_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2900,12 +2900,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL8x1_SUB - je .L1_8_16 + je L(1_8_16) - jmp .L1_8_12 + jmp L(1_8_12) ALIGN_4 -.L1_8_16: +L(1_8_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -2913,7 +2913,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_8_19 + je L(1_8_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 4 ; number of values @@ -2925,15 +2925,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_8_17: +L(1_8_17): KERNEL8x1_SUB - jl .L1_8_17 + jl L(1_8_17) ALIGN_4 -.L1_8_19: +L(1_8_19): SAVE8x1 @@ -2956,22 +2956,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_8_11 + jg L(1_8_11) ALIGN_4 /**************************************************************************************************/ -.L1_4_10: +L(1_4_10): testq $ 7, M - jz .L999 + jz L(999) testq $ 4, M - jz .L1_4_20 + jz L(1_4_20) -.L1_4_11: +L(1_4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3009,7 +3009,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_4_16 + je L(1_4_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -3020,7 +3020,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_12: +L(1_4_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3037,7 +3037,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_4_16 + je L(1_4_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3054,12 +3054,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_4_16 + je L(1_4_16) - jmp .L1_4_12 + jmp L(1_4_12) ALIGN_4 -.L1_4_16: +L(1_4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3067,7 +3067,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_4_19 + je L(1_4_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 4 ; number of values @@ -3079,15 +3079,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_17: +L(1_4_17): KERNEL4x1_SUB - jl .L1_4_17 + jl L(1_4_17) ALIGN_4 -.L1_4_19: +L(1_4_19): SAVE4x1 @@ -3116,13 +3116,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Rest of M ***************************************************************************/ -.L1_4_20: +L(1_4_20): testq $ 2, M - jz .L1_4_40 + jz L(1_4_40) ALIGN_4 -.L1_4_21: +L(1_4_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3160,7 +3160,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_4_26 + je L(1_4_26) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -3171,7 +3171,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_22: +L(1_4_22): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3186,7 +3186,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_4_26 + je L(1_4_26) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3201,12 +3201,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_4_26 + je L(1_4_26) - jmp .L1_4_22 + jmp L(1_4_22) ALIGN_4 -.L1_4_26: +L(1_4_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -3214,7 +3214,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_4_29 + je L(1_4_29) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2; number of values @@ -3226,15 +3226,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_27: +L(1_4_27): KERNEL2x1_SUB - jl .L1_4_27 + jl L(1_4_27) ALIGN_4 -.L1_4_29: +L(1_4_29): SAVE2x1 @@ -3260,13 +3260,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L1_4_40: +L(1_4_40): testq $ 1, M - jz .L999 // to next 2 lines of N + jz L(999) // to next 2 lines of N ALIGN_4 -.L1_4_41: +L(1_4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3304,7 +3304,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_4_46 + je L(1_4_46) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -3315,7 +3315,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_42: +L(1_4_42): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3329,7 +3329,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_4_46 + je L(1_4_46) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3343,12 +3343,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_4_46 + je L(1_4_46) - jmp .L1_4_42 + jmp L(1_4_42) ALIGN_4 -.L1_4_46: +L(1_4_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -3356,7 +3356,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_4_49 + je L(1_4_49) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -3368,15 +3368,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_47: +L(1_4_47): KERNEL1x1_SUB - jl .L1_4_47 + jl L(1_4_47) ALIGN_4 -.L1_4_49: +L(1_4_49): SAVE1x1 @@ -3402,7 +3402,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): vzeroupper movq SP, %rsp @@ -3493,13 +3493,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $ 0, OLD_M - je .L999 + je L(999) cmpq $ 0, OLD_N - je .L999 + je L(999) cmpq $ 0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -3527,36 +3527,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif #endif -.L2_0: +L(2_0): movq Ndiv6, J cmpq $ 0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $ 4*SIZE,BO1 addq $ 4*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -3570,12 +3570,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 3, I // i = (m >> 3) - je .L2_4_10 + je L(2_4_10) ALIGN_4 /**********************************************************************************************************/ -.L2_8_11: +L(2_8_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3613,7 +3613,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_8_16 + je L(2_8_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -3624,7 +3624,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_8_12: +L(2_8_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3646,7 +3646,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL8x2_SUB - je .L2_8_16 + je L(2_8_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3668,12 +3668,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL8x2_SUB - je .L2_8_16 + je L(2_8_16) - jmp .L2_8_12 + jmp L(2_8_12) ALIGN_4 -.L2_8_16: +L(2_8_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3681,7 +3681,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_8_19 + je L(2_8_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -3693,15 +3693,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_8_17: +L(2_8_17): KERNEL8x2_SUB - jl .L2_8_17 + jl L(2_8_17) ALIGN_4 -.L2_8_19: +L(2_8_19): SAVE8x2 @@ -3724,7 +3724,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_8_11 + jg L(2_8_11) ALIGN_4 @@ -3733,16 +3733,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_4_10: +L(2_4_10): testq $ 7, M - jz .L2_4_60 // to next 2 lines of N + jz L(2_4_60) // to next 2 lines of N testq $ 4, M - jz .L2_4_20 + jz L(2_4_20) ALIGN_4 -.L2_4_11: +L(2_4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3780,7 +3780,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_4_16 + je L(2_4_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -3791,7 +3791,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_12: +L(2_4_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3809,7 +3809,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_4_16 + je L(2_4_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3827,12 +3827,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_4_16 + je L(2_4_16) - jmp .L2_4_12 + jmp L(2_4_12) ALIGN_4 -.L2_4_16: +L(2_4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3840,7 +3840,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_4_19 + je L(2_4_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -3852,15 +3852,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_17: +L(2_4_17): KERNEL4x2_SUB - jl .L2_4_17 + jl L(2_4_17) ALIGN_4 -.L2_4_19: +L(2_4_19): SAVE4x2 @@ -3889,13 +3889,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Rest of M ***************************************************************************/ -.L2_4_20: +L(2_4_20): testq $ 2, M - jz .L2_4_40 + jz L(2_4_40) ALIGN_4 -.L2_4_21: +L(2_4_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3933,7 +3933,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_4_26 + je L(2_4_26) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -3944,7 +3944,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_22: +L(2_4_22): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3960,7 +3960,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_4_26 + je L(2_4_26) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3976,12 +3976,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_4_26 + je L(2_4_26) - jmp .L2_4_22 + jmp L(2_4_22) ALIGN_4 -.L2_4_26: +L(2_4_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -3989,7 +3989,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_4_29 + je L(2_4_29) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -4001,15 +4001,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_27: +L(2_4_27): KERNEL2x2_SUB - jl .L2_4_27 + jl L(2_4_27) ALIGN_4 -.L2_4_29: +L(2_4_29): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -4085,19 +4085,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L2_4_21 + jg L(2_4_21) ALIGN_4 /**************************************************************************/ -.L2_4_40: +L(2_4_40): testq $ 1, M - jz .L2_4_60 // to next 2 lines of N + jz L(2_4_60) // to next 2 lines of N ALIGN_4 -.L2_4_41: +L(2_4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4135,7 +4135,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_4_46 + je L(2_4_46) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -4146,7 +4146,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_42: +L(2_4_42): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -4161,7 +4161,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_4_46 + je L(2_4_46) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -4176,12 +4176,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_4_46 + je L(2_4_46) - jmp .L2_4_42 + jmp L(2_4_42) ALIGN_4 -.L2_4_46: +L(2_4_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -4189,7 +4189,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_4_49 + je L(2_4_49) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -4201,15 +4201,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_47: +L(2_4_47): KERNEL1x2_SUB - jl .L2_4_47 + jl L(2_4_47) ALIGN_4 -.L2_4_49: +L(2_4_49): SAVE1x2 @@ -4231,23 +4231,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 2 * SIZE, CO1 # coffset += 2 decq I # i -- - jg .L2_4_41 + jg L(2_4_41) ALIGN_4 -.L2_4_60: +L(2_4_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $ 2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -4255,30 +4255,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $ 1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $ 2*SIZE,BO1 addq $ 2*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -4292,13 +4292,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 3, I // i = (m >> 3) - je .L1_4_10 + je L(1_4_10) ALIGN_4 /**************************************************************************************************/ -.L1_8_11: +L(1_8_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4336,7 +4336,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_8_16 + je L(1_8_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -4347,7 +4347,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_8_12: +L(1_8_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -4368,7 +4368,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL8x1_SUB - je .L1_8_16 + je L(1_8_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -4389,12 +4389,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL8x1_SUB - je .L1_8_16 + je L(1_8_16) - jmp .L1_8_12 + jmp L(1_8_12) ALIGN_4 -.L1_8_16: +L(1_8_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -4402,7 +4402,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_8_19 + je L(1_8_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 4 ; number of values @@ -4414,15 +4414,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_8_17: +L(1_8_17): KERNEL8x1_SUB - jl .L1_8_17 + jl L(1_8_17) ALIGN_4 -.L1_8_19: +L(1_8_19): SAVE8x1 @@ -4445,22 +4445,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_8_11 + jg L(1_8_11) ALIGN_4 /**************************************************************************************************/ -.L1_4_10: +L(1_4_10): testq $ 7, M - jz .L999 + jz L(999) testq $ 4, M - jz .L1_4_20 + jz L(1_4_20) -.L1_4_11: +L(1_4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4498,7 +4498,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_4_16 + je L(1_4_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -4509,7 +4509,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_12: +L(1_4_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -4526,7 +4526,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_4_16 + je L(1_4_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -4543,12 +4543,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_4_16 + je L(1_4_16) - jmp .L1_4_12 + jmp L(1_4_12) ALIGN_4 -.L1_4_16: +L(1_4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -4556,7 +4556,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_4_19 + je L(1_4_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 4 ; number of values @@ -4568,15 +4568,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_17: +L(1_4_17): KERNEL4x1_SUB - jl .L1_4_17 + jl L(1_4_17) ALIGN_4 -.L1_4_19: +L(1_4_19): SAVE4x1 @@ -4605,13 +4605,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Rest of M ***************************************************************************/ -.L1_4_20: +L(1_4_20): testq $ 2, M - jz .L1_4_40 + jz L(1_4_40) ALIGN_4 -.L1_4_21: +L(1_4_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4649,7 +4649,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_4_26 + je L(1_4_26) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -4660,7 +4660,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_22: +L(1_4_22): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -4675,7 +4675,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_4_26 + je L(1_4_26) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -4690,12 +4690,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_4_26 + je L(1_4_26) - jmp .L1_4_22 + jmp L(1_4_22) ALIGN_4 -.L1_4_26: +L(1_4_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4703,7 +4703,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_4_29 + je L(1_4_29) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2; number of values @@ -4715,15 +4715,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_27: +L(1_4_27): KERNEL2x1_SUB - jl .L1_4_27 + jl L(1_4_27) ALIGN_4 -.L1_4_29: +L(1_4_29): SAVE2x1 @@ -4749,13 +4749,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L1_4_40: +L(1_4_40): testq $ 1, M - jz .L999 // to next 2 lines of N + jz L(999) // to next 2 lines of N ALIGN_4 -.L1_4_41: +L(1_4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4793,7 +4793,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_4_46 + je L(1_4_46) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -4804,7 +4804,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_42: +L(1_4_42): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -4818,7 +4818,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_4_46 + je L(1_4_46) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -4832,12 +4832,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_4_46 + je L(1_4_46) - jmp .L1_4_42 + jmp L(1_4_42) ALIGN_4 -.L1_4_46: +L(1_4_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -4845,7 +4845,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_4_49 + je L(1_4_49) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -4857,15 +4857,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_47: +L(1_4_47): KERNEL1x1_SUB - jl .L1_4_47 + jl L(1_4_47) ALIGN_4 -.L1_4_49: +L(1_4_49): SAVE1x1 @@ -4891,7 +4891,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): vzeroupper movq SP, %rsp diff --git a/kernel/x86_64/cgemm_kernel_8x2_sandy.S b/kernel/x86_64/cgemm_kernel_8x2_sandy.S index 988913591c..4c887ea9b1 100644 --- a/kernel/x86_64/cgemm_kernel_8x2_sandy.S +++ b/kernel/x86_64/cgemm_kernel_8x2_sandy.S @@ -963,13 +963,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $ 0, OLD_M - je .L999 + je L(999) cmpq $ 0, OLD_N - je .L999 + je L(999) cmpq $ 0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -997,36 +997,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif #endif -.L2_0: +L(2_0): movq Ndiv6, J cmpq $ 0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $ 4*SIZE,BO1 addq $ 4*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -1040,12 +1040,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 3, I // i = (m >> 3) - je .L2_4_10 + je L(2_4_10) ALIGN_4 /**********************************************************************************************************/ -.L2_8_11: +L(2_8_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1083,7 +1083,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_8_16 + je L(2_8_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1094,7 +1094,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_8_12: +L(2_8_12): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL8x2_1 @@ -1102,7 +1102,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 B_PR1(BO,BI,SIZE) KERNEL8x2_1 - je .L2_8_16 + je L(2_8_16) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL8x2_1 @@ -1110,12 +1110,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 B_PR1(BO,BI,SIZE) KERNEL8x2_1 - je .L2_8_16 + je L(2_8_16) - jmp .L2_8_12 + jmp L(2_8_12) ALIGN_4 -.L2_8_16: +L(2_8_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -1123,7 +1123,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_8_19 + je L(2_8_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1135,15 +1135,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_8_17: +L(2_8_17): KERNEL8x2_SUB - jl .L2_8_17 + jl L(2_8_17) ALIGN_4 -.L2_8_19: +L(2_8_19): SAVE8x2 @@ -1166,7 +1166,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_8_11 + jg L(2_8_11) ALIGN_4 @@ -1175,16 +1175,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_4_10: +L(2_4_10): testq $ 7, M - jz .L2_4_60 // to next 2 lines of N + jz L(2_4_60) // to next 2 lines of N testq $ 4, M - jz .L2_4_20 + jz L(2_4_20) ALIGN_4 -.L2_4_11: +L(2_4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1222,7 +1222,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_4_16 + je L(2_4_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1233,7 +1233,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_12: +L(2_4_12): prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL4x2_SUB @@ -1249,7 +1249,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_4_16 + je L(2_4_16) prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL4x2_SUB @@ -1265,12 +1265,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_4_16 + je L(2_4_16) - jmp .L2_4_12 + jmp L(2_4_12) ALIGN_4 -.L2_4_16: +L(2_4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -1278,7 +1278,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_4_19 + je L(2_4_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1290,15 +1290,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_17: +L(2_4_17): KERNEL4x2_SUB - jl .L2_4_17 + jl L(2_4_17) ALIGN_4 -.L2_4_19: +L(2_4_19): SAVE4x2 @@ -1327,13 +1327,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Rest of M ***************************************************************************/ -.L2_4_20: +L(2_4_20): testq $ 2, M - jz .L2_4_40 + jz L(2_4_40) ALIGN_4 -.L2_4_21: +L(2_4_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1371,7 +1371,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_4_26 + je L(2_4_26) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1382,7 +1382,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_22: +L(2_4_22): prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL2x2_SUB @@ -1396,7 +1396,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_4_26 + je L(2_4_26) prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL2x2_SUB @@ -1410,12 +1410,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_4_26 + je L(2_4_26) - jmp .L2_4_22 + jmp L(2_4_22) ALIGN_4 -.L2_4_26: +L(2_4_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -1423,7 +1423,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_4_29 + je L(2_4_29) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1435,15 +1435,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_27: +L(2_4_27): KERNEL2x2_SUB - jl .L2_4_27 + jl L(2_4_27) ALIGN_4 -.L2_4_29: +L(2_4_29): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -1519,19 +1519,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L2_4_21 + jg L(2_4_21) ALIGN_4 /**************************************************************************/ -.L2_4_40: +L(2_4_40): testq $ 1, M - jz .L2_4_60 // to next 2 lines of N + jz L(2_4_60) // to next 2 lines of N ALIGN_4 -.L2_4_41: +L(2_4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1569,7 +1569,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_4_46 + je L(2_4_46) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1580,7 +1580,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_42: +L(2_4_42): prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL1x2_SUB @@ -1593,7 +1593,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_4_46 + je L(2_4_46) prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL1x2_SUB @@ -1606,12 +1606,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_4_46 + je L(2_4_46) - jmp .L2_4_42 + jmp L(2_4_42) ALIGN_4 -.L2_4_46: +L(2_4_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -1619,7 +1619,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_4_49 + je L(2_4_49) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1631,15 +1631,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_47: +L(2_4_47): KERNEL1x2_SUB - jl .L2_4_47 + jl L(2_4_47) ALIGN_4 -.L2_4_49: +L(2_4_49): SAVE1x2 @@ -1661,23 +1661,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 2 * SIZE, CO1 # coffset += 2 decq I # i -- - jg .L2_4_41 + jg L(2_4_41) ALIGN_4 -.L2_4_60: +L(2_4_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $ 2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -1685,30 +1685,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $ 1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $ 2*SIZE,BO1 addq $ 2*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -1722,13 +1722,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 3, I // i = (m >> 3) - je .L1_4_10 + je L(1_4_10) ALIGN_4 /**************************************************************************************************/ -.L1_8_11: +L(1_8_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1766,7 +1766,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_8_16 + je L(1_8_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1777,7 +1777,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_8_12: +L(1_8_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -1798,7 +1798,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL8x1_SUB - je .L1_8_16 + je L(1_8_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -1819,12 +1819,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL8x1_SUB - je .L1_8_16 + je L(1_8_16) - jmp .L1_8_12 + jmp L(1_8_12) ALIGN_4 -.L1_8_16: +L(1_8_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -1832,7 +1832,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_8_19 + je L(1_8_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 4 ; number of values @@ -1844,15 +1844,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_8_17: +L(1_8_17): KERNEL8x1_SUB - jl .L1_8_17 + jl L(1_8_17) ALIGN_4 -.L1_8_19: +L(1_8_19): SAVE8x1 @@ -1875,22 +1875,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_8_11 + jg L(1_8_11) ALIGN_4 /**************************************************************************************************/ -.L1_4_10: +L(1_4_10): testq $ 7, M - jz .L999 + jz L(999) testq $ 4, M - jz .L1_4_20 + jz L(1_4_20) -.L1_4_11: +L(1_4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1928,7 +1928,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_4_16 + je L(1_4_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1939,7 +1939,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_12: +L(1_4_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -1956,7 +1956,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_4_16 + je L(1_4_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -1973,12 +1973,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_4_16 + je L(1_4_16) - jmp .L1_4_12 + jmp L(1_4_12) ALIGN_4 -.L1_4_16: +L(1_4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -1986,7 +1986,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_4_19 + je L(1_4_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 4 ; number of values @@ -1998,15 +1998,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_17: +L(1_4_17): KERNEL4x1_SUB - jl .L1_4_17 + jl L(1_4_17) ALIGN_4 -.L1_4_19: +L(1_4_19): SAVE4x1 @@ -2035,13 +2035,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Rest of M ***************************************************************************/ -.L1_4_20: +L(1_4_20): testq $ 2, M - jz .L1_4_40 + jz L(1_4_40) ALIGN_4 -.L1_4_21: +L(1_4_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2079,7 +2079,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_4_26 + je L(1_4_26) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -2090,7 +2090,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_22: +L(1_4_22): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2105,7 +2105,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_4_26 + je L(1_4_26) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2120,12 +2120,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_4_26 + je L(1_4_26) - jmp .L1_4_22 + jmp L(1_4_22) ALIGN_4 -.L1_4_26: +L(1_4_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -2133,7 +2133,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_4_29 + je L(1_4_29) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2; number of values @@ -2145,15 +2145,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_27: +L(1_4_27): KERNEL2x1_SUB - jl .L1_4_27 + jl L(1_4_27) ALIGN_4 -.L1_4_29: +L(1_4_29): SAVE2x1 @@ -2179,13 +2179,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L1_4_40: +L(1_4_40): testq $ 1, M - jz .L999 // to next 2 lines of N + jz L(999) // to next 2 lines of N ALIGN_4 -.L1_4_41: +L(1_4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2223,7 +2223,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_4_46 + je L(1_4_46) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -2234,7 +2234,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_42: +L(1_4_42): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2248,7 +2248,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_4_46 + je L(1_4_46) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2262,12 +2262,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_4_46 + je L(1_4_46) - jmp .L1_4_42 + jmp L(1_4_42) ALIGN_4 -.L1_4_46: +L(1_4_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -2275,7 +2275,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_4_49 + je L(1_4_49) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -2287,15 +2287,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_47: +L(1_4_47): KERNEL1x1_SUB - jl .L1_4_47 + jl L(1_4_47) ALIGN_4 -.L1_4_49: +L(1_4_49): SAVE1x1 @@ -2321,7 +2321,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): vzeroupper movq SP, %rsp diff --git a/kernel/x86_64/cgemv_n.S b/kernel/x86_64/cgemv_n.S index 206beb6735..1b90b5deec 100644 --- a/kernel/x86_64/cgemv_n.S +++ b/kernel/x86_64/cgemv_n.S @@ -167,7 +167,7 @@ movss %xmm0,ALPHAR movss %xmm1,ALPHAI -.L0t: +L(0t): xorq I,I addq $1,I salq $20,I @@ -175,13 +175,13 @@ movq I,M movss ALPHAR,%xmm0 movss ALPHAI,%xmm1 - jge .L00t + jge L(00t) movq MMM,M addq I,M - jle .L999x + jle L(999x) -.L00t: +L(00t): movq AA, A movq NN, N movq LDAX, LDA @@ -201,9 +201,9 @@ movlps %xmm0, ALPHA testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) ALIGN_3 subq $-32 * SIZE, A @@ -217,7 +217,7 @@ sarq $3, %rax ALIGN_3 -.L01: +L(01): movaps %xmm4, 0 * SIZE(Y1) movaps %xmm4, 4 * SIZE(Y1) movaps %xmm4, 8 * SIZE(Y1) @@ -225,10 +225,10 @@ subq $-16 * SIZE, Y1 decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L10: +L(10): #ifdef ALIGNED_ACCESS movq M, MM @@ -242,18 +242,18 @@ cmovge A2, MM testq $SIZE, A - jne .L200 + jne L(200) testq $2 * SIZE, LDA - jne .L100 + jne L(100) #endif #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $4, N leaq 32 * SIZE(BUFFER), Y1 @@ -347,7 +347,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L1X + je L(1X) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA), %xmm6 @@ -386,7 +386,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, Y1 ALIGN_3 -.L1X: +L(1X): #endif movaps -32 * SIZE(Y1), %xmm0 @@ -396,16 +396,16 @@ movq MM, I sarq $3, I - jle .L15 + jle L(15) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) decq I - jle .L14 + jle L(14) ALIGN_3 -.L13: +L(13): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -554,10 +554,10 @@ subq $1, I BRANCH - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): pshufd $0xb1, %xmm4, %xmm5 mulps %xmm8, %xmm4 addps %xmm4, %xmm0 @@ -683,9 +683,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L15: +L(15): testq $4, MM - je .L17 + je L(17) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) @@ -758,9 +758,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L17: +L(17): testq $2, MM - je .L18 + je L(18) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A2(-32 * SIZE, A1, LDA, 1, %xmm6) @@ -799,9 +799,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L18: +L(18): testq $1, MM - je .L19 + je L(19) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA), %xmm6 @@ -835,21 +835,21 @@ movlps %xmm0, -32 * SIZE(Y1) ALIGN_3 -.L19: +L(19): cmpq $4, N - jge .L11 + jge L(11) ALIGN_3 -.L20: +L(20): #endif cmpq $2, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 2 ALIGN_3 -.L21: +L(21): #endif subq $2, N @@ -917,7 +917,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L2X + je L(2X) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A2), %xmm6 @@ -943,7 +943,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, Y1 ALIGN_3 -.L2X: +L(2X): #endif movaps -32 * SIZE(Y1), %xmm0 @@ -954,7 +954,7 @@ movq MM, I sarq $3, I - jle .L25 + jle L(25) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) @@ -962,10 +962,10 @@ MOVUPS_A1(-20 * SIZE, A1, %xmm10) decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A2) #endif @@ -1050,10 +1050,10 @@ subq $1, I BRANCH - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): pshufd $0xb1, %xmm4, %xmm5 mulps %xmm12, %xmm4 addps %xmm4, %xmm0 @@ -1121,9 +1121,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L25: +L(25): testq $4, MM - je .L27 + je L(27) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) @@ -1166,9 +1166,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L27: +L(27): testq $2, MM - je .L28 + je L(28) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-32 * SIZE, A2, %xmm6) @@ -1193,12 +1193,12 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L28: +L(28): testq $1, MM #if GEMV_UNROLL == 2 - je .L29 + je L(29) #else - je .L30 + je L(30) #endif movsd -32 * SIZE(A1), %xmm4 @@ -1222,15 +1222,15 @@ #if GEMV_UNROLL == 2 ALIGN_3 -.L29: +L(29): cmpq $2, N - jge .L21 + jge L(21) #endif ALIGN_3 -.L30: +L(30): cmpq $1, N - jl .L990 + jl L(990) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -1279,7 +1279,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L3X + je L(3X) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(Y1), %xmm0 @@ -1296,7 +1296,7 @@ addq $2 * SIZE, A1 addq $2 * SIZE, Y1 ALIGN_3 -.L3X: +L(3X): #endif movaps -32 * SIZE(Y1), %xmm0 @@ -1307,7 +1307,7 @@ movq MM, I sarq $3, I - jle .L35 + jle L(35) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) @@ -1315,10 +1315,10 @@ MOVUPS_A1(-20 * SIZE, A1, %xmm10) decq I - jle .L34 + jle L(34) ALIGN_3 -.L33: +L(33): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -1371,10 +1371,10 @@ subq $1, I BRANCH - jg .L33 + jg L(33) ALIGN_3 -.L34: +L(34): pshufd $0xb1, %xmm4, %xmm5 mulps %xmm12, %xmm4 addps %xmm4, %xmm0 @@ -1413,9 +1413,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L35: +L(35): testq $4, MM - je .L37 + je L(37) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) @@ -1442,9 +1442,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L37: +L(37): testq $2, MM - je .L38 + je L(38) MOVUPS_A1(-32 * SIZE, A1, %xmm4) @@ -1462,9 +1462,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L38: +L(38): testq $1, MM - je .L990 + je L(990) movsd -32 * SIZE(A1), %xmm4 @@ -1478,16 +1478,16 @@ movlps %xmm0, -32 * SIZE(Y1) #ifdef ALIGNED_ACCESS - jmp .L990 + jmp L(990) ALIGN_3 -.L100: +L(100): #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L110 + jl L(110) ALIGN_3 -.L101: +L(101): subq $4, N leaq 32 * SIZE(BUFFER), Y1 @@ -1581,7 +1581,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L10X + je L(10X) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA), %xmm6 @@ -1620,7 +1620,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, Y1 ALIGN_3 -.L10X: +L(10X): #endif movaps -32 * SIZE(Y1), %xmm0 @@ -1630,16 +1630,16 @@ movq MM, I sarq $3, I - jle .L105 + jle L(105) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) decq I - jle .L104 + jle L(104) ALIGN_3 -.L103: +L(103): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -1796,10 +1796,10 @@ subq $1, I BRANCH - jg .L103 + jg L(103) ALIGN_3 -.L104: +L(104): pshufd $0xb1, %xmm4, %xmm5 mulps %xmm8, %xmm4 addps %xmm4, %xmm0 @@ -1933,9 +1933,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L105: +L(105): testq $4, MM - je .L107 + je L(107) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) @@ -2009,9 +2009,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L107: +L(107): testq $2, MM - je .L108 + je L(108) MOVUPS_A1(-32 * SIZE, A1, %xmm4) movsd -32 * SIZE(A1, LDA), %xmm6 @@ -2052,9 +2052,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L108: +L(108): testq $1, MM - je .L109 + je L(109) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA), %xmm6 @@ -2088,23 +2088,23 @@ movlps %xmm0, -32 * SIZE(Y1) ALIGN_3 -.L109: +L(109): cmpq $4, N - jge .L101 + jge L(101) ALIGN_3 -.L110: +L(110): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L120 + jl L(120) #if GEMV_UNROLL == 2 ALIGN_3 -.L111: +L(111): #endif subq $2, N @@ -2172,7 +2172,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L11X + je L(11X) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A2), %xmm6 @@ -2198,7 +2198,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, Y1 ALIGN_3 -.L11X: +L(11X): #endif movaps -32 * SIZE(Y1), %xmm0 @@ -2209,7 +2209,7 @@ movq MM, I sarq $3, I - jle .L115 + jle L(115) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) @@ -2217,10 +2217,10 @@ MOVUPS_A1(-20 * SIZE, A1, %xmm10) decq I - jle .L114 + jle L(114) ALIGN_3 -.L113: +L(113): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A2) #endif @@ -2309,10 +2309,10 @@ subq $1, I BRANCH - jg .L113 + jg L(113) ALIGN_3 -.L114: +L(114): pshufd $0xb1, %xmm4, %xmm5 mulps %xmm12, %xmm4 addps %xmm4, %xmm0 @@ -2384,9 +2384,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L115: +L(115): testq $4, MM - je .L117 + je L(117) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) @@ -2431,9 +2431,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L117: +L(117): testq $2, MM - je .L118 + je L(118) MOVUPS_A1(-32 * SIZE, A1, %xmm4) movsd -32 * SIZE(A2), %xmm6 @@ -2459,12 +2459,12 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L118: +L(118): testq $1, MM #if GEMV_UNROLL == 2 - je .L119 + je L(119) #else - je .L120 + je L(120) #endif movsd -32 * SIZE(A1), %xmm4 @@ -2488,17 +2488,17 @@ #if GEMV_UNROLL == 2 ALIGN_3 -.L119: +L(119): cmpq $2, N - jge .L111 + jge L(111) #endif ALIGN_3 -.L120: +L(120): #endif cmpq $1, N - jl .L990 + jl L(990) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -2547,7 +2547,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L12X + je L(12X) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(Y1), %xmm0 @@ -2564,7 +2564,7 @@ addq $2 * SIZE, A1 addq $2 * SIZE, Y1 ALIGN_3 -.L12X: +L(12X): #endif movaps -32 * SIZE(Y1), %xmm0 @@ -2575,7 +2575,7 @@ movq MM, I sarq $3, I - jle .L125 + jle L(125) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) @@ -2583,10 +2583,10 @@ MOVUPS_A1(-20 * SIZE, A1, %xmm10) decq I - jle .L124 + jle L(124) ALIGN_3 -.L123: +L(123): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -2639,10 +2639,10 @@ subq $1, I BRANCH - jg .L123 + jg L(123) ALIGN_3 -.L124: +L(124): pshufd $0xb1, %xmm4, %xmm5 mulps %xmm12, %xmm4 addps %xmm4, %xmm0 @@ -2681,9 +2681,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L125: +L(125): testq $4, MM - je .L127 + je L(127) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) @@ -2710,9 +2710,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L127: +L(127): testq $2, MM - je .L128 + je L(128) MOVUPS_A1(-32 * SIZE, A1, %xmm4) @@ -2730,9 +2730,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L128: +L(128): testq $1, MM - je .L990 + je L(990) movsd -32 * SIZE(A1), %xmm4 @@ -2745,18 +2745,18 @@ movlps %xmm0, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_3 -.L200: +L(200): testq $2 * SIZE, LDA - jne .L300 + jne L(300) cmpq $2, N - jl .L210 + jl L(210) ALIGN_3 -.L201: +L(201): subq $2, N leaq 32 * SIZE(BUFFER), Y1 @@ -2822,7 +2822,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L20X + je L(20X) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A2), %xmm6 @@ -2848,7 +2848,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, Y1 ALIGN_3 -.L20X: +L(20X): #endif movaps -33 * SIZE(A1), %xmm4 @@ -2861,17 +2861,17 @@ movq MM, I sarq $3, I - jle .L205 + jle L(205) movaps -29 * SIZE(A1), %xmm8 movaps -25 * SIZE(A1), %xmm9 movaps -21 * SIZE(A1), %xmm10 decq I - jle .L204 + jle L(204) ALIGN_3 -.L203: +L(203): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A2) #endif @@ -2976,10 +2976,10 @@ subq $1, I BRANCH - jg .L203 + jg L(203) ALIGN_3 -.L204: +L(204): movss %xmm8, %xmm4 shufps $0x39, %xmm4, %xmm4 pshufd $0xb1, %xmm4, %xmm5 @@ -3068,9 +3068,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L205: +L(205): testq $4, MM - je .L207 + je L(207) movaps -29 * SIZE(A1), %xmm8 movaps -25 * SIZE(A1), %xmm9 @@ -3124,9 +3124,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L207: +L(207): testq $2, MM - je .L208 + je L(208) movaps -29 * SIZE(A1), %xmm8 movaps -29 * SIZE(A2), %xmm9 @@ -3156,9 +3156,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L208: +L(208): testq $1, MM - je .L209 + je L(209) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A2), %xmm6 @@ -3179,14 +3179,14 @@ movlps %xmm0, -32 * SIZE(Y1) ALIGN_3 -.L209: +L(209): cmpq $2, N - jge .L201 + jge L(201) ALIGN_3 -.L210: +L(210): cmpq $1, N - jl .L990 + jl L(990) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -3235,7 +3235,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L21X + je L(21X) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(Y1), %xmm0 @@ -3252,7 +3252,7 @@ addq $2 * SIZE, A1 addq $2 * SIZE, Y1 ALIGN_3 -.L21X: +L(21X): #endif movaps -33 * SIZE(A1), %xmm4 @@ -3264,17 +3264,17 @@ movq MM, I sarq $3, I - jle .L215 + jle L(215) movaps -29 * SIZE(A1), %xmm6 movaps -25 * SIZE(A1), %xmm8 movaps -21 * SIZE(A1), %xmm10 decq I - jle .L214 + jle L(214) ALIGN_3 -.L213: +L(213): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -3337,10 +3337,10 @@ subq $1, I BRANCH - jg .L213 + jg L(213) ALIGN_3 -.L214: +L(214): movss %xmm6, %xmm4 shufps $0x39, %xmm4, %xmm4 pshufd $0xb1, %xmm4, %xmm5 @@ -3390,9 +3390,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L215: +L(215): testq $4, MM - je .L217 + je L(217) movaps -29 * SIZE(A1), %xmm6 movaps -25 * SIZE(A1), %xmm8 @@ -3425,9 +3425,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L217: +L(217): testq $2, MM - je .L218 + je L(218) movaps -29 * SIZE(A1), %xmm6 @@ -3447,9 +3447,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L218: +L(218): testq $1, MM - je .L990 + je L(990) movsd -32 * SIZE(A1), %xmm4 @@ -3461,15 +3461,15 @@ SUBPS %xmm5, %xmm0 movlps %xmm0, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_3 -.L300: +L(300): cmpq $2, N - jl .L310 + jl L(310) ALIGN_3 -.L301: +L(301): subq $2, N leaq 32 * SIZE(BUFFER), Y1 @@ -3535,7 +3535,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L30X + je L(30X) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A2), %xmm6 @@ -3561,7 +3561,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, Y1 ALIGN_3 -.L30X: +L(30X): #endif movaps -33 * SIZE(A1), %xmm4 @@ -3574,17 +3574,17 @@ movq MM, I sarq $3, I - jle .L305 + jle L(305) movaps -29 * SIZE(A1), %xmm8 movaps -25 * SIZE(A1), %xmm9 movaps -21 * SIZE(A1), %xmm10 decq I - jle .L304 + jle L(304) ALIGN_3 -.L303: +L(303): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A2) #endif @@ -3689,10 +3689,10 @@ subq $1, I BRANCH - jg .L303 + jg L(303) ALIGN_3 -.L304: +L(304): movss %xmm8, %xmm4 shufps $0x39, %xmm4, %xmm4 pshufd $0xb1, %xmm4, %xmm5 @@ -3781,9 +3781,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L305: +L(305): testq $4, MM - je .L307 + je L(307) movaps -29 * SIZE(A1), %xmm8 movaps -25 * SIZE(A1), %xmm9 @@ -3837,9 +3837,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L307: +L(307): testq $2, MM - je .L308 + je L(308) movaps -29 * SIZE(A1), %xmm8 movaps -31 * SIZE(A2), %xmm9 @@ -3869,9 +3869,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L308: +L(308): testq $1, MM - je .L309 + je L(309) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A2), %xmm6 @@ -3892,14 +3892,14 @@ movlps %xmm0, -32 * SIZE(Y1) ALIGN_3 -.L309: +L(309): cmpq $2, N - jge .L301 + jge L(301) ALIGN_3 -.L310: +L(310): cmpq $1, N - jl .L990 + jl L(990) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -3948,7 +3948,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L31X + je L(31X) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(Y1), %xmm0 @@ -3965,7 +3965,7 @@ addq $2 * SIZE, A1 addq $2 * SIZE, Y1 ALIGN_3 -.L31X: +L(31X): #endif movaps -33 * SIZE(A1), %xmm4 @@ -3977,17 +3977,17 @@ movq MM, I sarq $3, I - jle .L315 + jle L(315) movaps -29 * SIZE(A1), %xmm6 movaps -25 * SIZE(A1), %xmm8 movaps -21 * SIZE(A1), %xmm10 decq I - jle .L314 + jle L(314) ALIGN_3 -.L313: +L(313): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -4050,10 +4050,10 @@ subq $1, I BRANCH - jg .L313 + jg L(313) ALIGN_3 -.L314: +L(314): movss %xmm6, %xmm4 shufps $0x39, %xmm4, %xmm4 pshufd $0xb1, %xmm4, %xmm5 @@ -4103,9 +4103,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L315: +L(315): testq $4, MM - je .L317 + je L(317) movaps -29 * SIZE(A1), %xmm6 movaps -25 * SIZE(A1), %xmm8 @@ -4138,9 +4138,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L317: +L(317): testq $2, MM - je .L318 + je L(318) movaps -29 * SIZE(A1), %xmm6 @@ -4160,9 +4160,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L318: +L(318): testq $1, MM - je .L990 + je L(990) movsd -32 * SIZE(A1), %xmm4 @@ -4177,12 +4177,12 @@ #endif ALIGN_3 -.L990: +L(990): movq Y, Y1 #ifdef ALIGNED_ACCESS cmpq M, MM - je .L991 + je L(991) movsd (Y), %xmm0 addq INCY, Y @@ -4196,14 +4196,14 @@ addq INCY, Y1 ALIGN_3 -.L991: +L(991): #endif movq MM, %rax sarq $3, %rax - jle .L994 + jle L(994) ALIGN_3 -.L992: +L(992): movsd (Y), %xmm0 addq INCY, Y movhps (Y), %xmm0 @@ -4251,15 +4251,15 @@ addq $16 * SIZE, BUFFER decq %rax - jg .L992 + jg L(992) ALIGN_3 -.L994: +L(994): testq $7, MM - jle .L999 + jle L(999) testq $4, MM - jle .L995 + jle L(995) movsd (Y), %xmm0 addq INCY, Y @@ -4287,9 +4287,9 @@ addq $8 * SIZE, BUFFER ALIGN_3 -.L995: +L(995): testq $2, MM - jle .L996 + jle L(996) movsd (Y), %xmm0 addq INCY, Y @@ -4306,9 +4306,9 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L996: +L(996): testq $1, MM - jle .L999 + jle L(999) movsd (Y), %xmm0 @@ -4317,12 +4317,12 @@ movlps %xmm0, (Y1) ALIGN_3 -.L999: +L(999): movq M, I salq $ZBASE_SHIFT,I addq I,AA - jmp .L0t -.L999x: + jmp L(0t) +L(999x): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/cgemv_t.S b/kernel/x86_64/cgemv_t.S index 430586bba2..a32be60dea 100644 --- a/kernel/x86_64/cgemv_t.S +++ b/kernel/x86_64/cgemv_t.S @@ -164,7 +164,7 @@ movss %xmm0,ALPHAR movss %xmm1,ALPHAI -.L0t: +L(0t): xorq I,I addq $1,I salq $20,I @@ -172,13 +172,13 @@ movq I,M movss ALPHAR,%xmm0 movss ALPHAI,%xmm1 - jge .L00t + jge L(00t) movq MMM,M addq I,M - jle .L999x + jle L(999x) -.L00t: +L(00t): movq AA, A movq NN, N movq LDAX, LDA @@ -197,9 +197,9 @@ movlps %xmm0, ALPHA testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) subq $-32 * SIZE, A @@ -211,7 +211,7 @@ andq $4 * SIZE - 1, %rax cmpq $2 * SIZE, %rax - jl .L0X + jl L(0X) movsd (X), %xmm0 addq INCX, X @@ -221,15 +221,15 @@ addq $4 * SIZE, X1 decq MM -.L0X: +L(0X): #endif movq MM, I sarq $3, I - jle .L05 + jle L(05) ALIGN_4 -.L02: +L(02): movsd (X), %xmm0 addq INCX, X movhps (X), %xmm0 @@ -257,41 +257,41 @@ addq $16 * SIZE, X1 decq I - jg .L02 + jg L(02) ALIGN_4 -.L05: +L(05): movq MM, I andq $7, I - jle .L10 + jle L(10) ALIGN_2 -.L06: +L(06): movsd (X), %xmm0 addq INCX, X movlps %xmm0, 0 * SIZE(X1) addq $2 * SIZE, X1 decq I - jg .L06 + jg L(06) ALIGN_4 -.L10: +L(10): movq Y, Y1 #ifdef ALIGNED_ACCESS testq $SIZE, A - jne .L200 + jne L(200) testq $2 * SIZE, LDA - jne .L100 + jne L(100) #endif #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $4, N leaq 32 * SIZE(BUFFER), X1 @@ -311,7 +311,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L1X + je L(1X) #ifdef movsd xorps %xmm8, %xmm8 @@ -363,7 +363,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, X1 ALIGN_3 -.L1X: +L(1X): #endif movaps -32 * SIZE(X1), %xmm12 @@ -375,7 +375,7 @@ movq MM, I sarq $3, I - jle .L15 + jle L(15) MOVUPS_A1(-32 * SIZE, A1, %xmm8) MOVUPS_A2(-32 * SIZE, A1, LDA, 1, %xmm9) @@ -383,10 +383,10 @@ MOVUPS_A2(-32 * SIZE, A2, LDA, 1, %xmm11) decq I - jle .L14 + jle L(14) ALIGN_3 -.L13: +L(13): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -529,10 +529,10 @@ subq $1, I BRANCH - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): pshufd $0xb1, %xmm8, %xmm14 mulps %xmm12, %xmm8 addps %xmm8, %xmm0 @@ -650,9 +650,9 @@ subq $-16 * SIZE, X1 ALIGN_3 -.L15: +L(15): testq $4, MM - je .L17 + je L(17) MOVUPS_A1(-32 * SIZE, A1, %xmm8) MOVUPS_A2(-32 * SIZE, A1, LDA, 1, %xmm9) @@ -718,9 +718,9 @@ addq $8 * SIZE, X1 ALIGN_3 -.L17: +L(17): testq $2, MM - je .L18 + je L(18) MOVUPS_A1(-32 * SIZE, A1, %xmm8) MOVUPS_A2(-32 * SIZE, A1, LDA, 1, %xmm9) @@ -757,9 +757,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L18: +L(18): testq $1, MM - je .L19 + je L(19) #ifdef movsd xorps %xmm8, %xmm8 @@ -803,7 +803,7 @@ SUBPS %xmm15, %xmm7 ALIGN_3 -.L19: +L(19): pcmpeqb %xmm11, %xmm11 psllq $63, %xmm11 @@ -923,18 +923,18 @@ addq INCY, Y1 cmpq $4, N - jge .L11 + jge L(11) ALIGN_3 -.L20: +L(20): #endif cmpq $2, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 2 ALIGN_3 -.L21: +L(21): #endif subq $2, N @@ -951,7 +951,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L2X + je L(2X) #ifdef movsd xorps %xmm8, %xmm8 @@ -983,7 +983,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, X1 ALIGN_3 -.L2X: +L(2X): #endif movaps -32 * SIZE(X1), %xmm12 @@ -995,7 +995,7 @@ movq MM, I sarq $3, I - jle .L25 + jle L(25) MOVUPS_A1(-32 * SIZE, A1, %xmm8) MOVUPS_A1(-32 * SIZE, A2, %xmm9) @@ -1003,10 +1003,10 @@ MOVUPS_A1(-28 * SIZE, A2, %xmm11) decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -1089,10 +1089,10 @@ subq $1, I BRANCH - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): pshufd $0xb1, %xmm8, %xmm4 mulps %xmm12, %xmm8 addps %xmm8, %xmm0 @@ -1158,9 +1158,9 @@ subq $-16 * SIZE, X1 ALIGN_3 -.L25: +L(25): testq $4, MM - je .L27 + je L(27) MOVUPS_A1(-32 * SIZE, A1, %xmm8) MOVUPS_A1(-32 * SIZE, A2, %xmm9) @@ -1199,9 +1199,9 @@ addq $8 * SIZE, A2 ALIGN_3 -.L27: +L(27): testq $2, MM - je .L28 + je L(28) MOVUPS_A1(-32 * SIZE, A1, %xmm8) MOVUPS_A1(-32 * SIZE, A2, %xmm9) @@ -1224,9 +1224,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L28: +L(28): testq $1, MM - je .L29 + je L(29) #ifdef movsd xorps %xmm8, %xmm8 @@ -1250,7 +1250,7 @@ SUBPS %xmm5, %xmm3 ALIGN_3 -.L29: +L(29): pcmpeqb %xmm5, %xmm5 psllq $63, %xmm5 @@ -1326,13 +1326,13 @@ #if GEMV_UNROLL == 2 cmpq $2, N - jge .L21 + jge L(21) #endif ALIGN_3 -.L30: +L(30): cmpq $1, N - jl .L999 + jl L(999) leaq 32 * SIZE(BUFFER), X1 @@ -1343,7 +1343,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L3X + je L(3X) #ifdef movsd xorps %xmm8, %xmm8 @@ -1363,7 +1363,7 @@ addq $2 * SIZE, A1 addq $2 * SIZE, X1 ALIGN_3 -.L3X: +L(3X): #endif movaps -32 * SIZE(X1), %xmm12 @@ -1371,16 +1371,16 @@ movq MM, I sarq $3, I - jle .L35 + jle L(35) MOVUPS_A1(-32 * SIZE, A1, %xmm8) MOVUPS_A1(-28 * SIZE, A1, %xmm10) decq I - jle .L34 + jle L(34) ALIGN_3 -.L33: +L(33): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -1430,10 +1430,10 @@ subq $1, I BRANCH - jg .L33 + jg L(33) ALIGN_3 -.L34: +L(34): pshufd $0xb1, %xmm8, %xmm4 mulps %xmm12, %xmm8 addps %xmm8, %xmm0 @@ -1472,9 +1472,9 @@ subq $-16 * SIZE, X1 ALIGN_3 -.L35: +L(35): testq $4, MM - je .L37 + je L(37) MOVUPS_A1(-32 * SIZE, A1, %xmm8) MOVUPS_A1(-28 * SIZE, A1, %xmm10) @@ -1498,9 +1498,9 @@ addq $8 * SIZE, A1 ALIGN_3 -.L37: +L(37): testq $2, MM - je .L38 + je L(38) MOVUPS_A1(-32 * SIZE, A1, %xmm8) @@ -1515,9 +1515,9 @@ addq $4 * SIZE, A1 ALIGN_3 -.L38: +L(38): testq $1, MM - je .L39 + je L(39) #ifdef movsd xorps %xmm8, %xmm8 @@ -1531,7 +1531,7 @@ SUBPS %xmm4, %xmm1 ALIGN_3 -.L39: +L(39): pcmpeqb %xmm5, %xmm5 psllq $63, %xmm5 @@ -1590,17 +1590,17 @@ movlps %xmm0, (Y1) addq INCY, Y1 #ifdef ALIGNED_ACCESS - jmp .L999 + jmp L(999) ALIGN_3 -.L100: +L(100): #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L110 + jl L(110) ALIGN_3 -.L101: +L(101): subq $4, N leaq 32 * SIZE(BUFFER), X1 @@ -1620,7 +1620,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L10X + je L(10X) #ifdef movsd xorps %xmm8, %xmm8 @@ -1672,7 +1672,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, X1 ALIGN_3 -.L10X: +L(10X): #endif movaps -32 * SIZE(X1), %xmm12 @@ -1684,7 +1684,7 @@ movq MM, I sarq $3, I - jle .L105 + jle L(105) movaps -32 * SIZE(A1), %xmm8 movsd -32 * SIZE(A1, LDA), %xmm9 @@ -1695,10 +1695,10 @@ movhps -30 * SIZE(A2, LDA), %xmm11 decq I - jle .L104 + jle L(104) ALIGN_3 -.L103: +L(103): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -1849,10 +1849,10 @@ subq $1, I BRANCH - jg .L103 + jg L(103) ALIGN_3 -.L104: +L(104): pshufd $0xb1, %xmm8, %xmm14 mulps %xmm12, %xmm8 addps %xmm8, %xmm0 @@ -1976,9 +1976,9 @@ subq $-16 * SIZE, X1 ALIGN_3 -.L105: +L(105): testq $4, MM - je .L107 + je L(107) movaps -32 * SIZE(A1), %xmm8 movsd -32 * SIZE(A1, LDA), %xmm9 @@ -2049,9 +2049,9 @@ addq $8 * SIZE, X1 ALIGN_3 -.L107: +L(107): testq $2, MM - je .L108 + je L(108) movaps -32 * SIZE(A1), %xmm8 movsd -32 * SIZE(A1, LDA), %xmm9 @@ -2091,9 +2091,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L108: +L(108): testq $1, MM - je .L109 + je L(109) #ifdef movsd xorps %xmm8, %xmm8 @@ -2137,7 +2137,7 @@ SUBPS %xmm15, %xmm7 ALIGN_3 -.L109: +L(109): pcmpeqb %xmm11, %xmm11 psllq $63, %xmm11 @@ -2257,18 +2257,18 @@ addq INCY, Y1 cmpq $4, N - jge .L101 + jge L(101) ALIGN_3 -.L110: +L(110): #endif cmpq $2, N - jl .L120 + jl L(120) #if GEMV_UNROLL == 2 ALIGN_3 -.L111: +L(111): #endif subq $2, N @@ -2285,7 +2285,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L11X + je L(11X) #ifdef movsd xorps %xmm8, %xmm8 @@ -2317,7 +2317,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, X1 ALIGN_3 -.L11X: +L(11X): #endif movaps -32 * SIZE(X1), %xmm12 @@ -2329,7 +2329,7 @@ movq MM, I sarq $3, I - jle .L115 + jle L(115) movaps -32 * SIZE(A1), %xmm8 movsd -32 * SIZE(A2), %xmm9 @@ -2340,10 +2340,10 @@ movhps -26 * SIZE(A2), %xmm11 decq I - jle .L114 + jle L(114) ALIGN_3 -.L113: +L(113): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -2430,10 +2430,10 @@ subq $1, I BRANCH - jg .L113 + jg L(113) ALIGN_3 -.L114: +L(114): pshufd $0xb1, %xmm8, %xmm4 mulps %xmm12, %xmm8 addps %xmm8, %xmm0 @@ -2501,9 +2501,9 @@ subq $-16 * SIZE, X1 ALIGN_3 -.L115: +L(115): testq $4, MM - je .L117 + je L(117) movaps -32 * SIZE(A1), %xmm8 movsd -32 * SIZE(A2), %xmm9 @@ -2545,9 +2545,9 @@ addq $8 * SIZE, A2 ALIGN_3 -.L117: +L(117): testq $2, MM - je .L118 + je L(118) movaps -32 * SIZE(A1), %xmm8 movsd -32 * SIZE(A2), %xmm9 @@ -2571,9 +2571,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L118: +L(118): testq $1, MM - je .L119 + je L(119) #ifdef movsd xorps %xmm8, %xmm8 @@ -2597,7 +2597,7 @@ SUBPS %xmm5, %xmm3 ALIGN_3 -.L119: +L(119): pcmpeqb %xmm5, %xmm5 psllq $63, %xmm5 @@ -2673,13 +2673,13 @@ #if GEMV_UNROLL == 2 cmpq $2, N - jge .L111 + jge L(111) #endif ALIGN_3 -.L120: +L(120): cmpq $1, N - jl .L999 + jl L(999) leaq 32 * SIZE(BUFFER), X1 @@ -2690,7 +2690,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L12X + je L(12X) #ifdef movsd xorps %xmm8, %xmm8 @@ -2710,7 +2710,7 @@ addq $2 * SIZE, A1 addq $2 * SIZE, X1 ALIGN_3 -.L12X: +L(12X): #endif movaps -32 * SIZE(X1), %xmm12 @@ -2718,16 +2718,16 @@ movq MM, I sarq $3, I - jle .L125 + jle L(125) MOVUPS_A1(-32 * SIZE, A1, %xmm8) MOVUPS_A1(-28 * SIZE, A1, %xmm10) decq I - jle .L124 + jle L(124) ALIGN_3 -.L123: +L(123): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -2777,10 +2777,10 @@ subq $1, I BRANCH - jg .L123 + jg L(123) ALIGN_3 -.L124: +L(124): pshufd $0xb1, %xmm8, %xmm4 mulps %xmm12, %xmm8 addps %xmm8, %xmm0 @@ -2819,9 +2819,9 @@ subq $-16 * SIZE, X1 ALIGN_3 -.L125: +L(125): testq $4, MM - je .L127 + je L(127) MOVUPS_A1(-32 * SIZE, A1, %xmm8) MOVUPS_A1(-28 * SIZE, A1, %xmm10) @@ -2845,9 +2845,9 @@ addq $8 * SIZE, A1 ALIGN_3 -.L127: +L(127): testq $2, MM - je .L128 + je L(128) MOVUPS_A1(-32 * SIZE, A1, %xmm8) @@ -2862,9 +2862,9 @@ addq $4 * SIZE, A1 ALIGN_3 -.L128: +L(128): testq $1, MM - je .L129 + je L(129) #ifdef movsd xorps %xmm8, %xmm8 @@ -2878,7 +2878,7 @@ SUBPS %xmm4, %xmm1 ALIGN_3 -.L129: +L(129): pcmpeqb %xmm5, %xmm5 psllq $63, %xmm5 @@ -2936,19 +2936,19 @@ movlps %xmm0, (Y1) addq INCY, Y1 - jmp .L999 + jmp L(999) ALIGN_3 -.L200: +L(200): testq $2 * SIZE, LDA - jne .L300 + jne L(300) cmpq $2, N - jl .L210 + jl L(210) ALIGN_3 -.L201: +L(201): subq $2, N leaq 32 * SIZE(BUFFER), X1 @@ -2964,7 +2964,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L20X + je L(20X) #ifdef movsd xorps %xmm8, %xmm8 @@ -2996,7 +2996,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, X1 ALIGN_3 -.L20X: +L(20X): #endif movaps -33 * SIZE(A1), %xmm4 @@ -3011,16 +3011,16 @@ movq MM, I sarq $3, I - jle .L205 + jle L(205) movaps -29 * SIZE(A1), %xmm6 movaps -29 * SIZE(A2), %xmm7 decq I - jle .L204 + jle L(204) ALIGN_3 -.L203: +L(203): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -3115,10 +3115,10 @@ subq $1, I BRANCH - jg .L203 + jg L(203) ALIGN_3 -.L204: +L(204): movss %xmm6, %xmm4 shufps $0x39, %xmm4, %xmm4 pshufd $0xb1, %xmm4, %xmm14 @@ -3198,9 +3198,9 @@ subq $-16 * SIZE, X1 ALIGN_3 -.L205: +L(205): testq $4, MM - je .L207 + je L(207) movaps -29 * SIZE(A1), %xmm6 @@ -3252,9 +3252,9 @@ addq $8 * SIZE, A2 ALIGN_3 -.L207: +L(207): testq $2, MM - je .L208 + je L(208) movaps -29 * SIZE(A1), %xmm6 movaps -29 * SIZE(A2), %xmm7 @@ -3283,9 +3283,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L208: +L(208): testq $1, MM - je .L209 + je L(209) #ifdef movsd xorps %xmm8, %xmm8 @@ -3309,7 +3309,7 @@ SUBPS %xmm5, %xmm3 ALIGN_3 -.L209: +L(209): pcmpeqb %xmm5, %xmm5 psllq $63, %xmm5 @@ -3384,12 +3384,12 @@ addq INCY, Y1 cmpq $2, N - jge .L201 + jge L(201) ALIGN_3 -.L210: +L(210): cmpq $1, N - jl .L999 + jl L(999) leaq 32 * SIZE(BUFFER), X1 @@ -3400,7 +3400,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L21X + je L(21X) #ifdef movsd xorps %xmm8, %xmm8 @@ -3420,7 +3420,7 @@ addq $2 * SIZE, A1 addq $2 * SIZE, X1 ALIGN_3 -.L21X: +L(21X): #endif movaps -33 * SIZE(A1), %xmm4 @@ -3430,17 +3430,17 @@ movq MM, I sarq $3, I - jle .L215 + jle L(215) movaps -29 * SIZE(A1), %xmm5 movaps -25 * SIZE(A1), %xmm6 movaps -21 * SIZE(A1), %xmm7 decq I - jle .L214 + jle L(214) ALIGN_3 -.L213: +L(213): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -3494,10 +3494,10 @@ subq $1, I BRANCH - jg .L213 + jg L(213) ALIGN_3 -.L214: +L(214): movss %xmm5, %xmm4 shufps $0x39, %xmm4, %xmm4 pshufd $0xb1, %xmm4, %xmm14 @@ -3539,9 +3539,9 @@ subq $-16 * SIZE, X1 ALIGN_3 -.L215: +L(215): testq $4, MM - je .L217 + je L(217) movaps -29 * SIZE(A1), %xmm5 movaps -25 * SIZE(A1), %xmm6 @@ -3569,9 +3569,9 @@ addq $8 * SIZE, A1 ALIGN_3 -.L217: +L(217): testq $2, MM - je .L218 + je L(218) movaps -29 * SIZE(A1), %xmm5 @@ -3589,9 +3589,9 @@ addq $4 * SIZE, A1 ALIGN_3 -.L218: +L(218): testq $1, MM - je .L219 + je L(219) #ifdef movsd xorps %xmm8, %xmm8 @@ -3605,7 +3605,7 @@ SUBPS %xmm4, %xmm1 ALIGN_3 -.L219: +L(219): pcmpeqb %xmm5, %xmm5 psllq $63, %xmm5 @@ -3663,14 +3663,14 @@ movlps %xmm0, (Y1) addq INCY, Y1 - jmp .L999 + jmp L(999) -.L300: +L(300): cmpq $2, N - jl .L310 + jl L(310) ALIGN_3 -.L301: +L(301): subq $2, N leaq 32 * SIZE(BUFFER), X1 @@ -3686,7 +3686,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L30X + je L(30X) #ifdef movsd xorps %xmm8, %xmm8 @@ -3718,7 +3718,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, X1 ALIGN_3 -.L30X: +L(30X): #endif movaps -33 * SIZE(A1), %xmm4 @@ -3733,16 +3733,16 @@ movq MM, I sarq $3, I - jle .L305 + jle L(305) movaps -29 * SIZE(A1), %xmm6 movaps -31 * SIZE(A2), %xmm7 decq I - jle .L304 + jle L(304) ALIGN_3 -.L303: +L(303): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -3837,10 +3837,10 @@ subq $1, I BRANCH - jg .L303 + jg L(303) ALIGN_3 -.L304: +L(304): movss %xmm6, %xmm4 shufps $0x39, %xmm4, %xmm4 pshufd $0xb1, %xmm4, %xmm14 @@ -3920,9 +3920,9 @@ subq $-16 * SIZE, X1 ALIGN_3 -.L305: +L(305): testq $4, MM - je .L307 + je L(307) movaps -29 * SIZE(A1), %xmm6 @@ -3974,9 +3974,9 @@ addq $8 * SIZE, A2 ALIGN_3 -.L307: +L(307): testq $2, MM - je .L308 + je L(308) movaps -29 * SIZE(A1), %xmm6 movaps -31 * SIZE(A2), %xmm7 @@ -4005,9 +4005,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L308: +L(308): testq $1, MM - je .L309 + je L(309) #ifdef movsd xorps %xmm8, %xmm8 @@ -4031,7 +4031,7 @@ SUBPS %xmm5, %xmm3 ALIGN_3 -.L309: +L(309): pcmpeqb %xmm5, %xmm5 psllq $63, %xmm5 @@ -4106,12 +4106,12 @@ addq INCY, Y1 cmpq $2, N - jge .L301 + jge L(301) ALIGN_3 -.L310: +L(310): cmpq $1, N - jl .L999 + jl L(999) leaq 32 * SIZE(BUFFER), X1 @@ -4122,7 +4122,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L31X + je L(31X) #ifdef movsd xorps %xmm8, %xmm8 @@ -4142,7 +4142,7 @@ addq $2 * SIZE, A1 addq $2 * SIZE, X1 ALIGN_3 -.L31X: +L(31X): #endif movaps -33 * SIZE(A1), %xmm4 @@ -4152,17 +4152,17 @@ movq MM, I sarq $3, I - jle .L315 + jle L(315) movaps -29 * SIZE(A1), %xmm5 movaps -25 * SIZE(A1), %xmm6 movaps -21 * SIZE(A1), %xmm7 decq I - jle .L314 + jle L(314) ALIGN_3 -.L313: +L(313): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -4216,10 +4216,10 @@ subq $1, I BRANCH - jg .L313 + jg L(313) ALIGN_3 -.L314: +L(314): movss %xmm5, %xmm4 shufps $0x39, %xmm4, %xmm4 pshufd $0xb1, %xmm4, %xmm14 @@ -4261,9 +4261,9 @@ subq $-16 * SIZE, X1 ALIGN_3 -.L315: +L(315): testq $4, MM - je .L317 + je L(317) movaps -29 * SIZE(A1), %xmm5 movaps -25 * SIZE(A1), %xmm6 @@ -4291,9 +4291,9 @@ addq $8 * SIZE, A1 ALIGN_3 -.L317: +L(317): testq $2, MM - je .L318 + je L(318) movaps -29 * SIZE(A1), %xmm5 @@ -4311,9 +4311,9 @@ addq $4 * SIZE, A1 ALIGN_3 -.L318: +L(318): testq $1, MM - je .L319 + je L(319) #ifdef movsd xorps %xmm8, %xmm8 @@ -4327,7 +4327,7 @@ SUBPS %xmm4, %xmm1 ALIGN_3 -.L319: +L(319): pcmpeqb %xmm5, %xmm5 psllq $63, %xmm5 @@ -4388,12 +4388,12 @@ #endif ALIGN_3 -.L999: +L(999): movq M, I salq $ZBASE_SHIFT,I addq I,AA - jmp .L0t -.L999x: + jmp L(0t) +L(999x): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/copy.S b/kernel/x86_64/copy.S index 5729b29568..e28ba1ff23 100644 --- a/kernel/x86_64/copy.S +++ b/kernel/x86_64/copy.S @@ -63,22 +63,22 @@ EMMS testq N, N # if m == 0 goto End - jle .L999 + jle L(999) salq $BASE_SHIFT, INCX salq $BASE_SHIFT, INCY cmpq $SIZE, INCX # if incx != 1 - jne .L100 + jne L(100) cmpq $SIZE, INCY # if incy != 1 - jne .L100 + jne L(100) movq N, %rax # i = m sarq $3, %rax - jle .L20 + jle L(20) ALIGN_2 -.L11: +L(11): #ifdef XDOUBLE #ifdef PREFETCH @@ -194,16 +194,16 @@ addq $8 * SIZE, X addq $8 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_2 -.L20: +L(20): movq N, %rax andq $7, %rax - jle .L99 + jle L(99) ALIGN_2 -.L21: +L(21): #ifdef XDOUBLE movq 0(X), %mm0 movq 8(X), %mm1 @@ -217,21 +217,21 @@ addq $SIZE, X addq $SIZE, Y decq %rax - jg .L21 + jg L(21) -.L99: +L(99): xorq %rax,%rax EMMS ret ALIGN_3 -.L100: +L(100): movq N, %rax sarq $3, %rax - jle .L120 + jle L(120) ALIGN_2 -.L111: +L(111): #ifdef XDOUBLE movq 0(X), %mm0 movq 8(X), %mm1 @@ -333,15 +333,15 @@ #endif decq %rax - jg .L111 + jg L(111) -.L120: +L(120): movq N, %rax andq $7, %rax - jle .L999 + jle L(999) ALIGN_2 -.L121: +L(121): #ifdef XDOUBLE movq 0(X), %mm0 movq 8(X), %mm1 @@ -355,9 +355,9 @@ addq INCY, Y decq %rax - jg .L121 + jg L(121) -.L999: +L(999): xorq %rax,%rax EMMS ret diff --git a/kernel/x86_64/copy_sse.S b/kernel/x86_64/copy_sse.S index e949172931..dd96916da2 100644 --- a/kernel/x86_64/copy_sse.S +++ b/kernel/x86_64/copy_sse.S @@ -70,18 +70,18 @@ leaq (, INCY, SIZE), INCY cmpq $SIZE, INCX - jne .L50 + jne L(50) cmpq $SIZE, INCY - jne .L50 + jne L(50) cmpq $3, M - jle .L55 + jle L(55) subq $-32 * SIZE, X subq $-32 * SIZE, Y testq $SIZE, Y - je .L05 + je L(05) movss -32 * SIZE(X), %xmm0 movss %xmm0, -32 * SIZE(Y) @@ -90,25 +90,25 @@ decq M ALIGN_4 -.L05: +L(05): testq $2 * SIZE, Y - je .L10 + je L(10) movsd -32 * SIZE(X), %xmm0 movlps %xmm0, -32 * SIZE(Y) addq $2 * SIZE, X addq $2 * SIZE, Y subq $2, M - jle .L19 + jle L(19) ALIGN_4 -.L10: +L(10): testq $3 * SIZE, X - jne .L20 + jne L(20) movq M, %rax sarq $5, %rax - jle .L13 + jle L(13) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -120,10 +120,10 @@ movaps -4 * SIZE(X), %xmm7 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -163,10 +163,10 @@ subq $-32 * SIZE, Y subq $-32 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movaps %xmm0, -32 * SIZE(Y) movaps %xmm1, -28 * SIZE(Y) movaps %xmm2, -24 * SIZE(Y) @@ -180,9 +180,9 @@ subq $-32 * SIZE, X ALIGN_3 -.L13: +L(13): testq $16, M - jle .L14 + jle L(14) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -198,9 +198,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $8, M - jle .L15 + jle L(15) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -212,9 +212,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $4, M - jle .L16 + jle L(16) movaps -32 * SIZE(X), %xmm0 movaps %xmm0, -32 * SIZE(Y) @@ -223,9 +223,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $2, M - jle .L17 + jle L(17) movsd -32 * SIZE(X), %xmm0 movlps %xmm0, -32 * SIZE(Y) @@ -234,15 +234,15 @@ addq $2 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $1, M - jle .L19 + jle L(19) movss -32 * SIZE(X), %xmm0 movss %xmm0, -32 * SIZE(Y) ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -251,15 +251,15 @@ ALIGN_3 -.L20: +L(20): testq $SIZE, X - jne .L30 + jne L(30) movhps -32 * SIZE(X), %xmm0 movq M, %rax sarq $5, %rax - jle .L23 + jle L(23) movaps -30 * SIZE(X), %xmm1 movaps -26 * SIZE(X), %xmm2 @@ -270,10 +270,10 @@ movaps -6 * SIZE(X), %xmm7 decq %rax - jle .L22 + jle L(22) ALIGN_4 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) @@ -326,10 +326,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): shufps $0x4e, %xmm1, %xmm0 movaps %xmm0, -32 * SIZE(Y) movaps -2 * SIZE(X), %xmm0 @@ -359,9 +359,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L23: +L(23): testq $16, M - jle .L24 + jle L(24) ALIGN_3 movaps -30 * SIZE(X), %xmm1 @@ -384,9 +384,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $8, M - jle .L25 + jle L(25) ALIGN_3 movaps -30 * SIZE(X), %xmm1 @@ -403,9 +403,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $4, M - jle .L26 + jle L(26) ALIGN_3 movaps -30 * SIZE(X), %xmm1 @@ -416,9 +416,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $2, M - jle .L27 + jle L(27) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -429,9 +429,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $1, M - jle .L29 + jle L(29) ALIGN_3 movss -32 * SIZE(X), %xmm0 @@ -439,7 +439,7 @@ addq $SIZE, Y ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -447,15 +447,15 @@ ret ALIGN_3 -.L30: +L(30): testq $2 * SIZE, X - jne .L40 + jne L(40) movaps -33 * SIZE(X), %xmm0 movq M, %rax sarq $5, %rax - jle .L33 + jle L(33) movaps -29 * SIZE(X), %xmm1 movaps -25 * SIZE(X), %xmm2 @@ -466,10 +466,10 @@ movaps -5 * SIZE(X), %xmm7 decq %rax - jle .L32 + jle L(32) ALIGN_4 -.L31: +L(31): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -529,10 +529,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L32: +L(32): movss %xmm1, %xmm0 shufps $0x39, %xmm0, %xmm0 movaps %xmm0, -32 * SIZE(Y) @@ -570,9 +570,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L33: +L(33): testq $16, M - jle .L34 + jle L(34) ALIGN_3 movaps -29 * SIZE(X), %xmm1 @@ -602,9 +602,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L34: +L(34): testq $8, M - jle .L35 + jle L(35) ALIGN_3 movaps -29 * SIZE(X), %xmm1 @@ -623,9 +623,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L35: +L(35): testq $4, M - jle .L36 + jle L(36) ALIGN_3 movaps -29 * SIZE(X), %xmm1 @@ -639,9 +639,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L36: +L(36): testq $2, M - jle .L37 + jle L(37) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -651,9 +651,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L37: +L(37): testq $1, M - jle .L39 + jle L(39) ALIGN_3 movss -32 * SIZE(X), %xmm0 @@ -661,7 +661,7 @@ addq $SIZE, Y ALIGN_3 -.L39: +L(39): xorq %rax,%rax RESTOREREGISTERS @@ -669,12 +669,12 @@ ret ALIGN_3 -.L40: +L(40): movaps -35 * SIZE(X), %xmm0 movq M, %rax sarq $5, %rax - jle .L43 + jle L(43) movaps -31 * SIZE(X), %xmm1 movaps -27 * SIZE(X), %xmm2 @@ -685,10 +685,10 @@ movaps -7 * SIZE(X), %xmm7 decq %rax - jle .L42 + jle L(42) ALIGN_4 -.L41: +L(41): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -748,10 +748,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L42: +L(42): movss %xmm1, %xmm0 shufps $0x93, %xmm1, %xmm0 movaps %xmm0, -32 * SIZE(Y) @@ -789,9 +789,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L43: +L(43): testq $16, M - jle .L44 + jle L(44) ALIGN_3 movaps -31 * SIZE(X), %xmm1 @@ -821,9 +821,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L44: +L(44): testq $8, M - jle .L45 + jle L(45) ALIGN_3 movaps -31 * SIZE(X), %xmm1 @@ -843,9 +843,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L45: +L(45): testq $4, M - jle .L46 + jle L(46) ALIGN_3 movaps -31 * SIZE(X), %xmm1 @@ -859,9 +859,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L46: +L(46): testq $2, M - jle .L47 + jle L(47) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -871,9 +871,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L47: +L(47): testq $1, M - jle .L49 + jle L(49) ALIGN_3 movss -32 * SIZE(X), %xmm0 @@ -881,7 +881,7 @@ addq $SIZE, Y ALIGN_3 -.L49: +L(49): xorq %rax,%rax RESTOREREGISTERS @@ -889,13 +889,13 @@ ret ALIGN_4 -.L50: +L(50): movq M, %rax sarq $3, %rax - jle .L55 + jle L(55) ALIGN_3 -.L51: +L(51): movss (X), %xmm0 addq INCX, X movss (X), %xmm1 @@ -931,25 +931,25 @@ addq INCY, Y decq %rax - jg .L51 + jg L(51) ALIGN_3 -.L55: +L(55): movq M, %rax andq $7, %rax - jle .L57 + jle L(57) ALIGN_3 -.L56: +L(56): movss (X), %xmm0 addq INCX, X movss %xmm0, (Y) addq INCY, Y decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L57: +L(57): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/copy_sse2.S b/kernel/x86_64/copy_sse2.S index a5ab2ea91d..081c8850e9 100644 --- a/kernel/x86_64/copy_sse2.S +++ b/kernel/x86_64/copy_sse2.S @@ -70,26 +70,26 @@ leaq (, INCY, SIZE), INCY cmpq $SIZE, INCX - jne .L40 + jne L(40) cmpq $SIZE, INCY - jne .L40 + jne L(40) #ifdef ALIGNED_ACCESS testq $SIZE, Y #else testq $SIZE, X #endif - je .L10 + je L(10) movsd (X), %xmm0 movsd %xmm0, (Y) addq $1 * SIZE, X addq $1 * SIZE, Y decq M - jle .L19 + jle L(19) ALIGN_4 -.L10: +L(10): subq $-16 * SIZE, X subq $-16 * SIZE, Y @@ -98,11 +98,11 @@ #else testq $SIZE, Y #endif - jne .L20 + jne L(20) movq M, %rax sarq $4, %rax - jle .L13 + jle L(13) movups -16 * SIZE(X), %xmm0 movups -14 * SIZE(X), %xmm1 @@ -114,10 +114,10 @@ movups -2 * SIZE(X), %xmm7 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -157,10 +157,10 @@ subq $-16 * SIZE, Y subq $-16 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movups %xmm0, -16 * SIZE(Y) movups %xmm1, -14 * SIZE(Y) movups %xmm2, -12 * SIZE(Y) @@ -174,9 +174,9 @@ subq $-16 * SIZE, X ALIGN_3 -.L13: +L(13): testq $8, M - jle .L14 + jle L(14) ALIGN_3 movups -16 * SIZE(X), %xmm0 @@ -193,9 +193,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $4, M - jle .L15 + jle L(15) ALIGN_3 movups -16 * SIZE(X), %xmm0 @@ -208,9 +208,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $2, M - jle .L16 + jle L(16) ALIGN_3 movups -16 * SIZE(X), %xmm0 @@ -220,16 +220,16 @@ addq $2 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $1, M - jle .L19 + jle L(19) ALIGN_3 movsd -16 * SIZE(X), %xmm0 movsd %xmm0, -16 * SIZE(Y) ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -237,14 +237,14 @@ ret ALIGN_3 -.L20: +L(20): #ifdef ALIGNED_ACCESS movhps -16 * SIZE(X), %xmm0 movq M, %rax sarq $4, %rax - jle .L23 + jle L(23) movups -15 * SIZE(X), %xmm1 movups -13 * SIZE(X), %xmm2 @@ -255,10 +255,10 @@ movups -3 * SIZE(X), %xmm7 decq %rax - jle .L22 + jle L(22) ALIGN_4 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -310,10 +310,10 @@ subq $-16 * SIZE, X subq $-16 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): SHUFPD_1 %xmm1, %xmm0 movups %xmm0, -16 * SIZE(Y) LOAD(-1 * SIZE, X, %xmm0) @@ -340,9 +340,9 @@ subq $-16 * SIZE, Y ALIGN_3 -.L23: +L(23): testq $8, M - jle .L24 + jle L(24) ALIGN_3 movups -15 * SIZE(X), %xmm1 @@ -368,9 +368,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $4, M - jle .L25 + jle L(25) ALIGN_3 movups -15 * SIZE(X), %xmm1 @@ -387,9 +387,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $2, M - jle .L26 + jle L(26) ALIGN_3 movups -15 * SIZE(X), %xmm1 @@ -401,16 +401,16 @@ addq $2 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $1, M - jle .L29 + jle L(29) ALIGN_3 movsd -16 * SIZE(X), %xmm0 movsd %xmm0, -16 * SIZE(Y) ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -422,7 +422,7 @@ movq M, %rax sarq $4, %rax - jle .L23 + jle L(23) movups -16 * SIZE(X), %xmm0 movups -14 * SIZE(X), %xmm1 @@ -434,10 +434,10 @@ movups -2 * SIZE(X), %xmm7 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -485,10 +485,10 @@ subq $-16 * SIZE, Y subq $-16 * SIZE, X decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): movlps %xmm0, -16 * SIZE(Y) movhps %xmm0, -15 * SIZE(Y) movlps %xmm1, -14 * SIZE(Y) @@ -510,9 +510,9 @@ subq $-16 * SIZE, X ALIGN_3 -.L23: +L(23): testq $8, M - jle .L24 + jle L(24) ALIGN_3 movups -16 * SIZE(X), %xmm0 @@ -532,9 +532,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $4, M - jle .L25 + jle L(25) ALIGN_3 movups -16 * SIZE(X), %xmm0 @@ -548,9 +548,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $2, M - jle .L26 + jle L(26) ALIGN_3 movups -16 * SIZE(X), %xmm0 @@ -561,16 +561,16 @@ addq $2 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $1, M - jle .L29 + jle L(29) ALIGN_3 movsd -16 * SIZE(X), %xmm0 movsd %xmm0, -16 * SIZE(Y) ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -580,13 +580,13 @@ #endif -.L40: +L(40): movq M, %rax sarq $3, %rax - jle .L45 + jle L(45) ALIGN_3 -.L41: +L(41): movsd (X), %xmm0 addq INCX, X movhps (X), %xmm0 @@ -622,25 +622,25 @@ addq INCY, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L45: +L(45): movq M, %rax andq $7, %rax - jle .L47 + jle L(47) ALIGN_3 -.L46: +L(46): movsd (X), %xmm0 addq INCX, X movlps %xmm0, (Y) addq INCY, Y decq %rax - jg .L46 + jg L(46) ALIGN_3 -.L47: +L(47): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/daxpy_bulldozer.S b/kernel/x86_64/daxpy_bulldozer.S index 799dad02df..88dcc69830 100644 --- a/kernel/x86_64/daxpy_bulldozer.S +++ b/kernel/x86_64/daxpy_bulldozer.S @@ -87,15 +87,15 @@ leaq (, INCY, SIZE), INCY testq M, M - jle .L47 + jle L(47) cmpq $SIZE, INCX - jne .L40 + jne L(40) cmpq $SIZE, INCY - jne .L40 + jne L(40) testq $SIZE, Y - je .L10 + je L(10) movsd (X), %xmm0 mulsd ALPHA, %xmm0 @@ -104,16 +104,16 @@ addq $1 * SIZE, X addq $1 * SIZE, Y decq M - jle .L19 + jle L(19) ALIGN_4 -.L10: +L(10): subq $-16 * SIZE, X subq $-16 * SIZE, Y movq M, %rax sarq $4, %rax - jle .L13 + jle L(13) vmovups -16 * SIZE(X), %xmm0 vmovups -14 * SIZE(X), %xmm1 @@ -121,10 +121,10 @@ vmovups -10 * SIZE(X), %xmm3 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): prefetchnta A_PRE(Y) @@ -167,10 +167,10 @@ subq $-16 * SIZE, Y subq $-16 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): vmovups -8 * SIZE(X), %xmm4 vfmaddpd -16 * SIZE(Y), ALPHA, %xmm0 , %xmm0 @@ -201,12 +201,12 @@ subq $-16 * SIZE, X ALIGN_3 -.L13: +L(13): movq M, %rax andq $8, %rax - jle .L14 + jle L(14) ALIGN_3 vmovups -16 * SIZE(X), %xmm0 @@ -228,10 +228,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L14: +L(14): movq M, %rax andq $4, %rax - jle .L15 + jle L(15) ALIGN_3 vmovups -16 * SIZE(X), %xmm0 @@ -247,10 +247,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L15: +L(15): movq M, %rax andq $2, %rax - jle .L16 + jle L(16) ALIGN_3 vmovups -16 * SIZE(X), %xmm0 @@ -261,10 +261,10 @@ addq $2 * SIZE, Y ALIGN_3 -.L16: +L(16): movq M, %rax andq $1, %rax - jle .L19 + jle L(19) ALIGN_3 vmovsd -16 * SIZE(X), %xmm0 @@ -273,7 +273,7 @@ vmovsd %xmm0, -16 * SIZE(Y) ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -282,17 +282,17 @@ ALIGN_3 -.L40: +L(40): movq Y, YY movq M, %rax //If incx==0 || incy==0, avoid unloop. cmpq $0, INCX - je .L46 + je L(46) cmpq $0, INCY - je .L46 + je L(46) sarq $3, %rax - jle .L45 + jle L(45) prefetchnta 512(X) prefetchnta 512+64(X) @@ -305,7 +305,7 @@ prefetchnta 512+192(Y) ALIGN_3 -.L41: +L(41): vmovsd 0 * SIZE(X), %xmm0 addq INCX, X @@ -376,16 +376,16 @@ addq INCY, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L45: +L(45): movq M, %rax andq $7, %rax - jle .L47 + jle L(47) ALIGN_3 -.L46: +L(46): vmovsd (X), %xmm0 addq INCX, X @@ -395,10 +395,10 @@ addq INCY, Y decq %rax - jg .L46 + jg L(46) ALIGN_3 -.L47: +L(47): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/dcopy_bulldozer.S b/kernel/x86_64/dcopy_bulldozer.S index 87f1a4e31d..451f0f5800 100644 --- a/kernel/x86_64/dcopy_bulldozer.S +++ b/kernel/x86_64/dcopy_bulldozer.S @@ -69,29 +69,29 @@ leaq (, INCY, SIZE), INCY cmpq $SIZE, INCX - jne .L40 + jne L(40) cmpq $SIZE, INCY - jne .L40 + jne L(40) testq $SIZE, X - je .L10 + je L(10) vmovsd (X), %xmm0 vmovsd %xmm0, (Y) addq $1 * SIZE, X addq $1 * SIZE, Y decq M - jle .L19 + jle L(19) ALIGN_4 -.L10: +L(10): subq $-16 * SIZE, X subq $-16 * SIZE, Y movq M, %rax sarq $4, %rax - jle .L13 + jle L(13) vmovups -16 * SIZE(X), %xmm0 vmovups -14 * SIZE(X), %xmm1 @@ -103,10 +103,10 @@ vmovups -2 * SIZE(X), %xmm7 decq %rax - jle .L12 + jle L(12) ALIGN_4 -.L11: +L(11): prefetchnta A_PRE(X) nop @@ -139,10 +139,10 @@ subq $-16 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): vmovups %xmm0, -16 * SIZE(Y) vmovups %xmm1, -14 * SIZE(Y) vmovups %xmm2, -12 * SIZE(Y) @@ -156,9 +156,9 @@ subq $-16 * SIZE, X ALIGN_3 -.L13: +L(13): testq $8, M - jle .L14 + jle L(14) ALIGN_3 vmovups -16 * SIZE(X), %xmm0 @@ -175,9 +175,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $4, M - jle .L15 + jle L(15) ALIGN_3 vmovups -16 * SIZE(X), %xmm0 @@ -190,9 +190,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $2, M - jle .L16 + jle L(16) ALIGN_3 vmovups -16 * SIZE(X), %xmm0 @@ -202,16 +202,16 @@ addq $2 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $1, M - jle .L19 + jle L(19) ALIGN_3 vmovsd -16 * SIZE(X), %xmm0 vmovsd %xmm0, -16 * SIZE(Y) ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -221,13 +221,13 @@ -.L40: +L(40): movq M, %rax sarq $3, %rax - jle .L45 + jle L(45) ALIGN_3 -.L41: +L(41): vmovsd (X), %xmm0 addq INCX, X vmovsd (X), %xmm4 @@ -263,25 +263,25 @@ addq INCY, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L45: +L(45): movq M, %rax andq $7, %rax - jle .L47 + jle L(47) ALIGN_3 -.L46: +L(46): vmovsd (X), %xmm0 addq INCX, X vmovsd %xmm0, (Y) addq INCY, Y decq %rax - jg .L46 + jg L(46) ALIGN_3 -.L47: +L(47): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/ddot_bulldozer.S b/kernel/x86_64/ddot_bulldozer.S index 61c7571166..7a2d70885b 100644 --- a/kernel/x86_64/ddot_bulldozer.S +++ b/kernel/x86_64/ddot_bulldozer.S @@ -71,18 +71,18 @@ vxorps %xmm3, %xmm3 , %xmm3 cmpq $0, N - jle .L999 + jle L(999) cmpq $SIZE, INCX - jne .L50 + jne L(50) cmpq $SIZE, INCY - jne .L50 + jne L(50) subq $-16 * SIZE, X subq $-16 * SIZE, Y testq $SIZE, Y - je .L10 + je L(10) vmovsd -16 * SIZE(X), %xmm0 vmulsd -16 * SIZE(Y), %xmm0 , %xmm0 @@ -91,11 +91,11 @@ decq N ALIGN_2 -.L10: +L(10): movq N, %rax sarq $4, %rax - jle .L14 + jle L(14) vmovups -16 * SIZE(X), %xmm4 vmovups -14 * SIZE(X), %xmm5 @@ -108,11 +108,11 @@ vmovups -2 * SIZE(X), %xmm11 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): prefetchnta A_PRE(Y) vfmaddpd %xmm0 , -16 * SIZE(Y), %xmm4 , %xmm0 @@ -142,10 +142,10 @@ subq $-16 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): vfmaddpd %xmm0 , -16 * SIZE(Y), %xmm4 , %xmm0 vfmaddpd %xmm1 , -14 * SIZE(Y), %xmm5 , %xmm1 @@ -161,12 +161,12 @@ subq $-16 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $15, N - jle .L999 + jle L(999) testq $8, N - jle .L15 + jle L(15) vmovups -16 * SIZE(X), %xmm4 vmovups -14 * SIZE(X), %xmm5 @@ -182,9 +182,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $4, N - jle .L16 + jle L(16) vmovups -16 * SIZE(X), %xmm4 vmovups -14 * SIZE(X), %xmm5 @@ -196,9 +196,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $2, N - jle .L17 + jle L(17) vmovups -16 * SIZE(X), %xmm4 vfmaddpd %xmm0 , -16 * SIZE(Y), %xmm4 , %xmm0 @@ -208,24 +208,24 @@ addq $2 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $1, N - jle .L999 + jle L(999) vmovsd -16 * SIZE(X), %xmm4 vmovsd -16 * SIZE(Y), %xmm5 vfmaddpd %xmm0, %xmm4 , %xmm5 , %xmm0 - jmp .L999 + jmp L(999) ALIGN_3 -.L50: +L(50): movq N, %rax sarq $3, %rax - jle .L55 + jle L(55) ALIGN_3 -.L53: +L(53): vmovsd 0 * SIZE(X), %xmm4 @@ -276,16 +276,16 @@ vfmaddpd %xmm3 , %xmm7 , %xmm11, %xmm3 decq %rax - jg .L53 + jg L(53) ALIGN_3 -.L55: +L(55): movq N, %rax andq $7, %rax - jle .L999 + jle L(999) ALIGN_3 -.L56: +L(56): vmovsd 0 * SIZE(X), %xmm4 addq INCX, X vmovsd 0 * SIZE(Y), %xmm8 @@ -294,10 +294,10 @@ vfmaddpd %xmm0 , %xmm4 , %xmm8 , %xmm0 decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L999: +L(999): vaddpd %xmm1, %xmm0 , %xmm0 vaddpd %xmm3, %xmm2 , %xmm2 vaddpd %xmm2, %xmm0 , %xmm0 diff --git a/kernel/x86_64/dgemm_kernel_16x2_haswell.S b/kernel/x86_64/dgemm_kernel_16x2_haswell.S index 899c5f2419..67c19e0556 100644 --- a/kernel/x86_64/dgemm_kernel_16x2_haswell.S +++ b/kernel/x86_64/dgemm_kernel_16x2_haswell.S @@ -1685,13 +1685,13 @@ STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -1711,10 +1711,10 @@ movq Ndiv6, J cmpq $0, J - je .L2_0 + je L(2_0) ALIGN_4 -.L6_01: +L(6_01): // copy to sub buffer movq K, %rax salq $1,%rax // K * 2 ; read 2 values @@ -1723,10 +1723,10 @@ leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $3 , %rax // K / 8 - jz .L6_01a_2 + jz L(6_01a_2) ALIGN_4 -.L6_01a_1: +L(6_01a_1): prefetcht0 512(BO1) prefetcht0 512(BO2) @@ -1774,19 +1774,19 @@ addq $ 12*SIZE,BO decq %rax - jnz .L6_01a_1 + jnz L(6_01a_1) -.L6_01a_2: +L(6_01a_2): movq K, %rax andq $7, %rax // K % 8 - jz .L6_02c + jz L(6_02c) ALIGN_4 -.L6_02b: +L(6_02b): vmovups 0 * SIZE(BO1), %xmm0 vmovsd 0 * SIZE(BO2), %xmm2 @@ -1796,9 +1796,9 @@ addq $ 2*SIZE,BO2 addq $ 3*SIZE,BO decq %rax - jnz .L6_02b + jnz L(6_02b) -.L6_02c: +L(6_02c): movq K, %rax salq $1,%rax // K * 2 @@ -1807,10 +1807,10 @@ leaq BUFFER2, BO // second buffer to BO movq K, %rax sarq $3 , %rax // K / 8 - jz .L6_02c_2 + jz L(6_02c_2) ALIGN_4 -.L6_02c_1: +L(6_02c_1): prefetcht0 512(BO2) prefetchw 512(BO) @@ -1857,17 +1857,17 @@ addq $12*SIZE,BO decq %rax - jnz .L6_02c_1 + jnz L(6_02c_1) -.L6_02c_2: +L(6_02c_2): movq K, %rax andq $7, %rax // K % 8 - jz .L6_03c + jz L(6_03c) ALIGN_4 -.L6_03b: +L(6_03b): vmovsd 1*SIZE(BO1), %xmm0 vmovups 0*SIZE(BO2), %xmm1 @@ -1877,14 +1877,14 @@ addq $2*SIZE,BO2 addq $3*SIZE,BO decq %rax - jnz .L6_03b + jnz L(6_03b) -.L6_03c: +L(6_03c): movq BO2, B // next offset of B -.L6_10: +L(6_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -1895,11 +1895,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L6_20 + je L(6_20) ALIGN_4 -.L6_11: +L(6_11): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -1915,11 +1915,11 @@ movq K, %rax sarq $1, %rax // K / 8 - je .L6_16 + je L(6_16) ALIGN_5 -.L6_12: +L(6_12): /* prefetcht0 B_PR1(BO) prefetcht0 B_PR1+64(BO) @@ -1937,50 +1937,50 @@ KERNEL16x3_SUBN */ dec %rax - jne .L6_12 + jne L(6_12) -.L6_16: +L(6_16): movq K, %rax andq $1, %rax # if (k & 1) - je .L6_19 + je L(6_19) ALIGN_4 -.L6_17: +L(6_17): KERNEL16x3_SUBN dec %rax - jne .L6_17 + jne L(6_17) ALIGN_4 -.L6_19: +L(6_19): SAVE16x3 addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L6_11 + jg L(6_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L6_20: +L(6_20): // Test rest of M testq $15, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N testq $8, M - jz .L6_21pre + jz L(6_21pre) ALIGN_4 /**************************************************************************/ -.L6_20_1: +L(6_20_1): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -1989,11 +1989,11 @@ movq K, %rax sarq $3, %rax - je .L6_20_6 + je L(6_20_6) ALIGN_4 -.L6_20_2: +L(6_20_2): KERNEL8x3_SUBN KERNEL8x3_SUBN @@ -2005,28 +2005,28 @@ KERNEL8x3_SUBN KERNEL8x3_SUBN dec %rax - jne .L6_20_2 + jne L(6_20_2) ALIGN_4 -.L6_20_6: +L(6_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_20_9 + je L(6_20_9) ALIGN_4 -.L6_20_7: +L(6_20_7): KERNEL8x3_SUBN dec %rax - jne .L6_20_7 + jne L(6_20_7) ALIGN_4 -.L6_20_9: +L(6_20_9): SAVE8x3 @@ -2037,13 +2037,13 @@ /**************************************************************************/ -.L6_21pre: +L(6_21pre): testq $4, M - jz .L6_30 + jz L(6_30) ALIGN_4 -.L6_21: +L(6_21): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -2052,11 +2052,11 @@ movq K, %rax sarq $3, %rax - je .L6_26 + je L(6_26) ALIGN_4 -.L6_22: +L(6_22): KERNEL4x3_SUBN KERNEL4x3_SUBN @@ -2068,27 +2068,27 @@ KERNEL4x3_SUBN KERNEL4x3_SUBN dec %rax - jne .L6_22 + jne L(6_22) ALIGN_4 -.L6_26: +L(6_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_29 + je L(6_29) ALIGN_4 -.L6_27: +L(6_27): KERNEL4x3_SUBN dec %rax - jne .L6_27 + jne L(6_27) ALIGN_4 -.L6_29: +L(6_29): SAVE4x3 @@ -2096,13 +2096,13 @@ ALIGN_4 -.L6_30: +L(6_30): testq $2, M - jz .L6_40 + jz L(6_40) ALIGN_4 -.L6_31: +L(6_31): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -2111,10 +2111,10 @@ movq K, %rax sarq $3, %rax - je .L6_36 + je L(6_36) ALIGN_4 -.L6_32: +L(6_32): KERNEL2x3_SUBN KERNEL2x3_SUBN @@ -2126,40 +2126,40 @@ KERNEL2x3_SUBN KERNEL2x3_SUBN dec %rax - jne .L6_32 + jne L(6_32) ALIGN_4 -.L6_36: +L(6_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_39 + je L(6_39) ALIGN_4 -.L6_37: +L(6_37): KERNEL2x3_SUBN dec %rax - jne .L6_37 + jne L(6_37) ALIGN_4 -.L6_39: +L(6_39): SAVE2x3 addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L6_40: +L(6_40): testq $1, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N ALIGN_4 -.L6_41: +L(6_41): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -2168,11 +2168,11 @@ movq K, %rax sarq $3,%rax - je .L6_46 + je L(6_46) ALIGN_4 -.L6_42: +L(6_42): KERNEL1x3_SUBN KERNEL1x3_SUBN @@ -2185,27 +2185,27 @@ KERNEL1x3_SUBN dec %rax - jne .L6_42 + jne L(6_42) ALIGN_4 -.L6_46: +L(6_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_49 + je L(6_49) ALIGN_4 -.L6_47: +L(6_47): KERNEL1x3_SUBN dec %rax - jne .L6_47 + jne L(6_47) ALIGN_4 -.L6_49: +L(6_49): SAVE1x3 @@ -2217,7 +2217,7 @@ /***************************************************************************************************************/ -.L7_10: +L(7_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -2228,11 +2228,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L7_20 + je L(7_20) ALIGN_4 -.L7_11: +L(7_11): leaq BUFFER2, BO // second buffer to BO addq $12 * SIZE, BO @@ -2248,10 +2248,10 @@ movq K, %rax sarq $3, %rax // K / 8 - je .L7_16 + je L(7_16) ALIGN_5 -.L7_12: +L(7_12): /* prefetcht0 B_PR1(BO) prefetcht0 B_PR1+64(BO) @@ -2267,50 +2267,50 @@ KERNEL16x3_SUBN KERNEL16x3_SUBN dec %rax - jne .L7_12 + jne L(7_12) ALIGN_4 -.L7_16: +L(7_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_19 + je L(7_19) ALIGN_5 -.L7_17: +L(7_17): KERNEL16x3_SUBN dec %rax - jne .L7_17 + jne L(7_17) -.L7_19: +L(7_19): SAVE16x3 addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L7_11 + jg L(7_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L7_20: +L(7_20): // Test rest of M testq $15, M - jz .L7_60 // to next 3 lines of N + jz L(7_60) // to next 3 lines of N testq $8, M - jz .L7_21pre + jz L(7_21pre) ALIGN_4 /**************************************************************************/ -.L7_20_1: +L(7_20_1): leaq BUFFER2, BO // first buffer to BO addq $12 * SIZE, BO @@ -2319,11 +2319,11 @@ movq K, %rax sarq $3, %rax - je .L7_20_6 + je L(7_20_6) ALIGN_4 -.L7_20_2: +L(7_20_2): KERNEL8x3_SUBN KERNEL8x3_SUBN @@ -2336,26 +2336,26 @@ KERNEL8x3_SUBN dec %rax - jne .L7_20_2 + jne L(7_20_2) ALIGN_4 -.L7_20_6: +L(7_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_20_9 + je L(7_20_9) ALIGN_4 -.L7_20_7: +L(7_20_7): KERNEL8x3_SUBN dec %rax - jne .L7_20_7 + jne L(7_20_7) ALIGN_4 -.L7_20_9: +L(7_20_9): SAVE8x3 @@ -2366,13 +2366,13 @@ /**************************************************************************/ -.L7_21pre: +L(7_21pre): testq $4, M - jz .L7_30 + jz L(7_30) ALIGN_4 -.L7_21: +L(7_21): leaq BUFFER2, BO // second buffer to BO addq $12 * SIZE, BO @@ -2381,11 +2381,11 @@ movq K, %rax sarq $3, %rax - je .L7_26 + je L(7_26) ALIGN_4 -.L7_22: +L(7_22): KERNEL4x3_SUBN KERNEL4x3_SUBN @@ -2398,27 +2398,27 @@ KERNEL4x3_SUBN dec %rax - jne .L7_22 + jne L(7_22) ALIGN_4 -.L7_26: +L(7_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_29 + je L(7_29) ALIGN_4 -.L7_27: +L(7_27): KERNEL4x3_SUBN dec %rax - jne .L7_27 + jne L(7_27) ALIGN_4 -.L7_29: +L(7_29): SAVE4x3 @@ -2426,13 +2426,13 @@ ALIGN_4 -.L7_30: +L(7_30): testq $2, M - jz .L7_40 + jz L(7_40) ALIGN_4 -.L7_31: +L(7_31): leaq BUFFER2, BO // second buffer to BO addq $12 * SIZE, BO @@ -2441,11 +2441,11 @@ movq K, %rax sarq $3, %rax - je .L7_36 + je L(7_36) ALIGN_4 -.L7_32: +L(7_32): KERNEL2x3_SUBN KERNEL2x3_SUBN @@ -2458,40 +2458,40 @@ KERNEL2x3_SUBN dec %rax - jne .L7_32 + jne L(7_32) ALIGN_4 -.L7_36: +L(7_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_39 + je L(7_39) ALIGN_4 -.L7_37: +L(7_37): KERNEL2x3_SUBN dec %rax - jne .L7_37 + jne L(7_37) ALIGN_4 -.L7_39: +L(7_39): SAVE2x3 addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L7_40: +L(7_40): testq $1, M - jz .L7_60 // to next 3 lines of N + jz L(7_60) // to next 3 lines of N ALIGN_4 -.L7_41: +L(7_41): leaq BUFFER2, BO // second buffer to BO addq $12 * SIZE, BO @@ -2500,11 +2500,11 @@ movq K, %rax sarq $3, %rax - je .L7_46 + je L(7_46) ALIGN_4 -.L7_42: +L(7_42): KERNEL1x3_SUBN KERNEL1x3_SUBN KERNEL1x3_SUBN @@ -2516,27 +2516,27 @@ KERNEL1x3_SUBN dec %rax - jne .L7_42 + jne L(7_42) ALIGN_4 -.L7_46: +L(7_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_49 + je L(7_49) ALIGN_4 -.L7_47: +L(7_47): KERNEL1x3_SUBN dec %rax - jne .L7_47 + jne L(7_47) ALIGN_4 -.L7_49: +L(7_49): SAVE1x3 @@ -2545,15 +2545,15 @@ -.L7_60: +L(7_60): decq J // j -- - jg .L6_01 + jg L(6_01) -.L2_0: +L(2_0): cmpq $0, Nmod6 // N % 6 == 0 - je .L999 + je L(999) /************************************************************************************************ * Loop for Nmod6 / 2 > 0 @@ -2561,19 +2561,19 @@ movq Nmod6, J sarq $1, J // j = j / 2 - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L2_01b + jz L(2_01b) ALIGN_4 -.L2_01a: +L(2_01a): prefetcht0 512(BO1) prefetchw 512(BO) @@ -2590,30 +2590,30 @@ addq $8*SIZE,BO1 addq $8*SIZE,BO decq %rax - jnz .L2_01a + jnz L(2_01a) -.L2_01b: +L(2_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L2_02d + jz L(2_02d) ALIGN_4 -.L2_02c: +L(2_02c): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02c + jnz L(2_02c) -.L2_02d: +L(2_02d): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -2623,11 +2623,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2636,7 +2636,7 @@ movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2647,7 +2647,7 @@ negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI,8) KERNEL16x2_1 @@ -2661,7 +2661,7 @@ KERNEL16x2_3 KERNEL16x2_4 - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI,8) KERNEL16x2_1 @@ -2675,16 +2675,16 @@ KERNEL16x2_3 KERNEL16x2_4 - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2696,39 +2696,39 @@ negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE16x2 addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 3 lines of N + jz L(2_60) // to next 3 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2737,7 +2737,7 @@ movq K, %rax andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2748,7 +2748,7 @@ negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1 @@ -2762,7 +2762,7 @@ KERNEL8x2_3 KERNEL8x2_4 - je .L2_20_6 + je L(2_20_6) prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1 @@ -2776,16 +2776,16 @@ KERNEL8x2_3 KERNEL8x2_4 - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2797,15 +2797,15 @@ negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): SAVE8x2 @@ -2816,13 +2816,13 @@ /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2831,7 +2831,7 @@ movq K, %rax andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -2842,7 +2842,7 @@ negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1 @@ -2856,7 +2856,7 @@ KERNEL4x2_3 KERNEL4x2_4 - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1 @@ -2870,16 +2870,16 @@ KERNEL4x2_3 KERNEL4x2_4 - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2891,15 +2891,15 @@ negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): SAVE4x2 @@ -2907,13 +2907,13 @@ ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2922,7 +2922,7 @@ movq K, %rax andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2933,7 +2933,7 @@ negq %rax ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_1 KERNEL2x2_2 @@ -2945,7 +2945,7 @@ KERNEL2x2_3 KERNEL2x2_4 - je .L2_36 + je L(2_36) KERNEL2x2_1 KERNEL2x2_2 @@ -2957,16 +2957,16 @@ KERNEL2x2_3 KERNEL2x2_4 - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2978,28 +2978,28 @@ negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): SAVE2x2 addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -3008,7 +3008,7 @@ movq K, %rax andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3018,7 +3018,7 @@ negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_1 KERNEL1x2_2 @@ -3030,7 +3030,7 @@ KERNEL1x2_3 KERNEL1x2_4 - je .L2_46 + je L(2_46) KERNEL1x2_1 KERNEL1x2_2 @@ -3042,16 +3042,16 @@ KERNEL1x2_3 KERNEL1x2_4 - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3062,29 +3062,29 @@ negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): SAVE1x2 addq $1 * SIZE, CO1 # coffset += 1 ALIGN_4 -.L2_60: +L(2_60): decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -3092,30 +3092,30 @@ movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -3125,11 +3125,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3138,7 +3138,7 @@ movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -3148,7 +3148,7 @@ negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI,8) KERNEL16x1_1 @@ -3161,7 +3161,7 @@ KERNEL16x1_3 KERNEL16x1_4 - je .L1_16 + je L(1_16) prefetcht0 B_PR1(BO,BI,8) KERNEL16x1_1 @@ -3174,16 +3174,16 @@ KERNEL16x1_3 KERNEL16x1_4 - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -3194,39 +3194,39 @@ negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE16x1 addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3235,7 +3235,7 @@ movq K, %rax andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -3245,7 +3245,7 @@ negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1 @@ -3258,7 +3258,7 @@ KERNEL8x1_3 KERNEL8x1_4 - je .L1_20_6 + je L(1_20_6) prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1 @@ -3271,16 +3271,16 @@ KERNEL8x1_3 KERNEL8x1_4 - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -3291,15 +3291,15 @@ negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): SAVE8x1 @@ -3310,13 +3310,13 @@ /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3325,7 +3325,7 @@ movq K, %rax andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -3335,7 +3335,7 @@ negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1 @@ -3348,7 +3348,7 @@ KERNEL4x1_3 KERNEL4x1_4 - je .L1_26 + je L(1_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1 @@ -3361,16 +3361,16 @@ KERNEL4x1_3 KERNEL4x1_4 - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -3381,15 +3381,15 @@ negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): SAVE4x1 @@ -3397,13 +3397,13 @@ ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3412,7 +3412,7 @@ movq K, %rax andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -3422,7 +3422,7 @@ negq %rax ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_1 KERNEL2x1_2 @@ -3434,7 +3434,7 @@ KERNEL2x1_3 KERNEL2x1_4 - je .L1_36 + je L(1_36) KERNEL2x1_1 KERNEL2x1_2 @@ -3446,16 +3446,16 @@ KERNEL2x1_3 KERNEL2x1_4 - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -3466,28 +3466,28 @@ negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): SAVE2x1 addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3496,7 +3496,7 @@ movq K, %rax andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -3505,7 +3505,7 @@ negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_1 KERNEL1x1_2 @@ -3517,7 +3517,7 @@ KERNEL1x1_3 KERNEL1x1_4 - je .L1_46 + je L(1_46) KERNEL1x1_1 KERNEL1x1_2 @@ -3529,16 +3529,16 @@ KERNEL1x1_3 KERNEL1x1_4 - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -3548,15 +3548,15 @@ negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): SAVE1x1 @@ -3564,7 +3564,7 @@ ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp @@ -3654,13 +3654,13 @@ STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -3689,19 +3689,19 @@ movq Ndiv6, J cmpq $0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L2_01b + jz L(2_01b) ALIGN_4 -.L2_01a: +L(2_01a): prefetcht0 512(BO1) prefetchw 512(BO) @@ -3718,30 +3718,30 @@ addq $8*SIZE,BO1 addq $8*SIZE,BO decq %rax - jnz .L2_01a + jnz L(2_01a) -.L2_01b: +L(2_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L2_02d + jz L(2_02d) ALIGN_4 -.L2_02c: +L(2_02c): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02c + jnz L(2_02c) -.L2_02d: +L(2_02d): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -3755,11 +3755,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3796,7 +3796,7 @@ #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3807,7 +3807,7 @@ negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI,8) KERNEL16x2_1 @@ -3821,7 +3821,7 @@ KERNEL16x2_3 KERNEL16x2_4 - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI,8) KERNEL16x2_1 @@ -3835,12 +3835,12 @@ KERNEL16x2_3 KERNEL16x2_4 - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3848,7 +3848,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3860,15 +3860,15 @@ negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE16x2 @@ -3890,25 +3890,25 @@ addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 3 lines of N + jz L(2_60) // to next 3 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3946,7 +3946,7 @@ andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3957,7 +3957,7 @@ negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1 @@ -3971,7 +3971,7 @@ KERNEL8x2_3 KERNEL8x2_4 - je .L2_20_6 + je L(2_20_6) prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1 @@ -3985,12 +3985,12 @@ KERNEL8x2_3 KERNEL8x2_4 - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -3998,7 +3998,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4010,15 +4010,15 @@ negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): SAVE8x2 @@ -4045,13 +4045,13 @@ /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4089,7 +4089,7 @@ andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -4100,7 +4100,7 @@ negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1 @@ -4114,7 +4114,7 @@ KERNEL4x2_3 KERNEL4x2_4 - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1 @@ -4128,12 +4128,12 @@ KERNEL4x2_3 KERNEL4x2_4 - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4141,7 +4141,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4153,15 +4153,15 @@ negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): SAVE4x2 @@ -4185,13 +4185,13 @@ ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4229,7 +4229,7 @@ andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4240,7 +4240,7 @@ negq %rax ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_1 KERNEL2x2_2 @@ -4252,7 +4252,7 @@ KERNEL2x2_3 KERNEL2x2_4 - je .L2_36 + je L(2_36) KERNEL2x2_1 KERNEL2x2_2 @@ -4264,12 +4264,12 @@ KERNEL2x2_3 KERNEL2x2_4 - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -4277,7 +4277,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4289,15 +4289,15 @@ negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): SAVE2x2 @@ -4320,13 +4320,13 @@ addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4362,7 +4362,7 @@ #endif andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4372,7 +4372,7 @@ negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_1 KERNEL1x2_2 @@ -4384,7 +4384,7 @@ KERNEL1x2_3 KERNEL1x2_4 - je .L2_46 + je L(2_46) KERNEL1x2_1 KERNEL1x2_2 @@ -4396,12 +4396,12 @@ KERNEL1x2_3 KERNEL1x2_4 - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -4409,7 +4409,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4420,15 +4420,15 @@ negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): SAVE1x2 @@ -4454,17 +4454,17 @@ -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -4472,30 +4472,30 @@ movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -4509,11 +4509,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4549,7 +4549,7 @@ #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -4559,7 +4559,7 @@ negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI,8) KERNEL16x1_1 @@ -4572,7 +4572,7 @@ KERNEL16x1_3 KERNEL16x1_4 - je .L1_16 + je L(1_16) prefetcht0 B_PR1(BO,BI,8) KERNEL16x1_1 @@ -4585,12 +4585,12 @@ KERNEL16x1_3 KERNEL16x1_4 - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -4598,7 +4598,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -4609,15 +4609,15 @@ negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE16x1 @@ -4638,25 +4638,25 @@ addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4693,7 +4693,7 @@ andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -4703,7 +4703,7 @@ negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1 @@ -4716,7 +4716,7 @@ KERNEL8x1_3 KERNEL8x1_4 - je .L1_20_6 + je L(1_20_6) prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1 @@ -4729,12 +4729,12 @@ KERNEL8x1_3 KERNEL8x1_4 - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -4742,7 +4742,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -4753,15 +4753,15 @@ negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): SAVE8x1 @@ -4787,13 +4787,13 @@ /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4830,7 +4830,7 @@ andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -4840,7 +4840,7 @@ negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1 @@ -4853,7 +4853,7 @@ KERNEL4x1_3 KERNEL4x1_4 - je .L1_26 + je L(1_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1 @@ -4866,12 +4866,12 @@ KERNEL4x1_3 KERNEL4x1_4 - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4879,7 +4879,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -4890,15 +4890,15 @@ negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): SAVE4x1 @@ -4921,13 +4921,13 @@ ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4964,7 +4964,7 @@ andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -4974,7 +4974,7 @@ negq %rax ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_1 KERNEL2x1_2 @@ -4986,7 +4986,7 @@ KERNEL2x1_3 KERNEL2x1_4 - je .L1_36 + je L(1_36) KERNEL2x1_1 KERNEL2x1_2 @@ -4998,12 +4998,12 @@ KERNEL2x1_3 KERNEL2x1_4 - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -5011,7 +5011,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -5022,15 +5022,15 @@ negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): SAVE2x1 @@ -5052,13 +5052,13 @@ addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5093,7 +5093,7 @@ #endif andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -5102,7 +5102,7 @@ negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_1 KERNEL1x1_2 @@ -5114,7 +5114,7 @@ KERNEL1x1_3 KERNEL1x1_4 - je .L1_46 + je L(1_46) KERNEL1x1_1 KERNEL1x1_2 @@ -5126,12 +5126,12 @@ KERNEL1x1_3 KERNEL1x1_4 - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -5139,7 +5139,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -5149,15 +5149,15 @@ negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): SAVE1x1 @@ -5179,7 +5179,7 @@ ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/dgemm_kernel_16x2_skylakex.S b/kernel/x86_64/dgemm_kernel_16x2_skylakex.S index 91ac512805..02e04240d3 100644 --- a/kernel/x86_64/dgemm_kernel_16x2_skylakex.S +++ b/kernel/x86_64/dgemm_kernel_16x2_skylakex.S @@ -1608,13 +1608,13 @@ STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -1634,10 +1634,10 @@ movq Ndiv6, J cmpq $0, J - je .L2_0 + je L(2_0) ALIGN_4 -.L6_01: +L(6_01): // copy to sub buffer movq K, %rax salq $1,%rax // K * 2 ; read 2 values @@ -1646,10 +1646,10 @@ leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $3 , %rax // K / 8 - jz .L6_01a_2 + jz L(6_01a_2) ALIGN_4 -.L6_01a_1: +L(6_01a_1): prefetcht0 512(BO1) prefetcht0 512(BO2) @@ -1697,19 +1697,19 @@ addq $ 12*SIZE,BO decq %rax - jnz .L6_01a_1 + jnz L(6_01a_1) -.L6_01a_2: +L(6_01a_2): movq K, %rax andq $7, %rax // K % 8 - jz .L6_02c + jz L(6_02c) ALIGN_4 -.L6_02b: +L(6_02b): vmovups 0 * SIZE(BO1), %xmm0 vmovsd 0 * SIZE(BO2), %xmm2 @@ -1719,9 +1719,9 @@ addq $ 2*SIZE,BO2 addq $ 3*SIZE,BO decq %rax - jnz .L6_02b + jnz L(6_02b) -.L6_02c: +L(6_02c): movq K, %rax salq $1,%rax // K * 2 @@ -1730,10 +1730,10 @@ leaq BUFFER2, BO // second buffer to BO movq K, %rax sarq $3 , %rax // K / 8 - jz .L6_02c_2 + jz L(6_02c_2) ALIGN_4 -.L6_02c_1: +L(6_02c_1): prefetcht0 512(BO2) prefetchw 512(BO) @@ -1780,17 +1780,17 @@ addq $12*SIZE,BO decq %rax - jnz .L6_02c_1 + jnz L(6_02c_1) -.L6_02c_2: +L(6_02c_2): movq K, %rax andq $7, %rax // K % 8 - jz .L6_03c + jz L(6_03c) ALIGN_4 -.L6_03b: +L(6_03b): vmovsd 1*SIZE(BO1), %xmm0 vmovups 0*SIZE(BO2), %xmm1 @@ -1800,14 +1800,14 @@ addq $2*SIZE,BO2 addq $3*SIZE,BO decq %rax - jnz .L6_03b + jnz L(6_03b) -.L6_03c: +L(6_03c): movq BO2, B // next offset of B -.L6_10: +L(6_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -1818,11 +1818,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L6_20 + je L(6_20) ALIGN_4 -.L6_11: +L(6_11): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -1838,11 +1838,11 @@ movq K, %rax sarq $1, %rax // K / 8 - je .L6_16 + je L(6_16) ALIGN_5 -.L6_12: +L(6_12): /* prefetcht0 B_PR1(BO) prefetcht0 B_PR1+64(BO) @@ -1860,50 +1860,50 @@ KERNEL16x3_SUBN */ dec %rax - jne .L6_12 + jne L(6_12) -.L6_16: +L(6_16): movq K, %rax andq $1, %rax # if (k & 1) - je .L6_19 + je L(6_19) ALIGN_4 -.L6_17: +L(6_17): KERNEL16x3_SUBN dec %rax - jne .L6_17 + jne L(6_17) ALIGN_4 -.L6_19: +L(6_19): SAVE16x3 addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L6_11 + jg L(6_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L6_20: +L(6_20): // Test rest of M testq $15, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N testq $8, M - jz .L6_21pre + jz L(6_21pre) ALIGN_4 /**************************************************************************/ -.L6_20_1: +L(6_20_1): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -1912,11 +1912,11 @@ movq K, %rax sarq $3, %rax - je .L6_20_6 + je L(6_20_6) ALIGN_4 -.L6_20_2: +L(6_20_2): KERNEL8x3_SUBN KERNEL8x3_SUBN @@ -1928,28 +1928,28 @@ KERNEL8x3_SUBN KERNEL8x3_SUBN dec %rax - jne .L6_20_2 + jne L(6_20_2) ALIGN_4 -.L6_20_6: +L(6_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_20_9 + je L(6_20_9) ALIGN_4 -.L6_20_7: +L(6_20_7): KERNEL8x3_SUBN dec %rax - jne .L6_20_7 + jne L(6_20_7) ALIGN_4 -.L6_20_9: +L(6_20_9): SAVE8x3 @@ -1960,13 +1960,13 @@ /**************************************************************************/ -.L6_21pre: +L(6_21pre): testq $4, M - jz .L6_30 + jz L(6_30) ALIGN_4 -.L6_21: +L(6_21): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -1975,11 +1975,11 @@ movq K, %rax sarq $3, %rax - je .L6_26 + je L(6_26) ALIGN_4 -.L6_22: +L(6_22): KERNEL4x3_SUBN KERNEL4x3_SUBN @@ -1991,27 +1991,27 @@ KERNEL4x3_SUBN KERNEL4x3_SUBN dec %rax - jne .L6_22 + jne L(6_22) ALIGN_4 -.L6_26: +L(6_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_29 + je L(6_29) ALIGN_4 -.L6_27: +L(6_27): KERNEL4x3_SUBN dec %rax - jne .L6_27 + jne L(6_27) ALIGN_4 -.L6_29: +L(6_29): SAVE4x3 @@ -2019,13 +2019,13 @@ ALIGN_4 -.L6_30: +L(6_30): testq $2, M - jz .L6_40 + jz L(6_40) ALIGN_4 -.L6_31: +L(6_31): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -2034,10 +2034,10 @@ movq K, %rax sarq $3, %rax - je .L6_36 + je L(6_36) ALIGN_4 -.L6_32: +L(6_32): KERNEL2x3_SUBN KERNEL2x3_SUBN @@ -2049,40 +2049,40 @@ KERNEL2x3_SUBN KERNEL2x3_SUBN dec %rax - jne .L6_32 + jne L(6_32) ALIGN_4 -.L6_36: +L(6_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_39 + je L(6_39) ALIGN_4 -.L6_37: +L(6_37): KERNEL2x3_SUBN dec %rax - jne .L6_37 + jne L(6_37) ALIGN_4 -.L6_39: +L(6_39): SAVE2x3 addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L6_40: +L(6_40): testq $1, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N ALIGN_4 -.L6_41: +L(6_41): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -2091,11 +2091,11 @@ movq K, %rax sarq $3,%rax - je .L6_46 + je L(6_46) ALIGN_4 -.L6_42: +L(6_42): KERNEL1x3_SUBN KERNEL1x3_SUBN @@ -2108,27 +2108,27 @@ KERNEL1x3_SUBN dec %rax - jne .L6_42 + jne L(6_42) ALIGN_4 -.L6_46: +L(6_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_49 + je L(6_49) ALIGN_4 -.L6_47: +L(6_47): KERNEL1x3_SUBN dec %rax - jne .L6_47 + jne L(6_47) ALIGN_4 -.L6_49: +L(6_49): SAVE1x3 @@ -2140,7 +2140,7 @@ /***************************************************************************************************************/ -.L7_10: +L(7_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -2151,11 +2151,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L7_20 + je L(7_20) ALIGN_4 -.L7_11: +L(7_11): leaq BUFFER2, BO // second buffer to BO addq $12 * SIZE, BO @@ -2171,10 +2171,10 @@ movq K, %rax sarq $3, %rax // K / 8 - je .L7_16 + je L(7_16) ALIGN_5 -.L7_12: +L(7_12): /* prefetcht0 B_PR1(BO) prefetcht0 B_PR1+64(BO) @@ -2190,50 +2190,50 @@ KERNEL16x3_SUBN KERNEL16x3_SUBN dec %rax - jne .L7_12 + jne L(7_12) ALIGN_4 -.L7_16: +L(7_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_19 + je L(7_19) ALIGN_5 -.L7_17: +L(7_17): KERNEL16x3_SUBN dec %rax - jne .L7_17 + jne L(7_17) -.L7_19: +L(7_19): SAVE16x3 addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L7_11 + jg L(7_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L7_20: +L(7_20): // Test rest of M testq $15, M - jz .L7_60 // to next 3 lines of N + jz L(7_60) // to next 3 lines of N testq $8, M - jz .L7_21pre + jz L(7_21pre) ALIGN_4 /**************************************************************************/ -.L7_20_1: +L(7_20_1): leaq BUFFER2, BO // first buffer to BO addq $12 * SIZE, BO @@ -2242,11 +2242,11 @@ movq K, %rax sarq $3, %rax - je .L7_20_6 + je L(7_20_6) ALIGN_4 -.L7_20_2: +L(7_20_2): KERNEL8x3_SUBN KERNEL8x3_SUBN @@ -2259,26 +2259,26 @@ KERNEL8x3_SUBN dec %rax - jne .L7_20_2 + jne L(7_20_2) ALIGN_4 -.L7_20_6: +L(7_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_20_9 + je L(7_20_9) ALIGN_4 -.L7_20_7: +L(7_20_7): KERNEL8x3_SUBN dec %rax - jne .L7_20_7 + jne L(7_20_7) ALIGN_4 -.L7_20_9: +L(7_20_9): SAVE8x3 @@ -2289,13 +2289,13 @@ /**************************************************************************/ -.L7_21pre: +L(7_21pre): testq $4, M - jz .L7_30 + jz L(7_30) ALIGN_4 -.L7_21: +L(7_21): leaq BUFFER2, BO // second buffer to BO addq $12 * SIZE, BO @@ -2304,11 +2304,11 @@ movq K, %rax sarq $3, %rax - je .L7_26 + je L(7_26) ALIGN_4 -.L7_22: +L(7_22): KERNEL4x3_SUBN KERNEL4x3_SUBN @@ -2321,27 +2321,27 @@ KERNEL4x3_SUBN dec %rax - jne .L7_22 + jne L(7_22) ALIGN_4 -.L7_26: +L(7_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_29 + je L(7_29) ALIGN_4 -.L7_27: +L(7_27): KERNEL4x3_SUBN dec %rax - jne .L7_27 + jne L(7_27) ALIGN_4 -.L7_29: +L(7_29): SAVE4x3 @@ -2349,13 +2349,13 @@ ALIGN_4 -.L7_30: +L(7_30): testq $2, M - jz .L7_40 + jz L(7_40) ALIGN_4 -.L7_31: +L(7_31): leaq BUFFER2, BO // second buffer to BO addq $12 * SIZE, BO @@ -2364,11 +2364,11 @@ movq K, %rax sarq $3, %rax - je .L7_36 + je L(7_36) ALIGN_4 -.L7_32: +L(7_32): KERNEL2x3_SUBN KERNEL2x3_SUBN @@ -2381,40 +2381,40 @@ KERNEL2x3_SUBN dec %rax - jne .L7_32 + jne L(7_32) ALIGN_4 -.L7_36: +L(7_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_39 + je L(7_39) ALIGN_4 -.L7_37: +L(7_37): KERNEL2x3_SUBN dec %rax - jne .L7_37 + jne L(7_37) ALIGN_4 -.L7_39: +L(7_39): SAVE2x3 addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L7_40: +L(7_40): testq $1, M - jz .L7_60 // to next 3 lines of N + jz L(7_60) // to next 3 lines of N ALIGN_4 -.L7_41: +L(7_41): leaq BUFFER2, BO // second buffer to BO addq $12 * SIZE, BO @@ -2423,11 +2423,11 @@ movq K, %rax sarq $3, %rax - je .L7_46 + je L(7_46) ALIGN_4 -.L7_42: +L(7_42): KERNEL1x3_SUBN KERNEL1x3_SUBN KERNEL1x3_SUBN @@ -2439,27 +2439,27 @@ KERNEL1x3_SUBN dec %rax - jne .L7_42 + jne L(7_42) ALIGN_4 -.L7_46: +L(7_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_49 + je L(7_49) ALIGN_4 -.L7_47: +L(7_47): KERNEL1x3_SUBN dec %rax - jne .L7_47 + jne L(7_47) ALIGN_4 -.L7_49: +L(7_49): SAVE1x3 @@ -2468,15 +2468,15 @@ -.L7_60: +L(7_60): decq J // j -- - jg .L6_01 + jg L(6_01) -.L2_0: +L(2_0): cmpq $0, Nmod6 // N % 6 == 0 - je .L999 + je L(999) /************************************************************************************************ * Loop for Nmod6 / 2 > 0 @@ -2484,19 +2484,19 @@ movq Nmod6, J sarq $1, J // j = j / 2 - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L2_01b + jz L(2_01b) ALIGN_4 -.L2_01a: +L(2_01a): prefetcht0 512(BO1) prefetchw 512(BO) @@ -2513,30 +2513,30 @@ addq $8*SIZE,BO1 addq $8*SIZE,BO decq %rax - jnz .L2_01a + jnz L(2_01a) -.L2_01b: +L(2_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L2_02d + jz L(2_02d) ALIGN_4 -.L2_02c: +L(2_02c): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02c + jnz L(2_02c) -.L2_02d: +L(2_02d): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -2546,11 +2546,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2559,7 +2559,7 @@ movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2570,7 +2570,7 @@ negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI,8) KERNEL16x2_1 @@ -2584,7 +2584,7 @@ KERNEL16x2_3 KERNEL16x2_4 - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI,8) KERNEL16x2_1 @@ -2598,16 +2598,16 @@ KERNEL16x2_3 KERNEL16x2_4 - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2619,39 +2619,39 @@ negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE16x2 addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 3 lines of N + jz L(2_60) // to next 3 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2660,7 +2660,7 @@ movq K, %rax andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2671,7 +2671,7 @@ negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1 @@ -2685,7 +2685,7 @@ KERNEL8x2_3 KERNEL8x2_4 - je .L2_20_6 + je L(2_20_6) prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1 @@ -2699,16 +2699,16 @@ KERNEL8x2_3 KERNEL8x2_4 - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2720,15 +2720,15 @@ negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): SAVE8x2 @@ -2739,13 +2739,13 @@ /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2754,7 +2754,7 @@ movq K, %rax andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -2765,7 +2765,7 @@ negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1 @@ -2779,7 +2779,7 @@ KERNEL4x2_3 KERNEL4x2_4 - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1 @@ -2793,16 +2793,16 @@ KERNEL4x2_3 KERNEL4x2_4 - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2814,15 +2814,15 @@ negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): SAVE4x2 @@ -2830,13 +2830,13 @@ ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2845,7 +2845,7 @@ movq K, %rax andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2856,7 +2856,7 @@ negq %rax ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_1 KERNEL2x2_2 @@ -2868,7 +2868,7 @@ KERNEL2x2_3 KERNEL2x2_4 - je .L2_36 + je L(2_36) KERNEL2x2_1 KERNEL2x2_2 @@ -2880,16 +2880,16 @@ KERNEL2x2_3 KERNEL2x2_4 - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2901,28 +2901,28 @@ negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): SAVE2x2 addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2931,7 +2931,7 @@ movq K, %rax andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2941,7 +2941,7 @@ negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_1 KERNEL1x2_2 @@ -2953,7 +2953,7 @@ KERNEL1x2_3 KERNEL1x2_4 - je .L2_46 + je L(2_46) KERNEL1x2_1 KERNEL1x2_2 @@ -2965,16 +2965,16 @@ KERNEL1x2_3 KERNEL1x2_4 - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2985,29 +2985,29 @@ negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): SAVE1x2 addq $1 * SIZE, CO1 # coffset += 1 ALIGN_4 -.L2_60: +L(2_60): decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -3015,30 +3015,30 @@ movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -3048,11 +3048,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3061,7 +3061,7 @@ movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -3071,7 +3071,7 @@ negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI,8) KERNEL16x1_1 @@ -3084,7 +3084,7 @@ KERNEL16x1_3 KERNEL16x1_4 - je .L1_16 + je L(1_16) prefetcht0 B_PR1(BO,BI,8) KERNEL16x1_1 @@ -3097,16 +3097,16 @@ KERNEL16x1_3 KERNEL16x1_4 - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -3117,39 +3117,39 @@ negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE16x1 addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3158,7 +3158,7 @@ movq K, %rax andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -3168,7 +3168,7 @@ negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1 @@ -3181,7 +3181,7 @@ KERNEL8x1_3 KERNEL8x1_4 - je .L1_20_6 + je L(1_20_6) prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1 @@ -3194,16 +3194,16 @@ KERNEL8x1_3 KERNEL8x1_4 - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -3214,15 +3214,15 @@ negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): SAVE8x1 @@ -3233,13 +3233,13 @@ /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3248,7 +3248,7 @@ movq K, %rax andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -3258,7 +3258,7 @@ negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1 @@ -3271,7 +3271,7 @@ KERNEL4x1_3 KERNEL4x1_4 - je .L1_26 + je L(1_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1 @@ -3284,16 +3284,16 @@ KERNEL4x1_3 KERNEL4x1_4 - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -3304,15 +3304,15 @@ negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): SAVE4x1 @@ -3320,13 +3320,13 @@ ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3335,7 +3335,7 @@ movq K, %rax andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -3345,7 +3345,7 @@ negq %rax ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_1 KERNEL2x1_2 @@ -3357,7 +3357,7 @@ KERNEL2x1_3 KERNEL2x1_4 - je .L1_36 + je L(1_36) KERNEL2x1_1 KERNEL2x1_2 @@ -3369,16 +3369,16 @@ KERNEL2x1_3 KERNEL2x1_4 - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -3389,28 +3389,28 @@ negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): SAVE2x1 addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3419,7 +3419,7 @@ movq K, %rax andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -3428,7 +3428,7 @@ negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_1 KERNEL1x1_2 @@ -3440,7 +3440,7 @@ KERNEL1x1_3 KERNEL1x1_4 - je .L1_46 + je L(1_46) KERNEL1x1_1 KERNEL1x1_2 @@ -3452,16 +3452,16 @@ KERNEL1x1_3 KERNEL1x1_4 - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -3471,15 +3471,15 @@ negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): SAVE1x1 @@ -3487,7 +3487,7 @@ ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp @@ -3577,13 +3577,13 @@ STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -3612,19 +3612,19 @@ movq Ndiv6, J cmpq $0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L2_01b + jz L(2_01b) ALIGN_4 -.L2_01a: +L(2_01a): prefetcht0 512(BO1) prefetchw 512(BO) @@ -3641,30 +3641,30 @@ addq $8*SIZE,BO1 addq $8*SIZE,BO decq %rax - jnz .L2_01a + jnz L(2_01a) -.L2_01b: +L(2_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L2_02d + jz L(2_02d) ALIGN_4 -.L2_02c: +L(2_02c): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02c + jnz L(2_02c) -.L2_02d: +L(2_02d): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -3678,11 +3678,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3719,7 +3719,7 @@ #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3730,7 +3730,7 @@ negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI,8) KERNEL16x2_1 @@ -3744,7 +3744,7 @@ KERNEL16x2_3 KERNEL16x2_4 - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI,8) KERNEL16x2_1 @@ -3758,12 +3758,12 @@ KERNEL16x2_3 KERNEL16x2_4 - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3771,7 +3771,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3783,15 +3783,15 @@ negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE16x2 @@ -3813,25 +3813,25 @@ addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 3 lines of N + jz L(2_60) // to next 3 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3869,7 +3869,7 @@ andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3880,7 +3880,7 @@ negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1 @@ -3894,7 +3894,7 @@ KERNEL8x2_3 KERNEL8x2_4 - je .L2_20_6 + je L(2_20_6) prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1 @@ -3908,12 +3908,12 @@ KERNEL8x2_3 KERNEL8x2_4 - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -3921,7 +3921,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3933,15 +3933,15 @@ negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): SAVE8x2 @@ -3968,13 +3968,13 @@ /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4012,7 +4012,7 @@ andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -4023,7 +4023,7 @@ negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1 @@ -4037,7 +4037,7 @@ KERNEL4x2_3 KERNEL4x2_4 - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1 @@ -4051,12 +4051,12 @@ KERNEL4x2_3 KERNEL4x2_4 - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4064,7 +4064,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4076,15 +4076,15 @@ negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): SAVE4x2 @@ -4108,13 +4108,13 @@ ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4152,7 +4152,7 @@ andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4163,7 +4163,7 @@ negq %rax ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_1 KERNEL2x2_2 @@ -4175,7 +4175,7 @@ KERNEL2x2_3 KERNEL2x2_4 - je .L2_36 + je L(2_36) KERNEL2x2_1 KERNEL2x2_2 @@ -4187,12 +4187,12 @@ KERNEL2x2_3 KERNEL2x2_4 - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -4200,7 +4200,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4212,15 +4212,15 @@ negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): SAVE2x2 @@ -4243,13 +4243,13 @@ addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4285,7 +4285,7 @@ #endif andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4295,7 +4295,7 @@ negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_1 KERNEL1x2_2 @@ -4307,7 +4307,7 @@ KERNEL1x2_3 KERNEL1x2_4 - je .L2_46 + je L(2_46) KERNEL1x2_1 KERNEL1x2_2 @@ -4319,12 +4319,12 @@ KERNEL1x2_3 KERNEL1x2_4 - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -4332,7 +4332,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4343,15 +4343,15 @@ negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): SAVE1x2 @@ -4377,17 +4377,17 @@ -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -4395,30 +4395,30 @@ movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -4432,11 +4432,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4472,7 +4472,7 @@ #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -4482,7 +4482,7 @@ negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI,8) KERNEL16x1_1 @@ -4495,7 +4495,7 @@ KERNEL16x1_3 KERNEL16x1_4 - je .L1_16 + je L(1_16) prefetcht0 B_PR1(BO,BI,8) KERNEL16x1_1 @@ -4508,12 +4508,12 @@ KERNEL16x1_3 KERNEL16x1_4 - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -4521,7 +4521,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -4532,15 +4532,15 @@ negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE16x1 @@ -4561,25 +4561,25 @@ addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4616,7 +4616,7 @@ andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -4626,7 +4626,7 @@ negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1 @@ -4639,7 +4639,7 @@ KERNEL8x1_3 KERNEL8x1_4 - je .L1_20_6 + je L(1_20_6) prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1 @@ -4652,12 +4652,12 @@ KERNEL8x1_3 KERNEL8x1_4 - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -4665,7 +4665,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -4676,15 +4676,15 @@ negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): SAVE8x1 @@ -4710,13 +4710,13 @@ /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4753,7 +4753,7 @@ andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -4763,7 +4763,7 @@ negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1 @@ -4776,7 +4776,7 @@ KERNEL4x1_3 KERNEL4x1_4 - je .L1_26 + je L(1_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1 @@ -4789,12 +4789,12 @@ KERNEL4x1_3 KERNEL4x1_4 - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4802,7 +4802,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -4813,15 +4813,15 @@ negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): SAVE4x1 @@ -4844,13 +4844,13 @@ ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4887,7 +4887,7 @@ andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -4897,7 +4897,7 @@ negq %rax ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_1 KERNEL2x1_2 @@ -4909,7 +4909,7 @@ KERNEL2x1_3 KERNEL2x1_4 - je .L1_36 + je L(1_36) KERNEL2x1_1 KERNEL2x1_2 @@ -4921,12 +4921,12 @@ KERNEL2x1_3 KERNEL2x1_4 - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -4934,7 +4934,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -4945,15 +4945,15 @@ negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): SAVE2x1 @@ -4975,13 +4975,13 @@ addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5016,7 +5016,7 @@ #endif andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -5025,7 +5025,7 @@ negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_1 KERNEL1x1_2 @@ -5037,7 +5037,7 @@ KERNEL1x1_3 KERNEL1x1_4 - je .L1_46 + je L(1_46) KERNEL1x1_1 KERNEL1x1_2 @@ -5049,12 +5049,12 @@ KERNEL1x1_3 KERNEL1x1_4 - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -5062,7 +5062,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -5072,15 +5072,15 @@ negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): SAVE1x1 @@ -5102,7 +5102,7 @@ ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/dgemm_kernel_4x4_haswell.S b/kernel/x86_64/dgemm_kernel_4x4_haswell.S index 29501df8e3..5516742364 100644 --- a/kernel/x86_64/dgemm_kernel_4x4_haswell.S +++ b/kernel/x86_64/dgemm_kernel_4x4_haswell.S @@ -1295,13 +1295,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $ 0, OLD_M - je .L999 + je L(999) cmpq $ 0, OLD_N - je .L999 + je L(999) cmpq $ 0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -1321,10 +1321,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv12, J cmpq $ 0, J - je .L4_0 + je L(4_0) ALIGN_4 -.L12_01: +L(12_01): // copy to sub buffer movq K, %rax salq $2,%rax // K * 4 ; read 2 values @@ -1336,10 +1336,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $1 , %rax // K / 2 - jz .L12_01a_2 + jz L(12_01a_2) ALIGN_4 -.L12_01a_1: +L(12_01a_1): prefetcht0 512(BO1) prefetcht0 512(BO2) @@ -1368,19 +1368,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 24 *SIZE ,BO decq %rax - jnz .L12_01a_1 + jnz L(12_01a_1) -.L12_01a_2: +L(12_01a_2): movq K, %rax andq $1, %rax // K % 2 - jz .L12_03c + jz L(12_03c) ALIGN_4 -.L12_02b: +L(12_02b): vmovups 0 * SIZE(BO1), %ymm1 vmovups 0 * SIZE(BO2), %ymm2 @@ -1393,13 +1393,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4*SIZE,BO3 addq $ 12*SIZE,BO decq %rax - jnz .L12_02b + jnz L(12_02b) -.L12_03c: +L(12_03c): movq BO3, B // next offset of B -.L12_10: +L(12_10): movq C, CO1 leaq (C, LDC, 8), C leaq (C, LDC, 4), C // c += 12 * ldc @@ -1410,11 +1410,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L12_20 + je L(12_20) ALIGN_4 -.L12_11: +L(12_11): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -1423,7 +1423,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, %rax // K / 8 cmpq $2, %rax - jl .L12_13 + jl L(12_13) KERNEL4x12_I @@ -1437,10 +1437,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M2 subq $2, %rax - je .L12_12a + je L(12_12a) ALIGN_5 -.L12_12: +L(12_12): KERNEL4x12_M1 KERNEL4x12_M2 @@ -1453,9 +1453,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M2 dec %rax - jne .L12_12 + jne L(12_12) -.L12_12a: +L(12_12a): KERNEL4x12_M1 KERNEL4x12_M2 @@ -1467,13 +1467,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M1 KERNEL4x12_E - jmp .L12_16 + jmp L(12_16) -.L12_13: +L(12_13): test $1, %rax - jz .L12_14 + jz L(12_14) KERNEL4x12_I KERNEL4x12_M2 @@ -1485,56 +1485,56 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M1 KERNEL4x12_E - jmp .L12_16 + jmp L(12_16) -.L12_14: +L(12_14): INIT4x12 -.L12_16: +L(12_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L12_19 + je L(12_19) ALIGN_4 -.L12_17: +L(12_17): KERNEL4x12_SUB dec %rax - jne .L12_17 + jne L(12_17) ALIGN_4 -.L12_19: +L(12_19): SAVE4x12 decq I # i -- - jne .L12_11 + jne L(12_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L12_20: +L(12_20): // Test rest of M testq $3, M - jz .L12_100 // to next 16 lines of N + jz L(12_100) // to next 16 lines of N -.L12_30: +L(12_30): testq $2, M - jz .L12_40 + jz L(12_40) ALIGN_4 -.L12_31: +L(12_31): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -1543,10 +1543,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax - je .L12_36 + je L(12_36) ALIGN_4 -.L12_32: +L(12_32): KERNEL2x12_SUB KERNEL2x12_SUB @@ -1559,39 +1559,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x12_SUB dec %rax - jne .L12_32 + jne L(12_32) ALIGN_4 -.L12_36: +L(12_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L12_39 + je L(12_39) ALIGN_4 -.L12_37: +L(12_37): KERNEL2x12_SUB dec %rax - jne .L12_37 + jne L(12_37) ALIGN_4 -.L12_39: +L(12_39): SAVE2x12 ALIGN_4 -.L12_40: +L(12_40): testq $1, M - jz .L12_100 // to next 3 lines of N + jz L(12_100) // to next 3 lines of N ALIGN_4 -.L12_41: +L(12_41): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -1600,11 +1600,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3,%rax - je .L12_46 + je L(12_46) ALIGN_4 -.L12_42: +L(12_42): KERNEL1x12_SUB KERNEL1x12_SUB @@ -1618,48 +1618,48 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. dec %rax - jne .L12_42 + jne L(12_42) ALIGN_4 -.L12_46: +L(12_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L12_49 + je L(12_49) ALIGN_4 -.L12_47: +L(12_47): KERNEL1x12_SUB dec %rax - jne .L12_47 + jne L(12_47) ALIGN_4 -.L12_49: +L(12_49): SAVE1x12 ALIGN_4 -.L12_100: +L(12_100): decq J // j -- - jg .L12_01 + jg L(12_01) -.L4_0: +L(4_0): cmpq $ 0, Nmod12 // N % 12 == 0 - je .L999 + je L(999) movq Nmod12, J sarq $2, J // j = j / 4 - je .L2_0 + je L(2_0) -.L4_10: +L(4_10): movq C, CO1 leaq (C, LDC, 4), C // c += 4 * ldc @@ -1669,11 +1669,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L4_20 + je L(4_20) ALIGN_4 -.L4_11: +L(4_11): movq B, BO addq $12 * SIZE, BO @@ -1681,7 +1681,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, %rax // K / 8 cmpq $2, %rax - jl .L4_13 + jl L(4_13) KERNEL4x4_I @@ -1695,11 +1695,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 subq $2, %rax - je .L4_12a + je L(4_12a) ALIGN_5 -.L4_12: +L(4_12): KERNEL4x4_M1 KERNEL4x4_M2 @@ -1712,9 +1712,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 dec %rax - jne .L4_12 + jne L(4_12) -.L4_12a: +L(4_12a): KERNEL4x4_M1 KERNEL4x4_M2 @@ -1726,13 +1726,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - jmp .L4_16 + jmp L(4_16) -.L4_13: +L(4_13): test $1, %rax - jz .L4_14 + jz L(4_14) KERNEL4x4_I KERNEL4x4_M2 @@ -1744,56 +1744,56 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - jmp .L4_16 + jmp L(4_16) -.L4_14: +L(4_14): INIT4x4 -.L4_16: +L(4_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L4_19 + je L(4_19) ALIGN_4 -.L4_17: +L(4_17): KERNEL4x4_SUB dec %rax - jne .L4_17 + jne L(4_17) ALIGN_4 -.L4_19: +L(4_19): SAVE4x4 decq I # i -- - jg .L4_11 + jg L(4_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L4_20: +L(4_20): // Test rest of M testq $3, M - jz .L4_100 // to next 16 lines of N + jz L(4_100) // to next 16 lines of N -.L4_30: +L(4_30): testq $2, M - jz .L4_40 + jz L(4_40) ALIGN_4 -.L4_31: +L(4_31): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -1802,10 +1802,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax - je .L4_36 + je L(4_36) ALIGN_4 -.L4_32: +L(4_32): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1818,36 +1818,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB dec %rax - jne .L4_32 + jne L(4_32) ALIGN_4 -.L4_36: +L(4_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L4_39 + je L(4_39) ALIGN_4 -.L4_37: +L(4_37): KERNEL2x4_SUB dec %rax - jne .L4_37 + jne L(4_37) -.L4_39: +L(4_39): SAVE2x4 -.L4_40: +L(4_40): testq $1, M - jz .L4_100 // to next 3 lines of N + jz L(4_100) // to next 3 lines of N ALIGN_4 -.L4_41: +L(4_41): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -1856,11 +1856,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3,%rax - je .L4_46 + je L(4_46) ALIGN_4 -.L4_42: +L(4_42): KERNEL1x4_SUB KERNEL1x4_SUB @@ -1873,52 +1873,52 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB dec %rax - jne .L4_42 + jne L(4_42) ALIGN_4 -.L4_46: +L(4_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L4_49 + je L(4_49) ALIGN_4 -.L4_47: +L(4_47): KERNEL1x4_SUB dec %rax - jne .L4_47 + jne L(4_47) ALIGN_4 -.L4_49: +L(4_49): SAVE1x4 ALIGN_4 -.L4_100: +L(4_100): movq K, %rax salq $2, %rax // * 4 leaq (B , %rax, SIZE), B decq J // j -- - jg .L4_10 + jg L(4_10) /***************************************************************************************************************/ -.L2_0: +L(2_0): movq Nmod12, J testq $2, J - je .L1_0 + je L(1_0) -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -1928,11 +1928,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): movq B, BO addq $12 * SIZE, BO @@ -1941,11 +1941,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax // K / 8 - je .L2_16 + je L(2_16) ALIGN_5 -.L2_12: +L(2_12): KERNEL4x2_SUB KERNEL4x2_SUB @@ -1958,51 +1958,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB dec %rax - jne .L2_12 + jne L(2_12) -.L2_16: +L(2_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) ALIGN_4 -.L2_17: +L(2_17): KERNEL4x2_SUB dec %rax - jne .L2_17 + jne L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE4x2 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $3, M - jz .L2_100 // to next 16 lines of N + jz L(2_100) // to next 16 lines of N -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -2011,10 +2011,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax - je .L2_36 + je L(2_36) ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_SUB KERNEL2x2_SUB @@ -2027,33 +2027,33 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB dec %rax - jne .L2_32 + jne L(2_32) -.L2_36: +L(2_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB dec %rax - jne .L2_37 + jne L(2_37) -.L2_39: +L(2_39): SAVE2x2 -.L2_40: +L(2_40): testq $1, M - jz .L2_100 // to next 3 lines of N + jz L(2_100) // to next 3 lines of N -.L2_41: +L(2_41): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -2062,11 +2062,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3,%rax - je .L2_46 + je L(2_46) ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_SUB KERNEL1x2_SUB @@ -2079,28 +2079,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB dec %rax - jne .L2_42 + jne L(2_42) -.L2_46: +L(2_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB dec %rax - jne .L2_47 + jne L(2_47) -.L2_49: +L(2_49): SAVE1x2 -.L2_100: +L(2_100): movq K, %rax salq $1, %rax // * 2 @@ -2108,13 +2108,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /***************************************************************************************************************/ -.L1_0: +L(1_0): movq Nmod12, J testq $1, J - je .L999 + je L(999) -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -2124,11 +2124,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): movq B, BO addq $12 * SIZE, BO @@ -2137,59 +2137,59 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax // K / 8 - je .L1_16 + je L(1_16) ALIGN_5 -.L1_12: +L(1_12): KERNEL4x1 dec %rax - jne .L1_12 + jne L(1_12) -.L1_16: +L(1_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) ALIGN_4 -.L1_17: +L(1_17): KERNEL4x1_SUB dec %rax - jne .L1_17 + jne L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE4x1 decq I # i -- - jg .L1_11 + jg L(1_11) /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $3, M - jz .L1_100 + jz L(1_100) -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -2198,10 +2198,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax - je .L1_36 + je L(1_36) ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_SUB KERNEL2x1_SUB @@ -2215,33 +2215,33 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. dec %rax - jne .L1_32 + jne L(1_32) -.L1_36: +L(1_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB dec %rax - jne .L1_37 + jne L(1_37) -.L1_39: +L(1_39): SAVE2x1 -.L1_40: +L(1_40): testq $1, M - jz .L1_100 // to next 3 lines of N + jz L(1_100) // to next 3 lines of N -.L1_41: +L(1_41): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -2250,11 +2250,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3,%rax - je .L1_46 + je L(1_46) ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_SUB KERNEL1x1_SUB @@ -2267,34 +2267,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB dec %rax - jne .L1_42 + jne L(1_42) -.L1_46: +L(1_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB dec %rax - jne .L1_47 + jne L(1_47) -.L1_49: +L(1_49): SAVE1x1 -.L1_100: +L(1_100): -.L999: +L(999): vzeroupper movq SP, %rsp @@ -2386,13 +2386,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $ 0, OLD_M - je .L999 + je L(999) cmpq $ 0, OLD_N - je .L999 + je L(999) cmpq $ 0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -2421,10 +2421,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv12, J cmpq $ 0, J - je .L2_0 + je L(2_0) ALIGN_4 -.L4_10: +L(4_10): movq C, CO1 leaq (C, LDC, 4), C // c += 4 * ldc @@ -2439,11 +2439,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L4_20 + je L(4_20) ALIGN_4 -.L4_11: +L(4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2478,7 +2478,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, %rax // K / 8 cmpq $2, %rax - jl .L4_13 + jl L(4_13) KERNEL4x4_I @@ -2492,11 +2492,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 subq $2, %rax - je .L4_12a + je L(4_12a) ALIGN_5 -.L4_12: +L(4_12): KERNEL4x4_M1 KERNEL4x4_M2 @@ -2509,9 +2509,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 dec %rax - jne .L4_12 + jne L(4_12) -.L4_12a: +L(4_12a): KERNEL4x4_M1 KERNEL4x4_M2 @@ -2523,13 +2523,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - jmp .L4_16 + jmp L(4_16) -.L4_13: +L(4_13): test $1, %rax - jz .L4_14 + jz L(4_14) KERNEL4x4_I KERNEL4x4_M2 @@ -2541,32 +2541,32 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - jmp .L4_16 + jmp L(4_16) -.L4_14: +L(4_14): INIT4x4 -.L4_16: +L(4_16): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L4_19 + je L(4_19) ALIGN_4 -.L4_17: +L(4_17): KERNEL4x4_SUB dec %rax - jne .L4_17 + jne L(4_17) ALIGN_4 -.L4_19: +L(4_19): SAVE4x4 @@ -2584,26 +2584,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif decq I # i -- - jg .L4_11 + jg L(4_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L4_20: +L(4_20): // Test rest of M testq $3, M - jz .L4_100 // to next 16 lines of N + jz L(4_100) // to next 16 lines of N -.L4_30: +L(4_30): testq $2, M - jz .L4_40 + jz L(4_40) ALIGN_4 -.L4_31: +L(4_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2639,10 +2639,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT2x4 sarq $3, %rax - je .L4_36 + je L(4_36) ALIGN_4 -.L4_32: +L(4_32): KERNEL2x4_SUB KERNEL2x4_SUB @@ -2655,26 +2655,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB dec %rax - jne .L4_32 + jne L(4_32) ALIGN_4 -.L4_36: +L(4_36): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L4_39 + je L(4_39) ALIGN_4 -.L4_37: +L(4_37): KERNEL2x4_SUB dec %rax - jne .L4_37 + jne L(4_37) -.L4_39: +L(4_39): SAVE2x4 @@ -2692,13 +2692,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.L4_40: +L(4_40): testq $1, M - jz .L4_100 // to next 3 lines of N + jz L(4_100) // to next 3 lines of N ALIGN_4 -.L4_41: +L(4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2734,11 +2734,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT1x4 sarq $3,%rax - je .L4_46 + je L(4_46) ALIGN_4 -.L4_42: +L(4_42): KERNEL1x4_SUB KERNEL1x4_SUB @@ -2751,27 +2751,27 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB dec %rax - jne .L4_42 + jne L(4_42) ALIGN_4 -.L4_46: +L(4_46): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L4_49 + je L(4_49) ALIGN_4 -.L4_47: +L(4_47): KERNEL1x4_SUB dec %rax - jne .L4_47 + jne L(4_47) ALIGN_4 -.L4_49: +L(4_49): SAVE1x4 @@ -2788,7 +2788,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $1, KK // number of values in A #endif -.L4_100: +L(4_100): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK // number of values in B @@ -2799,20 +2799,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. salq $2, %rax // * 4 leaq (B , %rax, SIZE), B decq J // j -- - jg .L4_10 + jg L(4_10) /***************************************************************************************************************/ -.L2_0: +L(2_0): movq Nmod12, J testq $2, J - je .L1_0 + je L(1_0) -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -2828,11 +2828,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2869,11 +2869,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, %rax // K / 8 - je .L2_16 + je L(2_16) ALIGN_5 -.L2_12: +L(2_12): KERNEL4x2_SUB KERNEL4x2_SUB @@ -2886,27 +2886,27 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB dec %rax - jne .L2_12 + jne L(2_12) -.L2_16: +L(2_16): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) ALIGN_4 -.L2_17: +L(2_17): KERNEL4x2_SUB dec %rax - jne .L2_17 + jne L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE4x2 @@ -2925,26 +2925,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $3, M - jz .L2_100 // to next 16 lines of N + jz L(2_100) // to next 16 lines of N -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2980,10 +2980,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT2x2 sarq $3, %rax - je .L2_36 + je L(2_36) ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_SUB KERNEL2x2_SUB @@ -2996,25 +2996,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB dec %rax - jne .L2_32 + jne L(2_32) -.L2_36: +L(2_36): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB dec %rax - jne .L2_37 + jne L(2_37) -.L2_39: +L(2_39): SAVE2x2 @@ -3032,11 +3032,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.L2_40: +L(2_40): testq $1, M - jz .L2_100 // to next 3 lines of N + jz L(2_100) // to next 3 lines of N -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3072,11 +3072,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT1x2 sarq $3,%rax - je .L2_46 + je L(2_46) ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_SUB KERNEL1x2_SUB @@ -3089,24 +3089,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB dec %rax - jne .L2_42 + jne L(2_42) -.L2_46: +L(2_46): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB dec %rax - jne .L2_47 + jne L(2_47) -.L2_49: +L(2_49): SAVE1x2 @@ -3124,7 +3124,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.L2_100: +L(2_100): #if defined(TRMMKERNEL) && !defined(LEFT) @@ -3137,13 +3137,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /***************************************************************************************************************/ -.L1_0: +L(1_0): movq Nmod12, J testq $1, J - je .L999 + je L(999) -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -3157,11 +3157,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3197,36 +3197,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT4x1 sarq $3, %rax // K / 8 - je .L1_16 + je L(1_16) ALIGN_5 -.L1_12: +L(1_12): KERNEL4x1 dec %rax - jne .L1_12 + jne L(1_12) -.L1_16: +L(1_16): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) ALIGN_4 -.L1_17: +L(1_17): KERNEL4x1_SUB dec %rax - jne .L1_17 + jne L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE4x1 @@ -3245,25 +3245,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. decq I # i -- - jg .L1_11 + jg L(1_11) /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $3, M - jz .L1_100 + jz L(1_100) -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3299,10 +3299,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT2x1 sarq $3, %rax - je .L1_36 + je L(1_36) ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_SUB KERNEL2x1_SUB @@ -3316,24 +3316,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. dec %rax - jne .L1_32 + jne L(1_32) -.L1_36: +L(1_36): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB dec %rax - jne .L1_37 + jne L(1_37) -.L1_39: +L(1_39): SAVE2x1 @@ -3351,12 +3351,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.L1_40: +L(1_40): testq $1, M - jz .L1_100 // to next 3 lines of N + jz L(1_100) // to next 3 lines of N -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3392,11 +3392,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT1x1 sarq $3,%rax - je .L1_46 + je L(1_46) ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_SUB KERNEL1x1_SUB @@ -3409,25 +3409,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB dec %rax - jne .L1_42 + jne L(1_42) -.L1_46: +L(1_46): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB dec %rax - jne .L1_47 + jne L(1_47) -.L1_49: +L(1_49): SAVE1x1 @@ -3446,7 +3446,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L1_100: +L(1_100): #if defined(TRMMKERNEL) && !defined(LEFT) @@ -3455,7 +3455,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L999: +L(999): vzeroupper diff --git a/kernel/x86_64/dgemm_kernel_4x8_haswell.S b/kernel/x86_64/dgemm_kernel_4x8_haswell.S index adaa28bbc4..7d36e2b00d 100644 --- a/kernel/x86_64/dgemm_kernel_4x8_haswell.S +++ b/kernel/x86_64/dgemm_kernel_4x8_haswell.S @@ -1999,13 +1999,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $ 0, OLD_M - je .L999 + je L(999) cmpq $ 0, OLD_N - je .L999 + je L(999) cmpq $ 0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -2025,10 +2025,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv12, J cmpq $ 0, J - je .L8_0 + je L(8_0) ALIGN_4 -.L12_01: +L(12_01): // copy to sub buffer movq K, %rax salq $3,%rax // K * 8 ; read 8 values from BO1 @@ -2041,7 +2041,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L12_02b: +L(12_02b): vmovups 0 * SIZE(BO1), %ymm1 vmovups 4 * SIZE(BO1), %ymm2 @@ -2053,12 +2053,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 8*SIZE,BO2 addq $ 12*SIZE,BO decq %rax - jnz .L12_02b + jnz L(12_02b) -.L12_03c: +L(12_03c): -.L12_10: +L(12_10): movq C, CO1 leaq (C, LDC, 8), C leaq (C, LDC, 4), C // c += 12 * ldc @@ -2068,11 +2068,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L12_20 + je L(12_20) ALIGN_4 -.L12_11: +L(12_11): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -2081,7 +2081,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, %rax // K / 8 cmpq $2, %rax - jl .L12_13 + jl L(12_13) KERNEL4x12_I @@ -2095,10 +2095,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M2 subq $2, %rax - je .L12_12a + je L(12_12a) ALIGN_5 -.L12_12: +L(12_12): KERNEL4x12_M1 KERNEL4x12_M2 @@ -2111,9 +2111,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M2 dec %rax - jne .L12_12 + jne L(12_12) -.L12_12a: +L(12_12a): prefetcht0 ALPHA PREFETCHT0_C addq LDC,CO1 @@ -2134,13 +2134,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M1 KERNEL4x12_E - jmp .L12_16 + jmp L(12_16) -.L12_13: +L(12_13): test $1, %rax - jz .L12_14 + jz L(12_14) KERNEL4x12_I KERNEL4x12_M2 @@ -2152,32 +2152,32 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M1 KERNEL4x12_E - jmp .L12_16 + jmp L(12_16) -.L12_14: +L(12_14): INIT4x12 -.L12_16: +L(12_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L12_19 + je L(12_19) ALIGN_4 -.L12_17: +L(12_17): KERNEL4x12_SUB dec %rax - jne .L12_17 + jne L(12_17) ALIGN_4 -.L12_19: +L(12_19): SAVE4x12 @@ -2199,7 +2199,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, K decq I # i -- - jne .L12_11 + jne L(12_11) ALIGN_4 /************************************************************************** @@ -2216,20 +2216,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subq I, B -.L12_20: +L(12_20): // Test rest of M testq $3, M - jz .L12_100 // to next 16 lines of N + jz L(12_100) // to next 16 lines of N -.L12_30: +L(12_30): testq $2, M - jz .L12_40 + jz L(12_40) ALIGN_4 -.L12_31: +L(12_31): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -2238,10 +2238,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax - je .L12_36 + je L(12_36) ALIGN_4 -.L12_32: +L(12_32): KERNEL2x12_SUB KERNEL2x12_SUB @@ -2254,39 +2254,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x12_SUB dec %rax - jne .L12_32 + jne L(12_32) ALIGN_4 -.L12_36: +L(12_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L12_39 + je L(12_39) ALIGN_4 -.L12_37: +L(12_37): KERNEL2x12_SUB dec %rax - jne .L12_37 + jne L(12_37) ALIGN_4 -.L12_39: +L(12_39): SAVE2x12 ALIGN_4 -.L12_40: +L(12_40): testq $1, M - jz .L12_100 // to next 3 lines of N + jz L(12_100) // to next 3 lines of N ALIGN_4 -.L12_41: +L(12_41): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -2295,11 +2295,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3,%rax - je .L12_46 + je L(12_46) ALIGN_4 -.L12_42: +L(12_42): KERNEL1x12_SUB KERNEL1x12_SUB @@ -2313,39 +2313,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. dec %rax - jne .L12_42 + jne L(12_42) ALIGN_4 -.L12_46: +L(12_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L12_49 + je L(12_49) ALIGN_4 -.L12_47: +L(12_47): KERNEL1x12_SUB dec %rax - jne .L12_47 + jne L(12_47) ALIGN_4 -.L12_49: +L(12_49): SAVE1x12 ALIGN_4 -.L12_100: +L(12_100): /**************************************************************************************************/ -.L13_01: +L(13_01): // copy to sub buffer movq K, %rax salq $3,%rax // K * 8 ; read 8 values @@ -2360,7 +2360,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L13_02b: +L(13_02b): vmovups 4 * SIZE(BO2), %ymm1 vmovups 0 * SIZE(BO3), %ymm2 @@ -2372,11 +2372,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 8*SIZE,BO3 addq $ 12*SIZE,BO decq %rax - jnz .L13_02b + jnz L(13_02b) -.L13_10: +L(13_10): movq C, CO1 leaq (C, LDC, 8), C leaq (C, LDC, 4), C // c += 12 * ldc @@ -2387,11 +2387,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L13_20 + je L(13_20) ALIGN_4 -.L13_11: +L(13_11): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -2400,7 +2400,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, %rax // K / 8 cmpq $2, %rax - jl .L13_13 + jl L(13_13) KERNEL4x12_I @@ -2414,10 +2414,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M2 subq $2, %rax - je .L13_12a + je L(13_12a) ALIGN_5 -.L13_12: +L(13_12): KERNEL4x12_M1 KERNEL4x12_M2 @@ -2430,9 +2430,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M2 dec %rax - jne .L13_12 + jne L(13_12) -.L13_12a: +L(13_12a): prefetcht0 ALPHA PREFETCHT0_C addq LDC,CO1 @@ -2453,12 +2453,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M1 KERNEL4x12_E - jmp .L13_16 + jmp L(13_16) -.L13_13: +L(13_13): test $1, %rax - jz .L13_14 + jz L(13_14) KERNEL4x12_I KERNEL4x12_M2 @@ -2470,32 +2470,32 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M1 KERNEL4x12_E - jmp .L13_16 + jmp L(13_16) -.L13_14: +L(13_14): INIT4x12 -.L13_16: +L(13_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L13_19 + je L(13_19) ALIGN_4 -.L13_17: +L(13_17): KERNEL4x12_SUB dec %rax - jne .L13_17 + jne L(13_17) ALIGN_4 -.L13_19: +L(13_19): SAVE4x12 @@ -2517,7 +2517,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, K decq I # i -- - jne .L13_11 + jne L(13_11) ALIGN_4 /************************************************************************** @@ -2533,20 +2533,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subq I, B -.L13_20: +L(13_20): // Test rest of M testq $3, M - jz .L13_100 // to next 16 lines of N + jz L(13_100) // to next 16 lines of N -.L13_30: +L(13_30): testq $2, M - jz .L13_40 + jz L(13_40) ALIGN_4 -.L13_31: +L(13_31): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -2555,10 +2555,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax - je .L13_36 + je L(13_36) ALIGN_4 -.L13_32: +L(13_32): KERNEL2x12_SUB KERNEL2x12_SUB @@ -2571,39 +2571,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x12_SUB dec %rax - jne .L13_32 + jne L(13_32) ALIGN_4 -.L13_36: +L(13_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L13_39 + je L(13_39) ALIGN_4 -.L13_37: +L(13_37): KERNEL2x12_SUB dec %rax - jne .L13_37 + jne L(13_37) ALIGN_4 -.L13_39: +L(13_39): SAVE2x12 ALIGN_4 -.L13_40: +L(13_40): testq $1, M - jz .L13_100 // to next 3 lines of N + jz L(13_100) // to next 3 lines of N ALIGN_4 -.L13_41: +L(13_41): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -2612,11 +2612,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3,%rax - je .L13_46 + je L(13_46) ALIGN_4 -.L13_42: +L(13_42): KERNEL1x12_SUB KERNEL1x12_SUB @@ -2630,52 +2630,52 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. dec %rax - jne .L13_42 + jne L(13_42) ALIGN_4 -.L13_46: +L(13_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L13_49 + je L(13_49) ALIGN_4 -.L13_47: +L(13_47): KERNEL1x12_SUB dec %rax - jne .L13_47 + jne L(13_47) ALIGN_4 -.L13_49: +L(13_49): SAVE1x12 ALIGN_4 -.L13_100: +L(13_100): decq J // j -- - jg .L12_01 + jg L(12_01) /**************************************************************************************************/ -.L8_0: +L(8_0): cmpq $ 0, Nmod12 // N % 12 == 0 - je .L999 + je L(999) movq Nmod12, J sarq $3, J // j = j / 8 - je .L4_0 + je L(4_0) -.L8_10: +L(8_10): movq C, CO1 leaq (C, LDC, 8), C // c += 4 * ldc @@ -2685,11 +2685,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L8_20 + je L(8_20) ALIGN_4 -.L8_11: +L(8_11): movq B, BO addq $12 * SIZE, BO @@ -2697,7 +2697,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, %rax // K / 8 cmpq $2, %rax - jl .L8_13 + jl L(8_13) KERNEL4x8_I @@ -2711,11 +2711,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x8_M2 subq $2, %rax - je .L8_12a + je L(8_12a) ALIGN_5 -.L8_12: +L(8_12): KERNEL4x8_M1 KERNEL4x8_M2 @@ -2728,9 +2728,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x8_M2 dec %rax - jne .L8_12 + jne L(8_12) -.L8_12a: +L(8_12a): KERNEL4x8_M1 KERNEL4x8_M2 @@ -2742,13 +2742,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x8_M1 KERNEL4x8_E - jmp .L8_16 + jmp L(8_16) -.L8_13: +L(8_13): test $1, %rax - jz .L8_14 + jz L(8_14) KERNEL4x8_I KERNEL4x8_M2 @@ -2760,56 +2760,56 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x8_M1 KERNEL4x8_E - jmp .L8_16 + jmp L(8_16) -.L8_14: +L(8_14): INIT4x8 -.L8_16: +L(8_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L8_19 + je L(8_19) ALIGN_4 -.L8_17: +L(8_17): KERNEL4x8_SUB dec %rax - jne .L8_17 + jne L(8_17) ALIGN_4 -.L8_19: +L(8_19): SAVE4x8 decq I # i -- - jg .L8_11 + jg L(8_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L8_20: +L(8_20): // Test rest of M testq $3, M - jz .L8_100 // to next 16 lines of N + jz L(8_100) // to next 16 lines of N -.L8_30: +L(8_30): testq $2, M - jz .L8_40 + jz L(8_40) ALIGN_4 -.L8_31: +L(8_31): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -2818,10 +2818,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax - je .L8_36 + je L(8_36) ALIGN_4 -.L8_32: +L(8_32): KERNEL2x8_SUB KERNEL2x8_SUB @@ -2834,36 +2834,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x8_SUB dec %rax - jne .L8_32 + jne L(8_32) ALIGN_4 -.L8_36: +L(8_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L8_39 + je L(8_39) ALIGN_4 -.L8_37: +L(8_37): KERNEL2x8_SUB dec %rax - jne .L8_37 + jne L(8_37) -.L8_39: +L(8_39): SAVE2x8 -.L8_40: +L(8_40): testq $1, M - jz .L8_100 // to next 3 lines of N + jz L(8_100) // to next 3 lines of N ALIGN_4 -.L8_41: +L(8_41): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -2872,11 +2872,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3,%rax - je .L8_46 + je L(8_46) ALIGN_4 -.L8_42: +L(8_42): KERNEL1x8_SUB KERNEL1x8_SUB @@ -2889,54 +2889,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x8_SUB dec %rax - jne .L8_42 + jne L(8_42) ALIGN_4 -.L8_46: +L(8_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L8_49 + je L(8_49) ALIGN_4 -.L8_47: +L(8_47): KERNEL1x8_SUB dec %rax - jne .L8_47 + jne L(8_47) ALIGN_4 -.L8_49: +L(8_49): SAVE1x8 ALIGN_4 -.L8_100: +L(8_100): movq K, %rax salq $3, %rax // * 8 leaq (B , %rax, SIZE), B decq J // j -- - jg .L8_10 + jg L(8_10) /**************************************************************************************************/ -.L4_0: +L(4_0): cmpq $ 0, Nmod12 // N % 12 == 0 - je .L999 + je L(999) movq Nmod12, J testq $4, J // j = j / 4 - je .L2_0 + je L(2_0) -.L4_10: +L(4_10): movq C, CO1 leaq (C, LDC, 4), C // c += 4 * ldc @@ -2946,11 +2946,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L4_20 + je L(4_20) ALIGN_4 -.L4_11: +L(4_11): movq B, BO addq $12 * SIZE, BO @@ -2958,7 +2958,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, %rax // K / 8 cmpq $2, %rax - jl .L4_13 + jl L(4_13) KERNEL4x4_I @@ -2972,11 +2972,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 subq $2, %rax - je .L4_12a + je L(4_12a) ALIGN_5 -.L4_12: +L(4_12): KERNEL4x4_M1 KERNEL4x4_M2 @@ -2989,9 +2989,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 dec %rax - jne .L4_12 + jne L(4_12) -.L4_12a: +L(4_12a): KERNEL4x4_M1 KERNEL4x4_M2 @@ -3003,13 +3003,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - jmp .L4_16 + jmp L(4_16) -.L4_13: +L(4_13): test $1, %rax - jz .L4_14 + jz L(4_14) KERNEL4x4_I KERNEL4x4_M2 @@ -3021,57 +3021,57 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - jmp .L4_16 + jmp L(4_16) -.L4_14: +L(4_14): INIT4x4 -.L4_16: +L(4_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L4_19 + je L(4_19) ALIGN_4 -.L4_17: +L(4_17): KERNEL4x4_SUB dec %rax - jne .L4_17 + jne L(4_17) ALIGN_4 -.L4_19: +L(4_19): SAVE4x4 decq I # i -- - jg .L4_11 + jg L(4_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L4_20: +L(4_20): // Test rest of M testq $3, M - jz .L4_100 // to next 16 lines of N + jz L(4_100) // to next 16 lines of N -.L4_30: +L(4_30): testq $2, M - jz .L4_40 + jz L(4_40) ALIGN_4 -.L4_31: +L(4_31): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -3080,10 +3080,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax - je .L4_36 + je L(4_36) ALIGN_4 -.L4_32: +L(4_32): KERNEL2x4_SUB KERNEL2x4_SUB @@ -3096,36 +3096,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB dec %rax - jne .L4_32 + jne L(4_32) ALIGN_4 -.L4_36: +L(4_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L4_39 + je L(4_39) ALIGN_4 -.L4_37: +L(4_37): KERNEL2x4_SUB dec %rax - jne .L4_37 + jne L(4_37) -.L4_39: +L(4_39): SAVE2x4 -.L4_40: +L(4_40): testq $1, M - jz .L4_100 // to next 3 lines of N + jz L(4_100) // to next 3 lines of N ALIGN_4 -.L4_41: +L(4_41): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -3134,11 +3134,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3,%rax - je .L4_46 + je L(4_46) ALIGN_4 -.L4_42: +L(4_42): KERNEL1x4_SUB KERNEL1x4_SUB @@ -3151,33 +3151,33 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB dec %rax - jne .L4_42 + jne L(4_42) ALIGN_4 -.L4_46: +L(4_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L4_49 + je L(4_49) ALIGN_4 -.L4_47: +L(4_47): KERNEL1x4_SUB dec %rax - jne .L4_47 + jne L(4_47) ALIGN_4 -.L4_49: +L(4_49): SAVE1x4 ALIGN_4 -.L4_100: +L(4_100): movq K, %rax salq $2, %rax // * 4 @@ -3188,13 +3188,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /***************************************************************************************************************/ -.L2_0: +L(2_0): movq Nmod12, J testq $2, J - je .L1_0 + je L(1_0) -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -3204,11 +3204,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): movq B, BO addq $12 * SIZE, BO @@ -3217,11 +3217,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax // K / 8 - je .L2_16 + je L(2_16) ALIGN_5 -.L2_12: +L(2_12): KERNEL4x2_SUB KERNEL4x2_SUB @@ -3234,51 +3234,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB dec %rax - jne .L2_12 + jne L(2_12) -.L2_16: +L(2_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) ALIGN_4 -.L2_17: +L(2_17): KERNEL4x2_SUB dec %rax - jne .L2_17 + jne L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE4x2 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $3, M - jz .L2_100 // to next 16 lines of N + jz L(2_100) // to next 16 lines of N -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -3287,10 +3287,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax - je .L2_36 + je L(2_36) ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_SUB KERNEL2x2_SUB @@ -3303,33 +3303,33 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB dec %rax - jne .L2_32 + jne L(2_32) -.L2_36: +L(2_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB dec %rax - jne .L2_37 + jne L(2_37) -.L2_39: +L(2_39): SAVE2x2 -.L2_40: +L(2_40): testq $1, M - jz .L2_100 // to next 3 lines of N + jz L(2_100) // to next 3 lines of N -.L2_41: +L(2_41): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -3338,11 +3338,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3,%rax - je .L2_46 + je L(2_46) ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_SUB KERNEL1x2_SUB @@ -3355,28 +3355,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB dec %rax - jne .L2_42 + jne L(2_42) -.L2_46: +L(2_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB dec %rax - jne .L2_47 + jne L(2_47) -.L2_49: +L(2_49): SAVE1x2 -.L2_100: +L(2_100): movq K, %rax salq $1, %rax // * 2 @@ -3384,13 +3384,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /***************************************************************************************************************/ -.L1_0: +L(1_0): movq Nmod12, J testq $1, J - je .L999 + je L(999) -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -3400,11 +3400,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): movq B, BO addq $12 * SIZE, BO @@ -3413,59 +3413,59 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax // K / 8 - je .L1_16 + je L(1_16) ALIGN_5 -.L1_12: +L(1_12): KERNEL4x1 dec %rax - jne .L1_12 + jne L(1_12) -.L1_16: +L(1_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) ALIGN_4 -.L1_17: +L(1_17): KERNEL4x1_SUB dec %rax - jne .L1_17 + jne L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE4x1 decq I # i -- - jg .L1_11 + jg L(1_11) /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $3, M - jz .L1_100 + jz L(1_100) -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -3474,10 +3474,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax - je .L1_36 + je L(1_36) ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_SUB KERNEL2x1_SUB @@ -3491,33 +3491,33 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. dec %rax - jne .L1_32 + jne L(1_32) -.L1_36: +L(1_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB dec %rax - jne .L1_37 + jne L(1_37) -.L1_39: +L(1_39): SAVE2x1 -.L1_40: +L(1_40): testq $1, M - jz .L1_100 // to next 3 lines of N + jz L(1_100) // to next 3 lines of N -.L1_41: +L(1_41): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -3526,11 +3526,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3,%rax - je .L1_46 + je L(1_46) ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_SUB KERNEL1x1_SUB @@ -3543,34 +3543,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB dec %rax - jne .L1_42 + jne L(1_42) -.L1_46: +L(1_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB dec %rax - jne .L1_47 + jne L(1_47) -.L1_49: +L(1_49): SAVE1x1 -.L1_100: +L(1_100): -.L999: +L(999): vzeroupper movq SP, %rsp @@ -3662,13 +3662,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $ 0, OLD_M - je .L999 + je L(999) cmpq $ 0, OLD_N - je .L999 + je L(999) cmpq $ 0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -3694,13 +3694,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif /*************************************************************************************************/ -.L8_0: +L(8_0): movq Ndiv12, J cmpq $ 0, J - je .L4_0 + je L(4_0) ALIGN_4 -.L8_10: +L(8_10): movq C, CO1 leaq (C, LDC, 8), C // c += 8 * ldc @@ -3715,11 +3715,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L8_20 + je L(8_20) ALIGN_4 -.L8_11: +L(8_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3754,7 +3754,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, %rax // K / 8 cmpq $2, %rax - jl .L8_13 + jl L(8_13) KERNEL4x8_I @@ -3768,11 +3768,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x8_M2 subq $2, %rax - je .L8_12a + je L(8_12a) ALIGN_5 -.L8_12: +L(8_12): KERNEL4x8_M1 KERNEL4x8_M2 @@ -3785,9 +3785,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x8_M2 dec %rax - jne .L8_12 + jne L(8_12) -.L8_12a: +L(8_12a): KERNEL4x8_M1 KERNEL4x8_M2 @@ -3799,13 +3799,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x8_M1 KERNEL4x8_E - jmp .L8_16 + jmp L(8_16) -.L8_13: +L(8_13): test $1, %rax - jz .L8_14 + jz L(8_14) KERNEL4x8_I KERNEL4x8_M2 @@ -3817,32 +3817,32 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x8_M1 KERNEL4x8_E - jmp .L8_16 + jmp L(8_16) -.L8_14: +L(8_14): INIT4x8 -.L8_16: +L(8_16): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L8_19 + je L(8_19) ALIGN_4 -.L8_17: +L(8_17): KERNEL4x8_SUB dec %rax - jne .L8_17 + jne L(8_17) ALIGN_4 -.L8_19: +L(8_19): SAVE4x8 @@ -3860,26 +3860,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif decq I # i -- - jg .L8_11 + jg L(8_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L8_20: +L(8_20): // Test rest of M testq $3, M - jz .L8_100 // to next 16 lines of N + jz L(8_100) // to next 16 lines of N -.L8_30: +L(8_30): testq $2, M - jz .L8_40 + jz L(8_40) ALIGN_4 -.L8_31: +L(8_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3915,10 +3915,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT2x8 sarq $3, %rax - je .L8_36 + je L(8_36) ALIGN_4 -.L8_32: +L(8_32): KERNEL2x8_SUB KERNEL2x8_SUB @@ -3931,26 +3931,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x8_SUB dec %rax - jne .L8_32 + jne L(8_32) ALIGN_4 -.L8_36: +L(8_36): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L8_39 + je L(8_39) ALIGN_4 -.L8_37: +L(8_37): KERNEL2x8_SUB dec %rax - jne .L8_37 + jne L(8_37) -.L8_39: +L(8_39): SAVE2x8 @@ -3968,13 +3968,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.L8_40: +L(8_40): testq $1, M - jz .L8_100 // to next 3 lines of N + jz L(8_100) // to next 3 lines of N ALIGN_4 -.L8_41: +L(8_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4010,11 +4010,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT1x8 sarq $3,%rax - je .L8_46 + je L(8_46) ALIGN_4 -.L8_42: +L(8_42): KERNEL1x8_SUB KERNEL1x8_SUB @@ -4027,27 +4027,27 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x8_SUB dec %rax - jne .L8_42 + jne L(8_42) ALIGN_4 -.L8_46: +L(8_46): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L8_49 + je L(8_49) ALIGN_4 -.L8_47: +L(8_47): KERNEL1x8_SUB dec %rax - jne .L8_47 + jne L(8_47) ALIGN_4 -.L8_49: +L(8_49): SAVE1x8 @@ -4064,7 +4064,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $1, KK // number of values in A #endif -.L8_100: +L(8_100): #if defined(TRMMKERNEL) && !defined(LEFT) addq $8, KK // number of values in B @@ -4072,20 +4072,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. decq J // j -- - jg .L8_10 + jg L(8_10) /*************************************************************************************************/ -.L4_0: +L(4_0): movq Nmod12, J testq $4, J - je .L2_0 + je L(2_0) ALIGN_4 -.L4_10: +L(4_10): movq C, CO1 leaq (C, LDC, 4), C // c += 4 * ldc @@ -4100,11 +4100,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L4_20 + je L(4_20) ALIGN_4 -.L4_11: +L(4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4139,7 +4139,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, %rax // K / 8 cmpq $2, %rax - jl .L4_13 + jl L(4_13) KERNEL4x4_I @@ -4153,11 +4153,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 subq $2, %rax - je .L4_12a + je L(4_12a) ALIGN_5 -.L4_12: +L(4_12): KERNEL4x4_M1 KERNEL4x4_M2 @@ -4170,9 +4170,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 dec %rax - jne .L4_12 + jne L(4_12) -.L4_12a: +L(4_12a): KERNEL4x4_M1 KERNEL4x4_M2 @@ -4184,13 +4184,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - jmp .L4_16 + jmp L(4_16) -.L4_13: +L(4_13): test $1, %rax - jz .L4_14 + jz L(4_14) KERNEL4x4_I KERNEL4x4_M2 @@ -4202,32 +4202,32 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - jmp .L4_16 + jmp L(4_16) -.L4_14: +L(4_14): INIT4x4 -.L4_16: +L(4_16): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L4_19 + je L(4_19) ALIGN_4 -.L4_17: +L(4_17): KERNEL4x4_SUB dec %rax - jne .L4_17 + jne L(4_17) ALIGN_4 -.L4_19: +L(4_19): SAVE4x4 @@ -4245,26 +4245,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif decq I # i -- - jg .L4_11 + jg L(4_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L4_20: +L(4_20): // Test rest of M testq $3, M - jz .L4_100 // to next 16 lines of N + jz L(4_100) // to next 16 lines of N -.L4_30: +L(4_30): testq $2, M - jz .L4_40 + jz L(4_40) ALIGN_4 -.L4_31: +L(4_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4300,10 +4300,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT2x4 sarq $3, %rax - je .L4_36 + je L(4_36) ALIGN_4 -.L4_32: +L(4_32): KERNEL2x4_SUB KERNEL2x4_SUB @@ -4316,26 +4316,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB dec %rax - jne .L4_32 + jne L(4_32) ALIGN_4 -.L4_36: +L(4_36): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L4_39 + je L(4_39) ALIGN_4 -.L4_37: +L(4_37): KERNEL2x4_SUB dec %rax - jne .L4_37 + jne L(4_37) -.L4_39: +L(4_39): SAVE2x4 @@ -4353,13 +4353,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.L4_40: +L(4_40): testq $1, M - jz .L4_100 // to next 3 lines of N + jz L(4_100) // to next 3 lines of N ALIGN_4 -.L4_41: +L(4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4395,11 +4395,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT1x4 sarq $3,%rax - je .L4_46 + je L(4_46) ALIGN_4 -.L4_42: +L(4_42): KERNEL1x4_SUB KERNEL1x4_SUB @@ -4412,27 +4412,27 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB dec %rax - jne .L4_42 + jne L(4_42) ALIGN_4 -.L4_46: +L(4_46): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L4_49 + je L(4_49) ALIGN_4 -.L4_47: +L(4_47): KERNEL1x4_SUB dec %rax - jne .L4_47 + jne L(4_47) ALIGN_4 -.L4_49: +L(4_49): SAVE1x4 @@ -4449,7 +4449,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $1, KK // number of values in A #endif -.L4_100: +L(4_100): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK // number of values in B @@ -4465,13 +4465,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /***************************************************************************************************************/ -.L2_0: +L(2_0): movq Nmod12, J testq $2, J - je .L1_0 + je L(1_0) -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -4487,11 +4487,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4528,11 +4528,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, %rax // K / 8 - je .L2_16 + je L(2_16) ALIGN_5 -.L2_12: +L(2_12): KERNEL4x2_SUB KERNEL4x2_SUB @@ -4545,27 +4545,27 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB dec %rax - jne .L2_12 + jne L(2_12) -.L2_16: +L(2_16): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) ALIGN_4 -.L2_17: +L(2_17): KERNEL4x2_SUB dec %rax - jne .L2_17 + jne L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE4x2 @@ -4584,26 +4584,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $3, M - jz .L2_100 // to next 16 lines of N + jz L(2_100) // to next 16 lines of N -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4639,10 +4639,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT2x2 sarq $3, %rax - je .L2_36 + je L(2_36) ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_SUB KERNEL2x2_SUB @@ -4655,25 +4655,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB dec %rax - jne .L2_32 + jne L(2_32) -.L2_36: +L(2_36): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB dec %rax - jne .L2_37 + jne L(2_37) -.L2_39: +L(2_39): SAVE2x2 @@ -4691,11 +4691,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.L2_40: +L(2_40): testq $1, M - jz .L2_100 // to next 3 lines of N + jz L(2_100) // to next 3 lines of N -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4731,11 +4731,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT1x2 sarq $3,%rax - je .L2_46 + je L(2_46) ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_SUB KERNEL1x2_SUB @@ -4748,24 +4748,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB dec %rax - jne .L2_42 + jne L(2_42) -.L2_46: +L(2_46): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB dec %rax - jne .L2_47 + jne L(2_47) -.L2_49: +L(2_49): SAVE1x2 @@ -4783,7 +4783,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.L2_100: +L(2_100): #if defined(TRMMKERNEL) && !defined(LEFT) @@ -4796,13 +4796,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /***************************************************************************************************************/ -.L1_0: +L(1_0): movq Nmod12, J testq $1, J - je .L999 + je L(999) -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -4816,11 +4816,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4856,36 +4856,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT4x1 sarq $3, %rax // K / 8 - je .L1_16 + je L(1_16) ALIGN_5 -.L1_12: +L(1_12): KERNEL4x1 dec %rax - jne .L1_12 + jne L(1_12) -.L1_16: +L(1_16): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) ALIGN_4 -.L1_17: +L(1_17): KERNEL4x1_SUB dec %rax - jne .L1_17 + jne L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE4x1 @@ -4904,25 +4904,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. decq I # i -- - jg .L1_11 + jg L(1_11) /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $3, M - jz .L1_100 + jz L(1_100) -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4958,10 +4958,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT2x1 sarq $3, %rax - je .L1_36 + je L(1_36) ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_SUB KERNEL2x1_SUB @@ -4975,24 +4975,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. dec %rax - jne .L1_32 + jne L(1_32) -.L1_36: +L(1_36): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB dec %rax - jne .L1_37 + jne L(1_37) -.L1_39: +L(1_39): SAVE2x1 @@ -5010,12 +5010,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.L1_40: +L(1_40): testq $1, M - jz .L1_100 // to next 3 lines of N + jz L(1_100) // to next 3 lines of N -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -5051,11 +5051,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT1x1 sarq $3,%rax - je .L1_46 + je L(1_46) ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_SUB KERNEL1x1_SUB @@ -5068,25 +5068,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB dec %rax - jne .L1_42 + jne L(1_42) -.L1_46: +L(1_46): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB dec %rax - jne .L1_47 + jne L(1_47) -.L1_49: +L(1_49): SAVE1x1 @@ -5105,7 +5105,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L1_100: +L(1_100): #if defined(TRMMKERNEL) && !defined(LEFT) @@ -5114,7 +5114,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L999: +L(999): vzeroupper diff --git a/kernel/x86_64/dgemm_kernel_4x8_sandy.S b/kernel/x86_64/dgemm_kernel_4x8_sandy.S index 926395c493..50e2a6348e 100644 --- a/kernel/x86_64/dgemm_kernel_4x8_sandy.S +++ b/kernel/x86_64/dgemm_kernel_4x8_sandy.S @@ -249,9 +249,9 @@ movq %r11, kk MOVQ bn,j; SARQ $2,j; # Rn = 4 -JLE .L0_loopE; +JLE L(0_loopE); ALIGN_5; -.L0_bodyB:; +L(0_bodyB):; #if defined(TRMMKERNEL) && defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk; @@ -265,9 +265,9 @@ LEAQ (bb, k, 1), prebb; MOVQ ba,ptrba; MOVQ bm,i; SARQ $3,i; # Rm = 8 -JLE .L1_loopE; +JLE L(1_loopE); ALIGN_5; -.L1_bodyB:; +L(1_bodyB):; #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -312,9 +312,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2,k; -JLE .L2_loopE; +JLE L(2_loopE); ALIGN_5; -.L2_bodyB:; +L(2_bodyB):; # Computing kernel //#### Unroll times 1 #### @@ -430,11 +430,11 @@ MUL_DY yvec1, yvec4, yvec6; MUL_DY yvec1, yvec5, yvec7; ADD_DY yvec10, yvec6, yvec10; ADD_DY yvec8, yvec7, yvec8; -.L2_bodyE:; +L(2_bodyE):; DECQ k; -JG .L2_bodyB; +JG L(2_bodyB); ALIGN_5 -.L2_loopE:; +L(2_loopE):; PREFETCH2 0*SIZE(prebb); ADDQ $8*SIZE, prebb; #ifndef TRMMKERNEL @@ -443,9 +443,9 @@ TEST $2, bk; MOVQ kkk, %rax; TEST $2, %rax; #endif -JLE .L3_loopE; +JLE L(3_loopE); ALIGN_5 -.L3_bodyB: +L(3_bodyB): //#### Unroll times 1 #### PREFETCH0 64*SIZE(ptrba) LD_DY 4*SIZE(ptrba), yvec1; @@ -504,7 +504,7 @@ MUL_DY yvec1, yvec5, yvec7; ADD_DY yvec10, yvec6, yvec10; ADD_DY yvec8, yvec7, yvec8; -.L3_loopE: +L(3_loopE): PREFETCH2 0*SIZE(prebb); ADDQ $8*SIZE, prebb #ifndef TRMMKERNEL @@ -513,9 +513,9 @@ TEST $1, bk; MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L4_loopE; +JLE L(4_loopE); ALIGN_5 -.L4_bodyB:; +L(4_bodyB):; //#### Unroll times 1 #### PREFETCH0 64*SIZE(ptrba) LD_DY 4*SIZE(ptrba), yvec1; @@ -543,7 +543,7 @@ MUL_DY yvec1, yvec5, yvec7; ADD_DY yvec10, yvec6, yvec10; ADD_DY yvec8, yvec7, yvec8; -.L4_loopE:; +L(4_loopE):; //#### Load Alpha #### BROAD_DY MEMALPHA,yvec7; //#### Multiply Alpha #### @@ -572,7 +572,7 @@ REVS_DY $0x0a,yvec7,yvec8,yvec8; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L4_loopEx; # Unalign part write back +JNE L(4_loopEx); # Unalign part write back ALIGN_5 //#### Writing Back #### EXTRA_DY $1,yvec15,xvec7; @@ -629,12 +629,12 @@ ADDQ $8, kk #endif ADDQ $8*SIZE,C0; ADDQ $8*SIZE,C1; -.L1_bodyE:; +L(1_bodyE):; DECQ i; -JG .L1_bodyB; -JMP .L1_loopE; +JG L(1_bodyB); +JMP L(1_loopE); ALIGN_5; -.L4_loopEx:; +L(4_loopEx):; EXTRA_DY $1, yvec15, xvec7; #ifndef TRMMKERNEL LDL_DY 0*SIZE(C0), xvec6, xvec6; @@ -760,13 +760,13 @@ ADDQ $8, kk ADDQ $8*SIZE, C0; ADDQ $8*SIZE, C1; DECQ i; -JG .L1_bodyB; +JG L(1_bodyB); ALIGN_5 -.L1_loopE:; +L(1_loopE):; TEST $4, bm; # Rm = 4 -JLE .L5_loopE; +JLE L(5_loopE); ALIGN_5 -.L5_bodyB:; +L(5_bodyB):; #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -800,9 +800,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L6_loopE; +JLE L(6_loopE); ALIGN_5; -.L6_bodyB:; +L(6_bodyB):; # Computing kernel //#### Untoll time 1 #### @@ -871,18 +871,18 @@ VPERMILP_DY $0x05, yvec2, yvec3; MUL_DY yvec1, yvec5, yvec7; ADD_DY yvec9, yvec7, yvec9; DECQ k; -JG .L6_bodyB; +JG L(6_bodyB); ALIGN_5 -.L6_loopE:; +L(6_loopE):; #ifndef TRMMKERNEL TEST $2, bk; #else MOVQ kkk, %rax; TEST $2, %rax; #endif -JLE .L7_loopE; +JLE L(7_loopE); ALIGN_5 -.L7_bodyB:; +L(7_bodyB):; //#### Untoll time 1 #### LD_DY 4*SIZE(ptrba), yvec1; MUL_DY yvec0, yvec2, yvec6; @@ -917,16 +917,16 @@ VPERMILP_DY $0x05, yvec2, yvec3; MUL_DY yvec1, yvec5, yvec7; ADD_DY yvec9, yvec7, yvec9; -.L7_loopE:; +L(7_loopE):; #ifndef TRMMKERNEL TEST $1, bk #else MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L8_loopE; +JLE L(8_loopE); ALIGN_5 -.L8_bodyB:; +L(8_bodyB):; //#### Untoll time 1 #### MUL_DY yvec0, yvec2, yvec6; ADD_DY yvec15, yvec6, yvec15; @@ -942,7 +942,7 @@ ADDQ $4*SIZE, ptrbb; MUL_DY yvec0, yvec5, yvec7; ADD_DY yvec9, yvec7, yvec9; -.L8_loopE:; +L(8_loopE):; //#### Load Alpha #### BROAD_DY MEMALPHA, yvec7; //#### Multiply Alpha #### @@ -961,7 +961,7 @@ REVS_DY $0x0a,yvec7,yvec9,yvec9; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L8_loopEx; # Unalign part write back +JNE L(8_loopEx); # Unalign part write back ALIGN_5 //#### Writing Back #### EXTRA_DY $1,yvec15,xvec7; @@ -998,9 +998,9 @@ ADDQ $4, kk #endif ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -JMP .L5_loopE; +JMP L(5_loopE); ALIGN_5 -.L8_loopEx:; +L(8_loopEx):; EXTRA_DY $1,yvec15,xvec7; EXTRA_DY $1,yvec13,xvec5; EXTRA_DY $1,yvec11,xvec3; @@ -1062,11 +1062,11 @@ ADDQ $4, kk ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -.L5_loopE:; +L(5_loopE):; TEST $2, bm; -JLE .L9_loopE; +JLE L(9_loopE); ALIGN_5 -.L9_bodyB:; +L(9_bodyB):; #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -1101,9 +1101,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L10_loopE; +JLE L(10_loopE); ALIGN_5; -.L10_bodyB:; +L(10_bodyB):; # Computing kernel //#### Unroll time 1 #### @@ -1176,18 +1176,18 @@ SHUF_DX $0x4e, xvec2, xvec4; MUL_DX xvec1, xvec5, xvec5; ADD_DX xvec5, xvec9, xvec9; DECQ k; -JG .L10_bodyB; +JG L(10_bodyB); ALIGN_5 -.L10_loopE:; +L(10_loopE):; #ifndef TRMMKERNEL TEST $2, bk #else MOVQ kkk, %rax; TEST $2, %rax; #endif -JLE .L11_loopE; +JLE L(11_loopE); ALIGN_5 -.L11_bodyB:; +L(11_bodyB):; //#### Unroll time 1 #### LD_DX 4*SIZE(ptrbb), xvec6; SHUF_DX $0x4e, xvec3, xvec5; @@ -1225,16 +1225,16 @@ SHUF_DX $0x4e, xvec2, xvec4; MUL_DX xvec1, xvec5, xvec5; ADD_DX xvec5, xvec9, xvec9; -.L11_loopE:; +L(11_loopE):; #ifndef TRMMKERNEL TEST $1, bk #else MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L12_loopE; +JLE L(12_loopE); ALIGN_5 -.L12_bodyB:; +L(12_bodyB):; SHUF_DX $0x4e, xvec3, xvec5; MUL_DX xvec0, xvec2, xvec2; ADD_DX xvec2, xvec15, xvec15; @@ -1250,7 +1250,7 @@ ADD_DX xvec4, xvec13, xvec13; MUL_DX xvec0, xvec5, xvec5; ADD_DX xvec5, xvec9, xvec9; -.L12_loopE:; +L(12_loopE):; //#### Load Alpha #### BROAD_DX MEMALPHA, xvec7; //#### Multiply Alpha #### @@ -1269,7 +1269,7 @@ REVS_DX xvec6, xvec9, xvec9; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L12_loopEx; +JNE L(12_loopEx); ALIGN_5 //#### Writing Back #### #ifndef TRMMKERNEL @@ -1294,9 +1294,9 @@ ADDQ $2, kk #endif ADDQ $2*SIZE, C0 ADDQ $2*SIZE, C1 -JMP .L9_loopE; +JMP L(9_loopE); ALIGN_5 -.L12_loopEx: +L(12_loopEx): #ifndef TRMMKERNEL LDL_DX 0*SIZE(C0), xvec14, xvec14; LDH_DX 1*SIZE(C0), xvec14, xvec14; @@ -1331,11 +1331,11 @@ ADDQ $2, kk #endif ADDQ $2*SIZE, C0; ADDQ $2*SIZE, C1; -.L9_loopE:; +L(9_loopE):; TEST $1, bm -JLE .L13_loopE; +JLE L(13_loopE); ALIGN_5 -.L13_bodyB:; +L(13_bodyB):; #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -1363,9 +1363,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L14_loopE; +JLE L(14_loopE); ALIGN_5 -.L14_bodyB:; +L(14_bodyB):; BROAD_DY 0*SIZE(ptrba), yvec0; LD_DY 0*SIZE(ptrbb), yvec2; MUL_DY yvec0, yvec2, yvec6; @@ -1388,18 +1388,18 @@ ADD_DY yvec15, yvec7, yvec15; ADDQ $4*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; DECQ k; -JG .L14_bodyB; +JG L(14_bodyB); ALIGN_5 -.L14_loopE: +L(14_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else MOVQ kkk, %rax; TEST $2, %rax; #endif -JLE .L15_loopE; +JLE L(15_loopE); ALIGN_5 -.L15_bodyB: +L(15_bodyB): BROAD_DY 0*SIZE(ptrba), yvec0; LD_DY 0*SIZE(ptrbb), yvec2; MUL_DY yvec0, yvec2, yvec6; @@ -1411,16 +1411,16 @@ MUL_DY yvec1, yvec3, yvec7; ADD_DY yvec15, yvec7, yvec15; ADDQ $2*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L15_loopE:; +L(15_loopE):; #ifndef TRMMKERNEL TEST $1, bk; #else MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L16_loopE; +JLE L(16_loopE); ALIGN_5 -.L16_bodyB:; +L(16_bodyB):; BROAD_DY 0*SIZE(ptrba), yvec0; LD_DY 0*SIZE(ptrbb), yvec2; MUL_DY yvec0, yvec2, yvec6; @@ -1428,7 +1428,7 @@ ADD_DY yvec15, yvec6, yvec15; ADDQ $1*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L16_loopE: +L(16_loopE): //#### Load Alpha #### BROAD_DY MEMALPHA, yvec7; //#### Multiply Alpha #### @@ -1459,7 +1459,7 @@ ADDQ $1, kk #endif ADDQ $1*SIZE, C0 ADDQ $1*SIZE, C1 -.L13_loopE:; +L(13_loopE):; #if defined(TRMMKERNEL)&&!defined(LEFT) ADDQ $4, kk #endif @@ -1467,15 +1467,15 @@ MOVQ bk,k; SALQ $5,k; ADDQ k,bb; LEAQ (C,ldc,4),C; -.L0_bodyE:; +L(0_bodyE):; DECQ j; -JG .L0_bodyB; +JG L(0_bodyB); ALIGN_5; -.L0_loopE:; +L(0_loopE):; TEST $2, bn; -JLE .L20_loopE; +JLE L(20_loopE); ALIGN_5; -.L20_loopB:; +L(20_loopB):; #if defined(TRMMKERNEL) && defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk @@ -1485,9 +1485,9 @@ LEAQ (C, ldc, 1), C1; MOVQ ba, ptrba; MOVQ bm, i; SARQ $3, i; # Rm = 8 -JLE .L21_loopE; +JLE L(21_loopE); ALIGN_5; -.L21_bodyB:; +L(21_bodyB):; #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -1522,9 +1522,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L211_loopE; +JLE L(211_loopE); ALIGN_5; -.L211_bodyB: +L(211_bodyB): # Computing kernel //#### Unroll time 1 #### LD_DX 0*SIZE(ptrba), xvec0; @@ -1676,18 +1676,18 @@ ADD_DX xvec6, xvec9, xvec9; MUL_DX xvec3, xvec7, xvec7; ADD_DX xvec7, xvec8, xvec8; DECQ k; -JG .L211_bodyB; +JG L(211_bodyB); ALIGN_5 -.L211_loopE: +L(211_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else MOVQ kkk, %rax; TEST $2, %rax; #endif -JLE .L212_loopE; +JLE L(212_loopE); ALIGN_5; -.L212_bodyB: +L(212_bodyB): # Computing kernel //#### Unroll time 1 #### LD_DX 0*SIZE(ptrba), xvec0; @@ -1765,16 +1765,16 @@ ADD_DX xvec6, xvec9, xvec9; MUL_DX xvec3, xvec7, xvec7; ADD_DX xvec7, xvec8, xvec8; -.L212_loopE: +L(212_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L213_loopE; +JLE L(213_loopE); ALIGN_5 -.L213_bodyB: +L(213_bodyB): //#### Unroll time 1 #### LD_DX 0*SIZE(ptrba), xvec0; LD_DX 0*SIZE(ptrbb), xvec4; @@ -1814,7 +1814,7 @@ ADD_DX xvec6, xvec9, xvec9; MUL_DX xvec3, xvec7, xvec7; ADD_DX xvec7, xvec8, xvec8; -.L213_loopE: +L(213_loopE): //#### Multiply Alpha #### BROAD_DX MEMALPHA, xvec7; MUL_DX xvec7, xvec15, xvec15; @@ -1842,7 +1842,7 @@ REVS_DX xvec6, xvec8, xvec8; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L213_loopEx; +JNE L(213_loopEx); ALIGN_5 //#### Writing Back #### #ifndef TRMMKERNEL @@ -1876,10 +1876,10 @@ ADDQ $8, kk ADDQ $8*SIZE, C0; ADDQ $8*SIZE, C1; DECQ i; -JG .L21_bodyB; -JMP .L21_loopE; +JG L(21_bodyB); +JMP L(21_loopE); ALIGN_5 -.L213_loopEx:; +L(213_loopEx):; #ifndef TRMMKERNEL LDL_DX 0*SIZE(C0), xvec0, xvec0; LDH_DX 1*SIZE(C0), xvec0, xvec0; @@ -1937,12 +1937,12 @@ ADDQ $8, kk ADDQ $8*SIZE, C0; ADDQ $8*SIZE, C1; DECQ i; -JG .L21_bodyB; -.L21_loopE:; +JG L(21_bodyB); +L(21_loopE):; TEST $4, bm; # Rm = 4 -JLE .L22_loopE; +JLE L(22_loopE); ALIGN_5; -.L22_bodyB:; +L(22_bodyB):; #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -1973,9 +1973,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L221_loopE; +JLE L(221_loopE); ALIGN_5 -.L221_bodyB:; +L(221_bodyB):; # Computing kernel //#### Unroll time 1 #### LD_DX 0*SIZE(ptrba), xvec0; @@ -2055,18 +2055,18 @@ ADD_DX xvec4, xvec11, xvec11; MUL_DX xvec1, xvec5, xvec5; ADD_DX xvec5, xvec10, xvec10; DECQ k; -JG .L221_bodyB; +JG L(221_bodyB); ALIGN_5 -.L221_loopE:; +L(221_loopE):; #ifndef TRMMKERNEL TEST $2, bk; #else MOVQ kkk, %rax; TEST $2, %rax; #endif -JLE .L222_loopE; +JLE L(222_loopE); ALIGN_5 -.L222_bodyB: +L(222_bodyB): //#### Unroll time 1 #### LD_DX 0*SIZE(ptrba), xvec0; LD_DX 0*SIZE(ptrbb), xvec4; @@ -2106,16 +2106,16 @@ ADD_DX xvec4, xvec11, xvec11; MUL_DX xvec1, xvec5, xvec5; ADD_DX xvec5, xvec10, xvec10; -.L222_loopE: +L(222_loopE): #ifndef TRMMKERNEL TEST $1, bk #else MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L223_loopE; +JLE L(223_loopE); ALIGN_5 -.L223_bodyB: +L(223_bodyB): //#### Unroll time 1 #### LD_DX 0*SIZE(ptrba), xvec0; LD_DX 0*SIZE(ptrbb), xvec4; @@ -2137,7 +2137,7 @@ ADD_DX xvec4, xvec11, xvec11; MUL_DX xvec1, xvec5, xvec5; ADD_DX xvec5, xvec10, xvec10; -.L223_loopE: +L(223_loopE): //#### Multiply Alpha #### BROAD_DX MEMALPHA, xvec7; MUL_DX xvec7, xvec15, xvec15; @@ -2155,7 +2155,7 @@ REVS_DX xvec6, xvec10, xvec10; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L223_loopEx; +JNE L(223_loopEx); ALIGN_5 //#### Writing Back #### #ifndef TRMMKERNEL @@ -2180,9 +2180,9 @@ ADDQ $4, kk #endif ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -JMP .L22_loopE; +JMP L(22_loopE); ALIGN_5 -.L223_loopEx:; +L(223_loopEx):; #ifndef TRMMKERNEL LDL_DX 0*SIZE(C0), xvec0, xvec0; LDH_DX 1*SIZE(C0), xvec0, xvec0; @@ -2219,11 +2219,11 @@ ADDQ $4, kk #endif ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -.L22_loopE:; +L(22_loopE):; TEST $2, bm; // Rm = 2 -JLE .L23_loopE; +JLE L(23_loopE); ALIGN_5; -.L23_bodyB: +L(23_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -2251,9 +2251,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L231_loopE; +JLE L(231_loopE); ALIGN_5 -.L231_bodyB: +L(231_bodyB): # Computing kernel //#### Unroll time 1 #### LD_DX 0*SIZE(ptrba), xvec0; @@ -2293,18 +2293,18 @@ MUL_DX xvec0, xvec5, xvec5; ADD_DX xvec5, xvec11, xvec11; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L231_bodyB; +JG L(231_bodyB); ALIGN_5 -.L231_loopE: +L(231_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else MOVQ kkk, %rax; TEST $2, %rax; #endif -JLE .L232_loopE; +JLE L(232_loopE); ALIGN_5 -.L232_bodyB: +L(232_bodyB): //#### Unroll time 1 #### LD_DX 0*SIZE(ptrba), xvec0; LD_DX 0*SIZE(ptrbb), xvec4; @@ -2324,16 +2324,16 @@ ADDQ $4*SIZE, ptrba; MUL_DX xvec0, xvec5, xvec5; ADD_DX xvec5, xvec11, xvec11; ADDQ $4*SIZE, ptrbb; -.L232_loopE: +L(232_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L233_loopE; +JLE L(233_loopE); ALIGN_5 -.L233_bodyB: +L(233_bodyB): //#### Unroll time 1 #### LD_DX 0*SIZE(ptrba), xvec0; LD_DX 0*SIZE(ptrbb), xvec4; @@ -2344,7 +2344,7 @@ ADDQ $2*SIZE, ptrba; MUL_DX xvec0, xvec5, xvec5; ADD_DX xvec5, xvec11, xvec11; ADDQ $2*SIZE, ptrbb; -.L233_loopE: +L(233_loopE): //#### Multiply Alpha #### BROAD_DX MEMALPHA, xvec7; MUL_DX xvec7, xvec15, xvec15; @@ -2357,7 +2357,7 @@ REVS_DX xvec6, xvec11, xvec11; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L233_loopEx; +JNE L(233_loopEx); ALIGN_5 //#### Writing Back #### #ifndef TRMMKERNEL @@ -2378,9 +2378,9 @@ ADDQ $2, kk; #endif ADDQ $2*SIZE, C0; ADDQ $2*SIZE, C1; -JMP .L23_loopE; +JMP L(23_loopE); ALIGN_5 -.L233_loopEx:; +L(233_loopEx):; #ifndef TRMMKERNEL LDL_DX 0*SIZE(C0), xvec0, xvec0; LDH_DX 1*SIZE(C0), xvec0, xvec0; @@ -2407,11 +2407,11 @@ ADDQ $2, kk; #endif ADDQ $2*SIZE, C0; ADDQ $2*SIZE, C1; -.L23_loopE: +L(23_loopE): TEST $1, bm; // Rm = 1 -JLE .L24_loopE; +JLE L(24_loopE); ALIGN_5; -.L24_bodyB: +L(24_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -2438,9 +2438,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L241_loopE; +JLE L(241_loopE); ALIGN_5 -.L241_bodyB: +L(241_bodyB): BROAD_DX 0*SIZE(ptrba), xvec0; LD_DX 0*SIZE(ptrbb), xvec2; MUL_DX xvec0, xvec2, xvec2; @@ -2463,18 +2463,18 @@ ADD_DX xvec3, xvec15, xvec15; ADDQ $4*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L241_bodyB; +JG L(241_bodyB); ALIGN_5 -.L241_loopE: +L(241_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else MOVQ kkk, %rax; TEST $2, %rax; #endif -JLE .L242_loopE; +JLE L(242_loopE); ALIGN_5 -.L242_bodyB: +L(242_bodyB): BROAD_DX 0*SIZE(ptrba), xvec0; LD_DX 0*SIZE(ptrbb), xvec2; MUL_DX xvec0, xvec2, xvec2; @@ -2486,16 +2486,16 @@ MUL_DX xvec1, xvec3, xvec3; ADD_DX xvec3, xvec15, xvec15; ADDQ $2*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L242_loopE: +L(242_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L243_loopE; +JLE L(243_loopE); ALIGN_5 -.L243_bodyB: +L(243_bodyB): BROAD_DX 0*SIZE(ptrba), xvec0; LD_DX 0*SIZE(ptrbb), xvec2; MUL_DX xvec0, xvec2, xvec2; @@ -2503,7 +2503,7 @@ ADD_DX xvec2, xvec15, xvec15; ADDQ $1*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L243_loopE: +L(243_loopE): BROAD_DX MEMALPHA, xvec7; MUL_DX xvec7, xvec15, xvec15; #ifndef TRMMKERNEL @@ -2525,7 +2525,7 @@ ADDQ $1, kk; #endif ADDQ $1*SIZE, C0; ADDQ $1*SIZE, C1; -.L24_loopE: +L(24_loopE): #if defined(TRMMKERNEL) && !defined(LEFT) ADDQ $2, kk; #endif @@ -2533,11 +2533,11 @@ MOVQ bk, k; SALQ $4, k; ADDQ k, bb; LEAQ (C, ldc, 2), C; -.L20_loopE:; +L(20_loopE):; TEST $1, bn; // Rn = 1 -JLE .L30_loopE; +JLE L(30_loopE); ALIGN_5 -.L30_bodyB: +L(30_bodyB): #if defined(TRMMKERNEL)&&defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk; @@ -2546,9 +2546,9 @@ MOVQ C, C0; MOVQ ba, ptrba; MOVQ bm, i; SARQ $3, i; -JLE .L31_loopE; +JLE L(31_loopE); ALIGN_5 -.L31_bodyB: +L(31_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -2577,9 +2577,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L311_loopE; +JLE L(311_loopE); ALIGN_5 -.L311_bodyB: +L(311_bodyB): //#### Unroll time 1 #### LD_DY 0*SIZE(ptrba), yvec0; LD_DY 4*SIZE(ptrba), yvec1; @@ -2618,18 +2618,18 @@ MUL_DY yvec5, yvec4, yvec4; ADD_DY yvec4, yvec14, yvec14; ADDQ $4*SIZE, ptrbb; DECQ k; -JG .L311_bodyB; +JG L(311_bodyB); ALIGN_5 -.L311_loopE: +L(311_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else MOVQ kkk, %rax; TEST $2, %rax; #endif -JLE .L312_loopE; +JLE L(312_loopE); ALIGN_5 -.L312_bodyB: +L(312_bodyB): //#### Unroll time 1 #### LD_DY 0*SIZE(ptrba), yvec0; LD_DY 4*SIZE(ptrba), yvec1; @@ -2650,16 +2650,16 @@ MUL_DY yvec5, yvec4, yvec4 ADD_DY yvec4, yvec14, yvec14; ADDQ $2*SIZE, ptrbb; -.L312_loopE: +L(312_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L313_loopE; +JLE L(313_loopE); ALIGN_5 -.L313_bodyB: +L(313_bodyB): //#### Unroll time 1 #### LD_DY 0*SIZE(ptrba), yvec0; LD_DY 4*SIZE(ptrba), yvec1; @@ -2671,7 +2671,7 @@ MUL_DY yvec2, yvec1, yvec1; ADD_DY yvec1, yvec14, yvec14; ADDQ $1*SIZE, ptrbb; -.L313_loopE: +L(313_loopE): //#### Multiply Alpha #### BROAD_DY MEMALPHA, yvec7; MUL_DY yvec7, yvec15, yvec15; @@ -2680,7 +2680,7 @@ MUL_DY yvec7, yvec14, yvec14; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L313_loopEx; +JNE L(313_loopEx); ALIGN_5 //#### Writing Back #### EXTRA_DY $1, yvec15, xvec13; @@ -2707,10 +2707,10 @@ ADDQ $8, kk; #endif ADDQ $8*SIZE, C0; DECQ i; -JG .L31_bodyB; -JMP .L31_loopE; +JG L(31_bodyB); +JMP L(31_loopE); ALIGN_5 -.L313_loopEx: +L(313_loopEx): EXTRA_DY $1, yvec15, xvec13; EXTRA_DY $1, yvec14, xvec12; #ifndef TRMMKERNEL @@ -2747,12 +2747,12 @@ ADDQ $8, kk; #endif ADDQ $8*SIZE, C0; DECQ i; -JG .L31_bodyB; -.L31_loopE: +JG L(31_bodyB); +L(31_loopE): TEST $4, bm -JLE .L32_loopE; +JLE L(32_loopE); ALIGN_5 -.L32_bodyB: +L(32_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -2780,9 +2780,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk #endif SARQ $2, k; -JLE .L321_loopE; +JLE L(321_loopE); ALIGN_5 -.L321_bodyB: +L(321_bodyB): LD_DY 0*SIZE(ptrba), yvec0; BROAD_DY 0*SIZE(ptrbb), yvec1; MUL_DY yvec0, yvec1, yvec1; @@ -2805,18 +2805,18 @@ ADD_DY yvec7, yvec15, yvec15; ADDQ $16*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; DECQ k; -JG .L321_bodyB; +JG L(321_bodyB); ALIGN_5 -.L321_loopE: +L(321_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else MOVQ kkk, %rax; TEST $2, %rax; #endif -JLE .L322_loopE; +JLE L(322_loopE); ALIGN_5 -.L322_bodyB: +L(322_bodyB): LD_DY 0*SIZE(ptrba), yvec0; BROAD_DY 0*SIZE(ptrbb), yvec1; MUL_DY yvec0, yvec1, yvec1; @@ -2829,16 +2829,16 @@ ADD_DY yvec3, yvec15, yvec15; ADDQ $8*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L322_loopE: +L(322_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L323_loopE; +JLE L(323_loopE); ALIGN_5 -.L323_bodyB: +L(323_bodyB): LD_DY 0*SIZE(ptrba), yvec0; BROAD_DY 0*SIZE(ptrbb), yvec1; MUL_DY yvec0, yvec1, yvec1; @@ -2846,7 +2846,7 @@ ADD_DY yvec1, yvec15, yvec15; ADDQ $4*SIZE, ptrba; ADDQ $1*SIZE, ptrbb; -.L323_loopE: +L(323_loopE): //#### Multiply Alpha #### BROAD_DY MEMALPHA, yvec7; MUL_DY yvec7, yvec15, yvec15; @@ -2854,7 +2854,7 @@ MUL_DY yvec7, yvec15, yvec15; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L323_loopEx; +JNE L(323_loopEx); ALIGN_5 //#### Writing Back #### EXTRA_DY $1, yvec15, xvec14; @@ -2875,9 +2875,9 @@ ADDQ %rax, ptrbb; ADDQ $4, kk #endif ADDQ $4*SIZE, C0; -JMP .L32_loopE; +JMP L(32_loopE); ALIGN_5 -.L323_loopEx: +L(323_loopEx): //#### Writing Back #### EXTRA_DY $1, yvec15, xvec14; #ifndef TRMMKERNEL @@ -2903,11 +2903,11 @@ ADDQ %rax, ptrbb; ADDQ $4, kk #endif ADDQ $4*SIZE, C0; -.L32_loopE: +L(32_loopE): TEST $2, bm -JLE .L33_loopE; +JLE L(33_loopE); ALIGN_5 -.L33_bodyB: +L(33_bodyB): #if !defined(TRMMKERNEL) || (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -2935,9 +2935,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L331_loopE; +JLE L(331_loopE); ALIGN_5 -.L331_bodyB: +L(331_bodyB): LD_DX 0*SIZE(ptrba), xvec0; BROAD_DX 0*SIZE(ptrbb), xvec2; MUL_DX xvec0, xvec2, xvec2; @@ -2960,18 +2960,18 @@ ADD_DX xvec7, xvec15, xvec15; ADDQ $8*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; DECQ k; -JG .L331_bodyB; +JG L(331_bodyB); ALIGN_5 -.L331_loopE: +L(331_loopE): #ifndef TRMMKERNEL TEST $2,bk; #else MOVQ kkk, %rax; TEST $2, %rax #endif -JLE .L332_loopE; +JLE L(332_loopE); ALIGN_5 -.L332_bodyB: +L(332_bodyB): LD_DX 0*SIZE(ptrba), xvec0; BROAD_DX 0*SIZE(ptrbb), xvec2; MUL_DX xvec0, xvec2, xvec2; @@ -2983,23 +2983,23 @@ MUL_DX xvec1, xvec3, xvec3; ADD_DX xvec3, xvec15, xvec15; ADDQ $4*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L332_loopE: +L(332_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L333_loopE; +JLE L(333_loopE); ALIGN_5 -.L333_bodyB: +L(333_bodyB): LD_DX 0*SIZE(ptrba), xvec0; BROAD_DX 0*SIZE(ptrbb), xvec2; MUL_DX xvec0, xvec2, xvec2; ADD_DX xvec2, xvec15, xvec15; ADDQ $2*SIZE, ptrba; ADDQ $1*SIZE, ptrbb; -.L333_loopE: +L(333_loopE): //#### Multiply Alpha #### BROAD_DX MEMALPHA, xvec7; MUL_DX xvec7, xvec15, xvec15; @@ -3021,11 +3021,11 @@ ADDQ %rax, ptrbb; addq $2, kk #endif ADDQ $2*SIZE, C0; -.L33_loopE: +L(33_loopE): TEST $1, bm -JLE .L34_loopE; +JLE L(34_loopE); ALIGN_5 -.L34_bodyB: +L(34_bodyB): #if !defined(TRMMKERNEL) || (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -3052,9 +3052,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L341_loopE; +JLE L(341_loopE); ALIGN_5 -.L341_bodyB: +L(341_bodyB): vmovsd 0*SIZE(ptrba), xvec0; vmovsd 0*SIZE(ptrbb), xvec1; vmulsd xvec0, xvec1, xvec1; @@ -3077,18 +3077,18 @@ vaddsd xvec1, xvec15, xvec15; addq $4*SIZE, ptrba; addq $4*SIZE, ptrbb; decq k; -JG .L341_bodyB; +JG L(341_bodyB); ALIGN_5 -.L341_loopE: +L(341_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else MOVQ kkk, %rax; TEST $2, %rax; #endif -JLE .L342_loopE; +JLE L(342_loopE); ALIGN_5 -.L342_bodyB: +L(342_bodyB): vmovsd 0*SIZE(ptrba), xvec0; vmovsd 0*SIZE(ptrbb), xvec1; vmulsd xvec0, xvec1, xvec1; @@ -3101,16 +3101,16 @@ vaddsd xvec1, xvec15, xvec15; addq $2*SIZE, ptrba; addq $2*SIZE, ptrbb; -.L342_loopE: +L(342_loopE): #ifndef TRMMKERNEL TEST $1, bk #else MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L343_loopE; +JLE L(343_loopE); ALIGN_5 -.L343_bodyB: +L(343_bodyB): vmovsd 0*SIZE(ptrba), xvec0; vmovsd 0*SIZE(ptrbb), xvec1; vmulsd xvec0, xvec1, xvec1; @@ -3118,7 +3118,7 @@ vaddsd xvec1, xvec15, xvec15; addq $1*SIZE, ptrba; addq $1*SIZE, ptrbb; -.L343_loopE: +L(343_loopE): //#### Writing Back #### vmovsd MEMALPHA, xvec7; vmulsd xvec7, xvec15, xvec15; @@ -3138,13 +3138,13 @@ ADDQ %rax, ptrbb; addq $1, kk #endif addq $1*SIZE, C0; -.L34_loopE: +L(34_loopE): MOVQ bk, k SALQ $3, k; ADDQ k, bb; LEAQ (C, ldc, 1), C; -.L30_loopE: +L(30_loopE): movq 0(%rsp), %rbx; movq 8(%rsp), %rbp; movq 16(%rsp), %r12; diff --git a/kernel/x86_64/dgemm_kernel_4x8_skylakex.c b/kernel/x86_64/dgemm_kernel_4x8_skylakex.c index 6257e569ee..27848d0964 100644 --- a/kernel/x86_64/dgemm_kernel_4x8_skylakex.c +++ b/kernel/x86_64/dgemm_kernel_4x8_skylakex.c @@ -35,6 +35,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "common.h" #include +#if defined(__APPLE__) +#define L(x) "L" #x +#else +#define L(x) ".L" #x +#endif + /******************************************************************************************* * Macro definitions @@ -868,10 +874,10 @@ CNAME(BLASLONG m, BLASLONG n, BLASLONG k, double alpha, double * __restrict__ A, "vmovapd %%zmm1, %%zmm26\n" "vmovapd %%zmm1, %%zmm27\n" "vmovapd %%zmm1, %%zmm28\n" - "jmp .label24\n" + "jmp "L(label24)"\n" ".p2align 5\n" /* Inner math loop */ - ".label24:\n" + L(label24)":\n" "vmovupd -128(%[AO]),%%zmm0\n" "vmovupd -128(%[A1]),%%zmm10\n" "vmovupd -128(%[A2]),%%zmm20\n" @@ -924,7 +930,7 @@ CNAME(BLASLONG m, BLASLONG n, BLASLONG k, double alpha, double * __restrict__ A, "prefetch 512(%[A2])\n" "prefetch 512(%[BO])\n" "subl $1, %[kloop]\n" - "jg .label24\n" + "jg "L(label24)"\n" /* multiply the result by alpha */ "vbroadcastsd (%[alpha]), %%zmm9\n" /* And store additively in C */ @@ -1036,10 +1042,10 @@ CNAME(BLASLONG m, BLASLONG n, BLASLONG k, double alpha, double * __restrict__ A, "vmovapd %%zmm1, %%zmm16\n" "vmovapd %%zmm1, %%zmm17\n" "vmovapd %%zmm1, %%zmm18\n" - "jmp .label16\n" + "jmp "L(label16)"\n" ".p2align 5\n" /* Inner math loop */ - ".label16:\n" + L(label16)":\n" "vmovupd -128(%[AO]),%%zmm0\n" "vmovupd -128(%[A1]),%%zmm10\n" @@ -1081,7 +1087,7 @@ CNAME(BLASLONG m, BLASLONG n, BLASLONG k, double alpha, double * __restrict__ A, "prefetch 512(%[A1])\n" "prefetch 512(%[BO])\n" "subl $1, %[kloop]\n" - "jg .label16\n" + "jg "L(label16)"\n" /* multiply the result by alpha */ "vbroadcastsd (%[alpha]), %%zmm9\n" /* And store additively in C */ @@ -1164,10 +1170,10 @@ CNAME(BLASLONG m, BLASLONG n, BLASLONG k, double alpha, double * __restrict__ A, "vmovapd %%zmm1, %%zmm7\n" "vmovapd %%zmm1, %%zmm8\n" "vbroadcastsd (%[alpha]), %%zmm9\n" - "jmp .label1\n" + "jmp "L(label1)"\n" ".p2align 5\n" /* Inner math loop */ - ".label1:\n" + L(label1)":\n" "vmovupd -128(%[AO]),%%zmm0\n" "vfmadd231pd -96(%[BO])%{1to8%}, %%zmm0, %%zmm1\n" "vfmadd231pd -88(%[BO])%{1to8%}, %%zmm0, %%zmm2\n" @@ -1180,7 +1186,7 @@ CNAME(BLASLONG m, BLASLONG n, BLASLONG k, double alpha, double * __restrict__ A, "add $64, %[AO]\n" "add $64, %[BO]\n" "subl $1, %[kloop]\n" - "jg .label1\n" + "jg "L(label1)"\n" /* multiply the result by alpha and add to the memory */ "vfmadd213pd (%[C0]), %%zmm9, %%zmm1\n" "vfmadd213pd (%[C1]), %%zmm9, %%zmm2\n" diff --git a/kernel/x86_64/dgemm_kernel_6x4_piledriver.S b/kernel/x86_64/dgemm_kernel_6x4_piledriver.S index 66779648ba..90711d90a8 100644 --- a/kernel/x86_64/dgemm_kernel_6x4_piledriver.S +++ b/kernel/x86_64/dgemm_kernel_6x4_piledriver.S @@ -223,9 +223,9 @@ leaq (, LDC, SIZE), LDC MOVQ1280(_bk_j,j); SARQ1280($2,j); -JLE ._L_0_loopE; +JLE L(_L_0_loopE); ALIGN_4; -._L_0_bodyB:; +L(_L_0_bodyB):; MOVQ1280(_ptr_A,_ptr__A_0); MOVQ1280(_ptr_C,_ptr__C_0); LEAQ1280((_ptr_C,LDC,2),_ptr__C_1); @@ -234,8 +234,8 @@ SALQ1280($5,%rax); ADDQ1280(%rax,_pre_B); MOVQ1280(_bk_i,i); CMPQ1280($6,i); -JL ._L_1_loopE; -._L_1_bodyB:; +JL L(_L_1_loopE); +L(_L_1_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1282(XMM0,XMM0,XMM0); VXOR1282(XMM1,XMM1,XMM1); @@ -255,9 +255,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_2_loopE; +JLE L(_L_2_loopE); ALIGN_4; -._L_2_bodyB:; +L(_L_2_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM12); @@ -340,15 +340,15 @@ VMA1282(XMM13,XMM15,XMM10,XMM10); VMA1282(XMM14,XMM15,XMM11,XMM11); ADDQ1280($24*SIZE,_ptr__A_0); ADDQ1280($16*SIZE,_ptr__B_0); -._L_2_bodyE:; +L(_L_2_bodyE):; DECQ1280(k); -JG ._L_2_bodyB; +JG L(_L_2_bodyB); ALIGN_4; -._L_2_loopE:; +L(_L_2_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_3_loopE; +JLE L(_L_3_loopE); ALIGN_4; -._L_3_bodyB:; +L(_L_3_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM12); @@ -391,11 +391,11 @@ VMA1282(XMM13,XMM15,XMM10,XMM10); VMA1282(XMM14,XMM15,XMM11,XMM11); ADDQ1280($12*SIZE,_ptr__A_0); ADDQ1280($8*SIZE,_ptr__B_0); -._L_3_loopE:; +L(_L_3_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_4_loopE; +JLE L(_L_4_loopE); ALIGN_4; -._L_4_bodyB:; +L(_L_4_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM12); @@ -418,7 +418,7 @@ VMA1282(XMM13,XMM15,XMM10,XMM10); VMA1282(XMM14,XMM15,XMM11,XMM11); ADDQ1280($6*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_4_loopE:; +L(_L_4_loopE):; BROAD1282(alpha,XMM12); VLDU1282(0*SIZE(_ptr__C_0),XMM13); VMA21282(XMM12,XMM0,XMM13,XMM0); @@ -458,15 +458,15 @@ VMA21282(XMM12,XMM11,XMM15,XMM11); VSTU1282(XMM15,4*SIZE(_ptr__C_1,LDC,1)); ADDQ1280($6*SIZE,_ptr__C_0); ADDQ1280($6*SIZE,_ptr__C_1); -._L_1_bodyE:; +L(_L_1_bodyE):; SUBQ1280($6,i); -JG ._L_1_bodyB; +JG L(_L_1_bodyB); ALIGN_4; -._L_1_loopE:; +L(_L_1_loopE):; TESTQ1280($4,i); -JLE ._L_5_loopE; +JLE L(_L_5_loopE); ALIGN_4; -._L_5_bodyB:; +L(_L_5_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1282(XMM0,XMM0,XMM0); VXOR1282(XMM1,XMM1,XMM1); @@ -482,9 +482,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_6_loopE; +JLE L(_L_6_loopE); ALIGN_4; -._L_6_bodyB:; +L(_L_6_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM13); @@ -547,15 +547,15 @@ VMA1282(XMM13,XMM15,XMM6,XMM6); VMA1282(XMM14,XMM15,XMM7,XMM7); ADDQ1280($16*SIZE,_ptr__A_0); ADDQ1280($16*SIZE,_ptr__B_0); -._L_6_bodyE:; +L(_L_6_bodyE):; DECQ1280(k); -JG ._L_6_bodyB; +JG L(_L_6_bodyB); ALIGN_4; -._L_6_loopE:; +L(_L_6_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_7_loopE; +JLE L(_L_7_loopE); ALIGN_4; -._L_7_bodyB:; +L(_L_7_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM13); @@ -588,11 +588,11 @@ VMA1282(XMM13,XMM15,XMM6,XMM6); VMA1282(XMM14,XMM15,XMM7,XMM7); ADDQ1280($8*SIZE,_ptr__A_0); ADDQ1280($8*SIZE,_ptr__B_0); -._L_7_loopE:; +L(_L_7_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_8_loopE; +JLE L(_L_8_loopE); ALIGN_4; -._L_8_bodyB:; +L(_L_8_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM13); @@ -610,7 +610,7 @@ VMA1282(XMM13,XMM15,XMM6,XMM6); VMA1282(XMM14,XMM15,XMM7,XMM7); ADDQ1280($4*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_8_loopE:; +L(_L_8_loopE):; BROAD1282(alpha,XMM8); VLDU1282(0*SIZE(_ptr__C_0),XMM9); VMA21282(XMM8,XMM0,XMM9,XMM0); @@ -638,11 +638,11 @@ VMA21282(XMM8,XMM7,XMM9,XMM7); VSTU1282(XMM9,2*SIZE(_ptr__C_1,LDC,1)); ADDQ1280($4*SIZE,_ptr__C_0); ADDQ1280($4*SIZE,_ptr__C_1); -._L_5_loopE:; +L(_L_5_loopE):; TESTQ1280($2,i); -JLE ._L_9_loopE; +JLE L(_L_9_loopE); ALIGN_4; -._L_9_bodyB:; +L(_L_9_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1282(XMM0,XMM0,XMM0); VXOR1282(XMM1,XMM1,XMM1); @@ -654,9 +654,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_10_loopE; +JLE L(_L_10_loopE); ALIGN_4; -._L_10_bodyB:; +L(_L_10_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM14); @@ -699,15 +699,15 @@ BROAD1282(15*SIZE(_ptr__B_0),XMM15); VMA1282(XMM14,XMM15,XMM3,XMM3); ADDQ1280($8*SIZE,_ptr__A_0); ADDQ1280($16*SIZE,_ptr__B_0); -._L_10_bodyE:; +L(_L_10_bodyE):; DECQ1280(k); -JG ._L_10_bodyB; +JG L(_L_10_bodyB); ALIGN_4; -._L_10_loopE:; +L(_L_10_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_11_loopE; +JLE L(_L_11_loopE); ALIGN_4; -._L_11_bodyB:; +L(_L_11_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM14); @@ -730,11 +730,11 @@ BROAD1282(7*SIZE(_ptr__B_0),XMM15); VMA1282(XMM14,XMM15,XMM3,XMM3); ADDQ1280($4*SIZE,_ptr__A_0); ADDQ1280($8*SIZE,_ptr__B_0); -._L_11_loopE:; +L(_L_11_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_12_loopE; +JLE L(_L_12_loopE); ALIGN_4; -._L_12_bodyB:; +L(_L_12_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM14); @@ -747,7 +747,7 @@ BROAD1282(3*SIZE(_ptr__B_0),XMM15); VMA1282(XMM14,XMM15,XMM3,XMM3); ADDQ1280($2*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_12_loopE:; +L(_L_12_loopE):; BROAD1282(alpha,XMM4); VLDU1282(0*SIZE(_ptr__C_0),XMM5); VMA21282(XMM4,XMM0,XMM5,XMM0); @@ -763,11 +763,11 @@ VMA21282(XMM4,XMM3,XMM8,XMM3); VSTU1282(XMM8,0*SIZE(_ptr__C_1,LDC,1)); ADDQ1280($2*SIZE,_ptr__C_0); ADDQ1280($2*SIZE,_ptr__C_1); -._L_9_loopE:; +L(_L_9_loopE):; TESTQ1280($1,i); -JLE ._L_13_loopE; +JLE L(_L_13_loopE); ALIGN_4; -._L_13_bodyB:; +L(_L_13_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1281(XMM0,XMM0,XMM0); VXOR1281(XMM1,XMM1,XMM1); @@ -779,9 +779,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_14_loopE; +JLE L(_L_14_loopE); ALIGN_4; -._L_14_bodyB:; +L(_L_14_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1281(0*SIZE(_ptr__B_0),XMM15); VLD1281(0*SIZE(_ptr__A_0),XMM14); @@ -824,15 +824,15 @@ BROAD1281(15*SIZE(_ptr__B_0),XMM15); VMA1281(XMM14,XMM15,XMM3,XMM3); ADDQ1280($4*SIZE,_ptr__A_0); ADDQ1280($16*SIZE,_ptr__B_0); -._L_14_bodyE:; +L(_L_14_bodyE):; DECQ1280(k); -JG ._L_14_bodyB; +JG L(_L_14_bodyB); ALIGN_4; -._L_14_loopE:; +L(_L_14_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_15_loopE; +JLE L(_L_15_loopE); ALIGN_4; -._L_15_bodyB:; +L(_L_15_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1281(0*SIZE(_ptr__B_0),XMM15); VLD1281(0*SIZE(_ptr__A_0),XMM14); @@ -855,11 +855,11 @@ BROAD1281(7*SIZE(_ptr__B_0),XMM15); VMA1281(XMM14,XMM15,XMM3,XMM3); ADDQ1280($2*SIZE,_ptr__A_0); ADDQ1280($8*SIZE,_ptr__B_0); -._L_15_loopE:; +L(_L_15_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_16_loopE; +JLE L(_L_16_loopE); ALIGN_4; -._L_16_bodyB:; +L(_L_16_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1281(0*SIZE(_ptr__B_0),XMM15); VLD1281(0*SIZE(_ptr__A_0),XMM14); @@ -872,7 +872,7 @@ BROAD1281(3*SIZE(_ptr__B_0),XMM15); VMA1281(XMM14,XMM15,XMM3,XMM3); ADDQ1280($1*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_16_loopE:; +L(_L_16_loopE):; BROAD1281(alpha,XMM4); VLDU1281(0*SIZE(_ptr__C_0),XMM5); VMA21281(XMM4,XMM0,XMM5,XMM0); @@ -888,22 +888,22 @@ VMA21281(XMM4,XMM3,XMM8,XMM3); VSTU1281(XMM8,0*SIZE(_ptr__C_1,LDC,1)); ADDQ1280($1*SIZE,_ptr__C_0); ADDQ1280($1*SIZE,_ptr__C_1); -._L_13_loopE:; +L(_L_13_loopE):; MOVQ1280(LDC,%rax); SALQ1280($2,%rax); ADDQ1280(%rax,_ptr_C); MOVQ1280(_bk_l,%rax); SALQ1280($5,%rax); ADDQ1280(%rax,_ptr_B); -._L_0_bodyE:; +L(_L_0_bodyE):; DECQ1280(j); -JG ._L_0_bodyB; +JG L(_L_0_bodyB); ALIGN_4; -._L_0_loopE:; +L(_L_0_loopE):; TESTQ1280($2,_bk_j); -JLE ._L_17_loopE; +JLE L(_L_17_loopE); ALIGN_4; -._L_17_bodyB:; +L(_L_17_bodyB):; MOVQ1280(_ptr_A,_ptr__A_0); MOVQ1280(_ptr_C,_ptr__C_0); LEAQ1280((_ptr_C,LDC,1),_ptr__C_1); @@ -912,8 +912,8 @@ SALQ1280($4,%rax); ADDQ1280(%rax,_pre_B); MOVQ1280(_bk_i,i); CMPQ1280($6,i); -JL ._L_18_loopE; -._L_18_bodyB:; +JL L(_L_18_loopE); +L(_L_18_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1282(XMM0,XMM0,XMM0); VXOR1282(XMM1,XMM1,XMM1); @@ -927,9 +927,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_19_loopE; +JLE L(_L_19_loopE); ALIGN_4; -._L_19_bodyB:; +L(_L_19_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM12); @@ -980,15 +980,15 @@ VMA1282(XMM13,XMM15,XMM4,XMM4); VMA1282(XMM14,XMM15,XMM5,XMM5); ADDQ1280($24*SIZE,_ptr__A_0); ADDQ1280($8*SIZE,_ptr__B_0); -._L_19_bodyE:; +L(_L_19_bodyE):; DECQ1280(k); -JG ._L_19_bodyB; +JG L(_L_19_bodyB); ALIGN_4; -._L_19_loopE:; +L(_L_19_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_20_loopE; +JLE L(_L_20_loopE); ALIGN_4; -._L_20_bodyB:; +L(_L_20_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM12); @@ -1015,11 +1015,11 @@ VMA1282(XMM13,XMM15,XMM4,XMM4); VMA1282(XMM14,XMM15,XMM5,XMM5); ADDQ1280($12*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_20_loopE:; +L(_L_20_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_21_loopE; +JLE L(_L_21_loopE); ALIGN_4; -._L_21_bodyB:; +L(_L_21_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM12); @@ -1034,7 +1034,7 @@ VMA1282(XMM13,XMM15,XMM4,XMM4); VMA1282(XMM14,XMM15,XMM5,XMM5); ADDQ1280($6*SIZE,_ptr__A_0); ADDQ1280($2*SIZE,_ptr__B_0); -._L_21_loopE:; +L(_L_21_loopE):; BROAD1282(alpha,XMM6); VLDU1282(0*SIZE(_ptr__C_0),XMM7); VMA21282(XMM6,XMM0,XMM7,XMM0); @@ -1056,15 +1056,15 @@ VMA21282(XMM6,XMM5,XMM12,XMM5); VSTU1282(XMM12,4*SIZE(_ptr__C_0,LDC,1)); ADDQ1280($6*SIZE,_ptr__C_0); ADDQ1280($6*SIZE,_ptr__C_1); -._L_18_bodyE:; +L(_L_18_bodyE):; SUBQ1280($6,i); -JG ._L_18_bodyB; +JG L(_L_18_bodyB); ALIGN_4; -._L_18_loopE:; +L(_L_18_loopE):; TESTQ1280($4,i); -JLE ._L_22_loopE; +JLE L(_L_22_loopE); ALIGN_4; -._L_22_bodyB:; +L(_L_22_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1282(XMM0,XMM0,XMM0); VXOR1282(XMM1,XMM1,XMM1); @@ -1076,9 +1076,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_23_loopE; +JLE L(_L_23_loopE); ALIGN_4; -._L_23_bodyB:; +L(_L_23_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM13); @@ -1117,15 +1117,15 @@ VMA1282(XMM13,XMM15,XMM2,XMM2); VMA1282(XMM14,XMM15,XMM3,XMM3); ADDQ1280($16*SIZE,_ptr__A_0); ADDQ1280($8*SIZE,_ptr__B_0); -._L_23_bodyE:; +L(_L_23_bodyE):; DECQ1280(k); -JG ._L_23_bodyB; +JG L(_L_23_bodyB); ALIGN_4; -._L_23_loopE:; +L(_L_23_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_24_loopE; +JLE L(_L_24_loopE); ALIGN_4; -._L_24_bodyB:; +L(_L_24_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM13); @@ -1146,11 +1146,11 @@ VMA1282(XMM13,XMM15,XMM2,XMM2); VMA1282(XMM14,XMM15,XMM3,XMM3); ADDQ1280($8*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_24_loopE:; +L(_L_24_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_25_loopE; +JLE L(_L_25_loopE); ALIGN_4; -._L_25_bodyB:; +L(_L_25_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM13); @@ -1162,7 +1162,7 @@ VMA1282(XMM13,XMM15,XMM2,XMM2); VMA1282(XMM14,XMM15,XMM3,XMM3); ADDQ1280($4*SIZE,_ptr__A_0); ADDQ1280($2*SIZE,_ptr__B_0); -._L_25_loopE:; +L(_L_25_loopE):; BROAD1282(alpha,XMM4); VLDU1282(0*SIZE(_ptr__C_0),XMM5); VMA21282(XMM4,XMM0,XMM5,XMM0); @@ -1178,11 +1178,11 @@ VMA21282(XMM4,XMM3,XMM8,XMM3); VSTU1282(XMM8,2*SIZE(_ptr__C_0,LDC,1)); ADDQ1280($4*SIZE,_ptr__C_0); ADDQ1280($4*SIZE,_ptr__C_1); -._L_22_loopE:; +L(_L_22_loopE):; TESTQ1280($2,i); -JLE ._L_26_loopE; +JLE L(_L_26_loopE); ALIGN_4; -._L_26_bodyB:; +L(_L_26_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1282(XMM0,XMM0,XMM0); VXOR1282(XMM1,XMM1,XMM1); @@ -1192,9 +1192,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_27_loopE; +JLE L(_L_27_loopE); ALIGN_4; -._L_27_bodyB:; +L(_L_27_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM14); @@ -1221,15 +1221,15 @@ BROAD1282(7*SIZE(_ptr__B_0),XMM15); VMA1282(XMM14,XMM15,XMM1,XMM1); ADDQ1280($8*SIZE,_ptr__A_0); ADDQ1280($8*SIZE,_ptr__B_0); -._L_27_bodyE:; +L(_L_27_bodyE):; DECQ1280(k); -JG ._L_27_bodyB; +JG L(_L_27_bodyB); ALIGN_4; -._L_27_loopE:; +L(_L_27_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_28_loopE; +JLE L(_L_28_loopE); ALIGN_4; -._L_28_bodyB:; +L(_L_28_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM14); @@ -1244,11 +1244,11 @@ BROAD1282(3*SIZE(_ptr__B_0),XMM15); VMA1282(XMM14,XMM15,XMM1,XMM1); ADDQ1280($4*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_28_loopE:; +L(_L_28_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_29_loopE; +JLE L(_L_29_loopE); ALIGN_4; -._L_29_bodyB:; +L(_L_29_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM14); @@ -1257,7 +1257,7 @@ BROAD1282(1*SIZE(_ptr__B_0),XMM15); VMA1282(XMM14,XMM15,XMM1,XMM1); ADDQ1280($2*SIZE,_ptr__A_0); ADDQ1280($2*SIZE,_ptr__B_0); -._L_29_loopE:; +L(_L_29_loopE):; BROAD1282(alpha,XMM2); VLDU1282(0*SIZE(_ptr__C_0),XMM3); VMA21282(XMM2,XMM0,XMM3,XMM0); @@ -1267,11 +1267,11 @@ VMA21282(XMM2,XMM1,XMM4,XMM1); VSTU1282(XMM4,0*SIZE(_ptr__C_0,LDC,1)); ADDQ1280($2*SIZE,_ptr__C_0); ADDQ1280($2*SIZE,_ptr__C_1); -._L_26_loopE:; +L(_L_26_loopE):; TESTQ1280($1,i); -JLE ._L_30_loopE; +JLE L(_L_30_loopE); ALIGN_4; -._L_30_bodyB:; +L(_L_30_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1281(XMM0,XMM0,XMM0); VXOR1281(XMM1,XMM1,XMM1); @@ -1281,9 +1281,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_31_loopE; +JLE L(_L_31_loopE); ALIGN_4; -._L_31_bodyB:; +L(_L_31_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1281(0*SIZE(_ptr__B_0),XMM15); VLD1281(0*SIZE(_ptr__A_0),XMM14); @@ -1310,15 +1310,15 @@ BROAD1281(7*SIZE(_ptr__B_0),XMM15); VMA1281(XMM14,XMM15,XMM1,XMM1); ADDQ1280($4*SIZE,_ptr__A_0); ADDQ1280($8*SIZE,_ptr__B_0); -._L_31_bodyE:; +L(_L_31_bodyE):; DECQ1280(k); -JG ._L_31_bodyB; +JG L(_L_31_bodyB); ALIGN_4; -._L_31_loopE:; +L(_L_31_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_32_loopE; +JLE L(_L_32_loopE); ALIGN_4; -._L_32_bodyB:; +L(_L_32_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1281(0*SIZE(_ptr__B_0),XMM15); VLD1281(0*SIZE(_ptr__A_0),XMM14); @@ -1333,11 +1333,11 @@ BROAD1281(3*SIZE(_ptr__B_0),XMM15); VMA1281(XMM14,XMM15,XMM1,XMM1); ADDQ1280($2*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_32_loopE:; +L(_L_32_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_33_loopE; +JLE L(_L_33_loopE); ALIGN_4; -._L_33_bodyB:; +L(_L_33_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1281(0*SIZE(_ptr__B_0),XMM15); VLD1281(0*SIZE(_ptr__A_0),XMM14); @@ -1346,7 +1346,7 @@ BROAD1281(1*SIZE(_ptr__B_0),XMM15); VMA1281(XMM14,XMM15,XMM1,XMM1); ADDQ1280($1*SIZE,_ptr__A_0); ADDQ1280($2*SIZE,_ptr__B_0); -._L_33_loopE:; +L(_L_33_loopE):; BROAD1281(alpha,XMM2); VLDU1281(0*SIZE(_ptr__C_0),XMM3); VMA21281(XMM2,XMM0,XMM3,XMM0); @@ -1356,18 +1356,18 @@ VMA21281(XMM2,XMM1,XMM4,XMM1); VSTU1281(XMM4,0*SIZE(_ptr__C_0,LDC,1)); ADDQ1280($1*SIZE,_ptr__C_0); ADDQ1280($1*SIZE,_ptr__C_1); -._L_30_loopE:; +L(_L_30_loopE):; MOVQ1280(LDC,%rax); SALQ1280($1,%rax); ADDQ1280(%rax,_ptr_C); MOVQ1280(_bk_l,%rax); SALQ1280($4,%rax); ADDQ1280(%rax,_ptr_B); -._L_17_loopE:; +L(_L_17_loopE):; TESTQ1280($1,_bk_j); -JLE ._L_34_loopE; +JLE L(_L_34_loopE); ALIGN_4; -._L_34_bodyB:; +L(_L_34_bodyB):; MOVQ1280(_ptr_A,_ptr__A_0); MOVQ1280(_ptr_C,_ptr__C_0); MOVQ1280(_bk_l,%rax); @@ -1375,8 +1375,8 @@ SALQ1280($3,%rax); ADDQ1280(%rax,_pre_B); MOVQ1280(_bk_i,i); CMPQ1280($6,i); -JL ._L_35_loopE; -._L_35_bodyB:; +JL L(_L_35_loopE); +L(_L_35_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1282(XMM0,XMM0,XMM0); VXOR1282(XMM1,XMM1,XMM1); @@ -1387,9 +1387,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_36_loopE; +JLE L(_L_36_loopE); ALIGN_4; -._L_36_bodyB:; +L(_L_36_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM12); @@ -1424,15 +1424,15 @@ VLD1282(22*SIZE(_ptr__A_0),XMM14); VMA1282(XMM14,XMM15,XMM2,XMM2); ADDQ1280($24*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_36_bodyE:; +L(_L_36_bodyE):; DECQ1280(k); -JG ._L_36_bodyB; +JG L(_L_36_bodyB); ALIGN_4; -._L_36_loopE:; +L(_L_36_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_37_loopE; +JLE L(_L_37_loopE); ALIGN_4; -._L_37_bodyB:; +L(_L_37_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM12); @@ -1451,11 +1451,11 @@ VLD1282(10*SIZE(_ptr__A_0),XMM14); VMA1282(XMM14,XMM15,XMM2,XMM2); ADDQ1280($12*SIZE,_ptr__A_0); ADDQ1280($2*SIZE,_ptr__B_0); -._L_37_loopE:; +L(_L_37_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_38_loopE; +JLE L(_L_38_loopE); ALIGN_4; -._L_38_bodyB:; +L(_L_38_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM12); @@ -1466,7 +1466,7 @@ VLD1282(4*SIZE(_ptr__A_0),XMM14); VMA1282(XMM14,XMM15,XMM2,XMM2); ADDQ1280($6*SIZE,_ptr__A_0); ADDQ1280($1*SIZE,_ptr__B_0); -._L_38_loopE:; +L(_L_38_loopE):; BROAD1282(alpha,XMM3); VLDU1282(0*SIZE(_ptr__C_0),XMM4); VMA21282(XMM3,XMM0,XMM4,XMM0); @@ -1479,15 +1479,15 @@ VMA21282(XMM3,XMM2,XMM6,XMM2); VSTU1282(XMM6,4*SIZE(_ptr__C_0)); ADDQ1280($6*SIZE,_ptr__C_0); ADDQ1280($6*SIZE,_ptr__C_1); -._L_35_bodyE:; +L(_L_35_bodyE):; SUBQ1280($6,i); -JG ._L_35_bodyB; +JG L(_L_35_bodyB); ALIGN_4; -._L_35_loopE:; +L(_L_35_loopE):; TESTQ1280($4,i); -JLE ._L_39_loopE; +JLE L(_L_39_loopE); ALIGN_4; -._L_39_bodyB:; +L(_L_39_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1282(XMM0,XMM0,XMM0); VXOR1282(XMM1,XMM1,XMM1); @@ -1497,9 +1497,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_40_loopE; +JLE L(_L_40_loopE); ALIGN_4; -._L_40_bodyB:; +L(_L_40_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM13); @@ -1526,15 +1526,15 @@ VLD1282(14*SIZE(_ptr__A_0),XMM14); VMA1282(XMM14,XMM15,XMM1,XMM1); ADDQ1280($16*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_40_bodyE:; +L(_L_40_bodyE):; DECQ1280(k); -JG ._L_40_bodyB; +JG L(_L_40_bodyB); ALIGN_4; -._L_40_loopE:; +L(_L_40_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_41_loopE; +JLE L(_L_41_loopE); ALIGN_4; -._L_41_bodyB:; +L(_L_41_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM13); @@ -1549,11 +1549,11 @@ VLD1282(6*SIZE(_ptr__A_0),XMM14); VMA1282(XMM14,XMM15,XMM1,XMM1); ADDQ1280($8*SIZE,_ptr__A_0); ADDQ1280($2*SIZE,_ptr__B_0); -._L_41_loopE:; +L(_L_41_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_42_loopE; +JLE L(_L_42_loopE); ALIGN_4; -._L_42_bodyB:; +L(_L_42_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM13); @@ -1562,7 +1562,7 @@ VLD1282(2*SIZE(_ptr__A_0),XMM14); VMA1282(XMM14,XMM15,XMM1,XMM1); ADDQ1280($4*SIZE,_ptr__A_0); ADDQ1280($1*SIZE,_ptr__B_0); -._L_42_loopE:; +L(_L_42_loopE):; BROAD1282(alpha,XMM2); VLDU1282(0*SIZE(_ptr__C_0),XMM3); VMA21282(XMM2,XMM0,XMM3,XMM0); @@ -1572,11 +1572,11 @@ VMA21282(XMM2,XMM1,XMM4,XMM1); VSTU1282(XMM4,2*SIZE(_ptr__C_0)); ADDQ1280($4*SIZE,_ptr__C_0); ADDQ1280($4*SIZE,_ptr__C_1); -._L_39_loopE:; +L(_L_39_loopE):; TESTQ1280($2,i); -JLE ._L_43_loopE; +JLE L(_L_43_loopE); ALIGN_4; -._L_43_bodyB:; +L(_L_43_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1282(XMM0,XMM0,XMM0); PREFETCHN1280(3*SIZE(_ptr__C_0),N); @@ -1585,9 +1585,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_44_loopE; +JLE L(_L_44_loopE); ALIGN_4; -._L_44_bodyB:; +L(_L_44_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM14); @@ -1606,15 +1606,15 @@ VLD1282(6*SIZE(_ptr__A_0),XMM14); VMA1282(XMM14,XMM15,XMM0,XMM0); ADDQ1280($8*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_44_bodyE:; +L(_L_44_bodyE):; DECQ1280(k); -JG ._L_44_bodyB; +JG L(_L_44_bodyB); ALIGN_4; -._L_44_loopE:; +L(_L_44_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_45_loopE; +JLE L(_L_45_loopE); ALIGN_4; -._L_45_bodyB:; +L(_L_45_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM14); @@ -1625,29 +1625,29 @@ VLD1282(2*SIZE(_ptr__A_0),XMM14); VMA1282(XMM14,XMM15,XMM0,XMM0); ADDQ1280($4*SIZE,_ptr__A_0); ADDQ1280($2*SIZE,_ptr__B_0); -._L_45_loopE:; +L(_L_45_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_46_loopE; +JLE L(_L_46_loopE); ALIGN_4; -._L_46_bodyB:; +L(_L_46_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM14); VMA1282(XMM14,XMM15,XMM0,XMM0); ADDQ1280($2*SIZE,_ptr__A_0); ADDQ1280($1*SIZE,_ptr__B_0); -._L_46_loopE:; +L(_L_46_loopE):; BROAD1282(alpha,XMM1); VLDU1282(0*SIZE(_ptr__C_0),XMM2); VMA21282(XMM1,XMM0,XMM2,XMM0); VSTU1282(XMM2,0*SIZE(_ptr__C_0)); ADDQ1280($2*SIZE,_ptr__C_0); ADDQ1280($2*SIZE,_ptr__C_1); -._L_43_loopE:; +L(_L_43_loopE):; TESTQ1280($1,i); -JLE ._L_47_loopE; +JLE L(_L_47_loopE); ALIGN_4; -._L_47_bodyB:; +L(_L_47_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1281(XMM0,XMM0,XMM0); PREFETCHN1280(3*SIZE(_ptr__C_0),N); @@ -1656,9 +1656,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_48_loopE; +JLE L(_L_48_loopE); ALIGN_4; -._L_48_bodyB:; +L(_L_48_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1281(0*SIZE(_ptr__B_0),XMM15); VLD1281(0*SIZE(_ptr__A_0),XMM14); @@ -1677,15 +1677,15 @@ VLD1281(3*SIZE(_ptr__A_0),XMM14); VMA1281(XMM14,XMM15,XMM0,XMM0); ADDQ1280($4*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_48_bodyE:; +L(_L_48_bodyE):; DECQ1280(k); -JG ._L_48_bodyB; +JG L(_L_48_bodyB); ALIGN_4; -._L_48_loopE:; +L(_L_48_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_49_loopE; +JLE L(_L_49_loopE); ALIGN_4; -._L_49_bodyB:; +L(_L_49_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1281(0*SIZE(_ptr__B_0),XMM15); VLD1281(0*SIZE(_ptr__A_0),XMM14); @@ -1696,31 +1696,31 @@ VLD1281(1*SIZE(_ptr__A_0),XMM14); VMA1281(XMM14,XMM15,XMM0,XMM0); ADDQ1280($2*SIZE,_ptr__A_0); ADDQ1280($2*SIZE,_ptr__B_0); -._L_49_loopE:; +L(_L_49_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_50_loopE; +JLE L(_L_50_loopE); ALIGN_4; -._L_50_bodyB:; +L(_L_50_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1281(0*SIZE(_ptr__B_0),XMM15); VLD1281(0*SIZE(_ptr__A_0),XMM14); VMA1281(XMM14,XMM15,XMM0,XMM0); ADDQ1280($1*SIZE,_ptr__A_0); ADDQ1280($1*SIZE,_ptr__B_0); -._L_50_loopE:; +L(_L_50_loopE):; BROAD1281(alpha,XMM1); VLDU1281(0*SIZE(_ptr__C_0),XMM2); VMA21281(XMM1,XMM0,XMM2,XMM0); VSTU1281(XMM2,0*SIZE(_ptr__C_0)); ADDQ1280($1*SIZE,_ptr__C_0); ADDQ1280($1*SIZE,_ptr__C_1); -._L_47_loopE:; +L(_L_47_loopE):; MOVQ1280(LDC,%rax); ADDQ1280(%rax,_ptr_C); MOVQ1280(_bk_l,%rax); SALQ1280($3,%rax); ADDQ1280(%rax,_ptr_B); -._L_34_loopE:; +L(_L_34_loopE):; vzeroupper movq 0(%rsp), %rbx; movq 8(%rsp), %rbp; diff --git a/kernel/x86_64/dgemm_kernel_8x2_bulldozer.S b/kernel/x86_64/dgemm_kernel_8x2_bulldozer.S index c353a59136..d534f6fd2e 100644 --- a/kernel/x86_64/dgemm_kernel_8x2_bulldozer.S +++ b/kernel/x86_64/dgemm_kernel_8x2_bulldozer.S @@ -922,13 +922,13 @@ STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -950,10 +950,10 @@ movq Ndiv6, J cmpq $0, J - je .L2_0 + je L(2_0) ALIGN_4 -.L6_01: +L(6_01): // copy to sub buffer movq K, %rax salq $1,%rax // K * 2 @@ -962,10 +962,10 @@ leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L6_02a + jz L(6_02a) ALIGN_4 -.L6_02: +L(6_02): prefetcht0 512(BO1) prefetcht0 512(BO2) prefetchw 512(BO) @@ -989,16 +989,16 @@ addq $8*SIZE,BO2 addq $12*SIZE,BO decq %rax - jnz .L6_02 + jnz L(6_02) -.L6_02a: +L(6_02a): movq K, %rax andq $3, %rax // K % 4 - jz .L6_02c + jz L(6_02c) ALIGN_4 -.L6_02b: +L(6_02b): vmovups (BO1), %xmm0 vmovsd (BO2), %xmm1 @@ -1008,9 +1008,9 @@ addq $2*SIZE,BO2 addq $3*SIZE,BO decq %rax - jnz .L6_02b + jnz L(6_02b) -.L6_02c: +L(6_02c): movq K, %rax salq $1,%rax // K * 2 @@ -1019,11 +1019,11 @@ leaq BUFFER2, BO // second buffer to BO movq K, %rax sarq $2, %rax // k / 4 - jz .L6_03a + jz L(6_03a) ALIGN_4 -.L6_03: +L(6_03): prefetcht0 512(BO2) prefetchw 512(BO) @@ -1047,17 +1047,17 @@ addq $8*SIZE,BO2 addq $12*SIZE,BO decq %rax - jnz .L6_03 + jnz L(6_03) -.L6_03a: +L(6_03a): movq K, %rax andq $3, %rax // K % 4 - jz .L6_03c + jz L(6_03c) ALIGN_4 -.L6_03b: +L(6_03b): vmovsd 1*SIZE(BO1), %xmm0 vmovups (BO2), %xmm1 @@ -1067,14 +1067,14 @@ addq $2*SIZE,BO2 addq $3*SIZE,BO decq %rax - jnz .L6_03b + jnz L(6_03b) -.L6_03c: +L(6_03c): movq BO2, B // next offset of B -.L6_10: +L(6_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -1085,11 +1085,11 @@ movq M, I sarq $3, I // i = (m >> 3) - je .L6_20 + je L(6_20) ALIGN_4 -.L6_11: +L(6_11): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1098,7 +1098,7 @@ movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L6_16 + je L(6_16) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1109,7 +1109,7 @@ negq %rax ALIGN_4 -.L6_12: +L(6_12): prefetcht0 B_PR1(BO,BI,8) KERNEL8x3_1(xxx) @@ -1124,7 +1124,7 @@ KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L6_16 + je L(6_16) prefetcht0 B_PR1(BO,BI,8) KERNEL8x3_1(xxx) @@ -1139,16 +1139,16 @@ KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L6_16 + je L(6_16) - jmp .L6_12 + jmp L(6_12) ALIGN_4 -.L6_16: +L(6_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_19 + je L(6_19) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1160,16 +1160,16 @@ negq %rax ALIGN_4 -.L6_17: +L(6_17): KERNEL8x3_SUB(xxx) addq $3, BI addq $8, %rax - jl .L6_17 + jl L(6_17) ALIGN_4 -.L6_19: +L(6_19): vmovddup ALPHA, %xmm0 @@ -1207,24 +1207,24 @@ addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L6_11 + jg L(6_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L6_20: +L(6_20): // Test rest of M testq $7, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N testq $4, M - jz .L6_30 + jz L(6_30) ALIGN_4 -.L6_21: +L(6_21): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1233,7 +1233,7 @@ movq K, %rax andq $-8, %rax - je .L6_26 + je L(6_26) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1244,7 +1244,7 @@ negq %rax ALIGN_4 -.L6_22: +L(6_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x3_1(xxx) @@ -1259,7 +1259,7 @@ KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L6_26 + je L(6_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x3_1(xxx) @@ -1274,16 +1274,16 @@ KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L6_26 + je L(6_26) - jmp .L6_22 + jmp L(6_22) ALIGN_4 -.L6_26: +L(6_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_29 + je L(6_29) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1295,16 +1295,16 @@ negq %rax ALIGN_4 -.L6_27: +L(6_27): KERNEL4x3_SUB(xxx) addq $3, BI addq $4, %rax - jl .L6_27 + jl L(6_27) ALIGN_4 -.L6_29: +L(6_29): vmovddup ALPHA, %xmm0 @@ -1333,13 +1333,13 @@ ALIGN_4 -.L6_30: +L(6_30): testq $2, M - jz .L6_40 + jz L(6_40) ALIGN_4 -.L6_31: +L(6_31): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1349,7 +1349,7 @@ movq K, %rax andq $-8, %rax - je .L6_36 + je L(6_36) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1360,7 +1360,7 @@ negq %rax ALIGN_4 -.L6_32: +L(6_32): prefetcht0 B_PR1(BO,BI,8) KERNEL2x3_1(xxx) @@ -1375,7 +1375,7 @@ KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L6_36 + je L(6_36) prefetcht0 B_PR1(BO,BI,8) KERNEL2x3_1(xxx) @@ -1390,16 +1390,16 @@ KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L6_36 + je L(6_36) - jmp .L6_32 + jmp L(6_32) ALIGN_4 -.L6_36: +L(6_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_39 + je L(6_39) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1411,16 +1411,16 @@ negq %rax ALIGN_4 -.L6_37: +L(6_37): KERNEL2x3_SUB(xxx) addq $3, BI addq $2, %rax - jl .L6_37 + jl L(6_37) ALIGN_4 -.L6_39: +L(6_39): vmovddup ALPHA, %xmm0 @@ -1438,13 +1438,13 @@ addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L6_40: +L(6_40): testq $1, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N ALIGN_4 -.L6_41: +L(6_41): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1453,7 +1453,7 @@ movq K, %rax andq $-8, %rax - je .L6_46 + je L(6_46) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1463,7 +1463,7 @@ negq %rax ALIGN_4 -.L6_42: +L(6_42): prefetcht0 B_PR1(BO,BI,8) KERNEL1x3_1(xxx) @@ -1478,7 +1478,7 @@ KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L6_46 + je L(6_46) prefetcht0 B_PR1(BO,BI,8) KERNEL1x3_1(xxx) @@ -1493,16 +1493,16 @@ KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L6_46 + je L(6_46) - jmp .L6_42 + jmp L(6_42) ALIGN_4 -.L6_46: +L(6_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_49 + je L(6_49) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1513,16 +1513,16 @@ negq %rax ALIGN_4 -.L6_47: +L(6_47): KERNEL1x3_SUB(xxx) addq $3, BI addq $1, %rax - jl .L6_47 + jl L(6_47) ALIGN_4 -.L6_49: +L(6_49): vmovddup ALPHA, %xmm0 @@ -1544,7 +1544,7 @@ /***************************************************************************************************************/ -.L7_10: +L(7_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -1555,10 +1555,10 @@ movq M, I sarq $3, I // i = (m >> 3) - je .L7_20 + je L(7_20) ALIGN_4 -.L7_11: +L(7_11): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -1569,7 +1569,7 @@ andq $-8, %rax - je .L7_16 + je L(7_16) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1581,7 +1581,7 @@ ALIGN_4 -.L7_12: +L(7_12): prefetcht0 B_PR1(BO,BI,8) KERNEL8x3_1(xxx) @@ -1596,7 +1596,7 @@ KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L7_16 + je L(7_16) prefetcht0 B_PR1(BO,BI,8) KERNEL8x3_1(xxx) @@ -1611,16 +1611,16 @@ KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L7_16 + je L(7_16) - jmp .L7_12 + jmp L(7_12) ALIGN_4 -.L7_16: +L(7_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_19 + je L(7_19) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1632,16 +1632,16 @@ negq %rax ALIGN_4 -.L7_17: +L(7_17): KERNEL8x3_SUB(xxx) addq $3, BI addq $8, %rax - jl .L7_17 + jl L(7_17) ALIGN_4 -.L7_19: +L(7_19): vmovddup ALPHA, %xmm0 @@ -1681,21 +1681,21 @@ addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L7_11 + jg L(7_11) ALIGN_4 -.L7_20: +L(7_20): // Test rest of M testq $7, M - jz .L7_60 // to next 6 lines of N + jz L(7_60) // to next 6 lines of N testq $4, M - jz .L7_30 + jz L(7_30) ALIGN_4 -.L7_21: +L(7_21): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -1705,7 +1705,7 @@ movq K, %rax andq $-8, %rax - je .L7_26 + je L(7_26) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1716,7 +1716,7 @@ negq %rax ALIGN_4 -.L7_22: +L(7_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x3_1(xxx) @@ -1731,7 +1731,7 @@ KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L7_26 + je L(7_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x3_1(xxx) @@ -1746,16 +1746,16 @@ KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L7_26 + je L(7_26) - jmp .L7_22 + jmp L(7_22) ALIGN_4 -.L7_26: +L(7_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_29 + je L(7_29) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1767,16 +1767,16 @@ negq %rax ALIGN_4 -.L7_27: +L(7_27): KERNEL4x3_SUB(xxx) addq $3, BI addq $4, %rax - jl .L7_27 + jl L(7_27) ALIGN_4 -.L7_29: +L(7_29): vmovddup ALPHA, %xmm0 @@ -1805,13 +1805,13 @@ ALIGN_4 -.L7_30: +L(7_30): testq $2, M - jz .L7_40 + jz L(7_40) ALIGN_4 -.L7_31: +L(7_31): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -1820,7 +1820,7 @@ movq K, %rax andq $-8, %rax - je .L7_36 + je L(7_36) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1831,7 +1831,7 @@ negq %rax ALIGN_4 -.L7_32: +L(7_32): prefetcht0 B_PR1(BO,BI,8) KERNEL2x3_1(xxx) @@ -1846,7 +1846,7 @@ KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L7_36 + je L(7_36) prefetcht0 B_PR1(BO,BI,8) KERNEL2x3_1(xxx) @@ -1861,16 +1861,16 @@ KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L7_36 + je L(7_36) - jmp .L7_32 + jmp L(7_32) ALIGN_4 -.L7_36: +L(7_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_39 + je L(7_39) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1882,16 +1882,16 @@ negq %rax ALIGN_4 -.L7_37: +L(7_37): KERNEL2x3_SUB(xxx) addq $3, BI addq $2, %rax - jl .L7_37 + jl L(7_37) ALIGN_4 -.L7_39: +L(7_39): vmovddup ALPHA, %xmm0 @@ -1913,13 +1913,13 @@ -.L7_40: +L(7_40): testq $1, M - jz .L7_60 // to next 6 lines of N + jz L(7_60) // to next 6 lines of N ALIGN_4 -.L7_41: +L(7_41): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -1929,7 +1929,7 @@ andq $-8, %rax - je .L7_46 + je L(7_46) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1939,7 +1939,7 @@ negq %rax ALIGN_4 -.L7_42: +L(7_42): prefetcht0 B_PR1(BO,BI,8) KERNEL1x3_1(xxx) @@ -1954,7 +1954,7 @@ KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L7_46 + je L(7_46) prefetcht0 B_PR1(BO,BI,8) KERNEL1x3_1(xxx) @@ -1969,16 +1969,16 @@ KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L7_46 + je L(7_46) - jmp .L7_42 + jmp L(7_42) ALIGN_4 -.L7_46: +L(7_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_49 + je L(7_49) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1989,16 +1989,16 @@ negq %rax ALIGN_4 -.L7_47: +L(7_47): KERNEL1x3_SUB(xxx) addq $3, BI addq $1, %rax - jl .L7_47 + jl L(7_47) ALIGN_4 -.L7_49: +L(7_49): vmovddup ALPHA, %xmm0 @@ -2015,15 +2015,15 @@ addq $1 * SIZE, CO1 # coffset += 1 -.L7_60: +L(7_60): decq J // j -- - jg .L6_01 + jg L(6_01) -.L2_0: +L(2_0): cmpq $0, Nmod6 // N % 6 == 0 - je .L999 + je L(999) /************************************************************************************************ * Loop for Nmod6 / 2 > 0 @@ -2031,30 +2031,30 @@ movq Nmod6, J sarq $1, J // j = j / 2 - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -2064,11 +2064,11 @@ movq M, I sarq $3, I // i = (m >> 3) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2078,7 +2078,7 @@ movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2089,7 +2089,7 @@ negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1(xxx) @@ -2103,7 +2103,7 @@ KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1(xxx) @@ -2117,16 +2117,16 @@ KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2138,16 +2138,16 @@ negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL8x2_SUB(xxx) addq $2, BI addq $8, %rax - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vmovddup ALPHA, %xmm0 @@ -2174,24 +2174,24 @@ addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $7, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2200,7 +2200,7 @@ movq K, %rax andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2211,7 +2211,7 @@ negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1(xxx) @@ -2225,7 +2225,7 @@ KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1(xxx) @@ -2239,16 +2239,16 @@ KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2260,16 +2260,16 @@ negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB(xxx) addq $2, BI addq $4, %rax - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): vmovddup ALPHA, %xmm0 @@ -2289,13 +2289,13 @@ ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2304,7 +2304,7 @@ movq K, %rax andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2315,7 +2315,7 @@ negq %rax ALIGN_4 -.L2_32: +L(2_32): prefetcht0 B_PR1(BO,BI,8) KERNEL2x2_1(xxx) @@ -2329,7 +2329,7 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) prefetcht0 B_PR1(BO,BI,8) KERNEL2x2_1(xxx) @@ -2343,16 +2343,16 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2364,16 +2364,16 @@ negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB(xxx) addq $2, BI addq $2, %rax - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): vmovddup ALPHA, %xmm0 @@ -2387,13 +2387,13 @@ ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2402,7 +2402,7 @@ movq K, %rax andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2412,7 +2412,7 @@ negq %rax ALIGN_4 -.L2_42: +L(2_42): prefetcht0 B_PR1(BO,BI,8) KERNEL1x2_1(xxx) @@ -2426,7 +2426,7 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) prefetcht0 B_PR1(BO,BI,8) KERNEL1x2_1(xxx) @@ -2440,16 +2440,16 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2460,16 +2460,16 @@ negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) addq $2, BI addq $1, %rax - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vmovddup ALPHA, %xmm0 @@ -2485,14 +2485,14 @@ -.L2_60: +L(2_60): decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -2500,30 +2500,30 @@ movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -2532,11 +2532,11 @@ movq M, I sarq $3, I // i = (m >> 3) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -2545,7 +2545,7 @@ movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -2555,7 +2555,7 @@ negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1(xxx) @@ -2568,7 +2568,7 @@ KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_16 + je L(1_16) prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1(xxx) @@ -2581,16 +2581,16 @@ KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -2601,16 +2601,16 @@ negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL8x1_SUB(xxx) addq $1, BI addq $8, %rax - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vmovddup ALPHA, %xmm0 @@ -2626,24 +2626,24 @@ addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $7, M - jz .L999 + jz L(999) testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -2653,7 +2653,7 @@ movq K, %rax andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -2663,7 +2663,7 @@ negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1(xxx) @@ -2676,7 +2676,7 @@ KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1(xxx) @@ -2689,16 +2689,16 @@ KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -2709,16 +2709,16 @@ negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB(xxx) addq $1, BI addq $4, %rax - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): vmovddup ALPHA, %xmm0 @@ -2732,13 +2732,13 @@ ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -2748,7 +2748,7 @@ movq K, %rax andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -2758,7 +2758,7 @@ negq %rax ALIGN_4 -.L1_32: +L(1_32): prefetcht0 B_PR1(BO,BI,8) KERNEL2x1_1(xxx) @@ -2771,7 +2771,7 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) KERNEL2x1_1(xxx) KERNEL2x1_2(xxx) @@ -2783,16 +2783,16 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -2803,16 +2803,16 @@ negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB(xxx) addq $1, BI addq $2, %rax - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): vmovddup ALPHA, %xmm0 @@ -2824,13 +2824,13 @@ ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -2839,7 +2839,7 @@ movq K, %rax andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, 8), AO @@ -2848,7 +2848,7 @@ negq %rax ALIGN_4 -.L1_42: +L(1_42): prefetcht0 B_PR1(BO,BI,8) KERNEL1x1_1(xxx) @@ -2861,7 +2861,7 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) prefetcht0 B_PR1(BO,BI,8) KERNEL1x1_1(xxx) @@ -2874,16 +2874,16 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -2893,16 +2893,16 @@ negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) addq $1, BI addq $1, %rax - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vmovddup ALPHA, %xmm0 @@ -2914,7 +2914,7 @@ ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp @@ -3002,13 +3002,13 @@ STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -3037,32 +3037,32 @@ movq Ndiv6, J cmpq $0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_0: +L(2_0): -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -3076,11 +3076,11 @@ movq M, I sarq $3, I // i = (m >> 3) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3118,7 +3118,7 @@ andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3129,7 +3129,7 @@ negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1(xxx) @@ -3143,7 +3143,7 @@ KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1(xxx) @@ -3157,12 +3157,12 @@ KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3170,7 +3170,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3182,16 +3182,16 @@ negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL8x2_SUB(xxx) addq $2, BI addq $8, %rax - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vmovddup ALPHA, %xmm0 @@ -3249,24 +3249,24 @@ addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $7, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3303,7 +3303,7 @@ andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3314,7 +3314,7 @@ negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1(xxx) @@ -3328,7 +3328,7 @@ KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1(xxx) @@ -3342,12 +3342,12 @@ KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -3355,7 +3355,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3367,16 +3367,16 @@ negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB(xxx) addq $2, BI addq $4, %rax - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): vmovddup ALPHA, %xmm0 @@ -3424,13 +3424,13 @@ ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3468,7 +3468,7 @@ andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3479,7 +3479,7 @@ negq %rax ALIGN_4 -.L2_32: +L(2_32): prefetcht0 B_PR1(BO,BI,8) KERNEL2x2_1(xxx) @@ -3493,7 +3493,7 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) prefetcht0 B_PR1(BO,BI,8) KERNEL2x2_1(xxx) @@ -3507,12 +3507,12 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -3520,7 +3520,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3532,16 +3532,16 @@ negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB(xxx) addq $2, BI addq $2, %rax - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): vmovddup ALPHA, %xmm0 @@ -3579,13 +3579,13 @@ ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3620,7 +3620,7 @@ #endif andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3630,7 +3630,7 @@ negq %rax ALIGN_4 -.L2_42: +L(2_42): prefetcht0 B_PR1(BO,BI,8) KERNEL1x2_1(xxx) @@ -3644,7 +3644,7 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) prefetcht0 B_PR1(BO,BI,8) KERNEL1x2_1(xxx) @@ -3658,12 +3658,12 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -3671,7 +3671,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3682,16 +3682,16 @@ negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) addq $2, BI addq $1, %rax - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vmovddup ALPHA, %xmm0 @@ -3730,17 +3730,17 @@ -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -3748,30 +3748,30 @@ movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -3785,11 +3785,11 @@ movq M, I sarq $3, I // i = (m >> 3) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3825,7 +3825,7 @@ #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -3835,7 +3835,7 @@ negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1(xxx) @@ -3848,7 +3848,7 @@ KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_16 + je L(1_16) prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1(xxx) @@ -3861,12 +3861,12 @@ KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3874,7 +3874,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -3885,16 +3885,16 @@ negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL8x1_SUB(xxx) addq $1, BI addq $8, %rax - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vmovddup ALPHA, %xmm0 @@ -3934,24 +3934,24 @@ #endif addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $7, M - jz .L999 + jz L(999) testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3988,7 +3988,7 @@ andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -3998,7 +3998,7 @@ negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1(xxx) @@ -4011,7 +4011,7 @@ KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1(xxx) @@ -4024,12 +4024,12 @@ KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4037,7 +4037,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -4048,16 +4048,16 @@ negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB(xxx) addq $1, BI addq $4, %rax - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): vmovddup ALPHA, %xmm0 @@ -4094,13 +4094,13 @@ ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4137,7 +4137,7 @@ andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -4147,7 +4147,7 @@ negq %rax ALIGN_4 -.L1_32: +L(1_32): prefetcht0 B_PR1(BO,BI,8) KERNEL2x1_1(xxx) @@ -4160,7 +4160,7 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) KERNEL2x1_1(xxx) KERNEL2x1_2(xxx) @@ -4172,12 +4172,12 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -4185,7 +4185,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -4196,16 +4196,16 @@ negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB(xxx) addq $1, BI addq $2, %rax - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): vmovddup ALPHA, %xmm0 @@ -4238,13 +4238,13 @@ ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4279,7 +4279,7 @@ #endif andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, 8), AO @@ -4288,7 +4288,7 @@ negq %rax ALIGN_4 -.L1_42: +L(1_42): prefetcht0 B_PR1(BO,BI,8) KERNEL1x1_1(xxx) @@ -4301,7 +4301,7 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) prefetcht0 B_PR1(BO,BI,8) KERNEL1x1_1(xxx) @@ -4314,12 +4314,12 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -4327,7 +4327,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -4337,16 +4337,16 @@ negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) addq $1, BI addq $1, %rax - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vmovddup ALPHA, %xmm0 @@ -4379,7 +4379,7 @@ ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/dgemm_kernel_8x2_piledriver.S b/kernel/x86_64/dgemm_kernel_8x2_piledriver.S index 48eb1bcbe1..b403895ef7 100644 --- a/kernel/x86_64/dgemm_kernel_8x2_piledriver.S +++ b/kernel/x86_64/dgemm_kernel_8x2_piledriver.S @@ -1105,13 +1105,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -1130,10 +1130,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv6, J cmpq $0, J - je .L2_0 + je L(2_0) ALIGN_4 -.L6_01: +L(6_01): // copy to sub buffer movq K, %rax salq $1,%rax // K * 2 @@ -1142,10 +1142,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L6_02a + jz L(6_02a) ALIGN_4 -.L6_02: +L(6_02): prefetcht0 B_PR1(BO1) prefetcht0 B_PR1(BO2) prefetchw B_PR1(BO) @@ -1169,16 +1169,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 8*SIZE,BO2 addq $ 12*SIZE,BO decq %rax - jnz .L6_02 + jnz L(6_02) -.L6_02a: +L(6_02a): movq K, %rax andq $3, %rax // K % 4 - jz .L6_02c + jz L(6_02c) ALIGN_4 -.L6_02b: +L(6_02b): vmovups (BO1), %xmm0 vmovsd (BO2), %xmm1 @@ -1188,9 +1188,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 2*SIZE,BO2 addq $ 3*SIZE,BO decq %rax - jnz .L6_02b + jnz L(6_02b) -.L6_02c: +L(6_02c): movq K, %rax salq $1,%rax // K * 2 @@ -1199,11 +1199,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. leaq BUFFER2, BO // second buffer to BO movq K, %rax sarq $2, %rax // k / 4 - jz .L6_03a + jz L(6_03a) ALIGN_4 -.L6_03: +L(6_03): prefetcht0 B_PR1(BO2) prefetchw B_PR1(BO) @@ -1227,17 +1227,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 8*SIZE,BO2 addq $ 12*SIZE,BO decq %rax - jnz .L6_03 + jnz L(6_03) -.L6_03a: +L(6_03a): movq K, %rax andq $3, %rax // K % 4 - jz .L6_03c + jz L(6_03c) ALIGN_4 -.L6_03b: +L(6_03b): vmovsd 1*SIZE(BO1), %xmm0 vmovups (BO2), %xmm1 @@ -1247,14 +1247,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 2*SIZE,BO2 addq $ 3*SIZE,BO decq %rax - jnz .L6_03b + jnz L(6_03b) -.L6_03c: +L(6_03c): movq BO2, B // next offset of B -.L6_10: +L(6_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -1265,18 +1265,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $3, I // i = (m >> 3) - je .L6_20 + je L(6_20) ALIGN_4 -.L6_11: +L(6_11): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO movq K, %rax sarq $3, %rax // K / 8 cmpq $3, %rax - jl .L6_13 + jl L(6_13) prefetcht0 B_PR1(BO) prefetcht0 B_PR1+64(BO) @@ -1294,7 +1294,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_5 -.L6_12: +L(6_12): prefetcht0 B_PR1-24(BO) prefetcht0 B_PR1+40(BO) @@ -1309,9 +1309,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_M8 dec %rax - jne .L6_12 + jne L(6_12) -.L6_12_E: +L(6_12_E): prefetcht0 B_PR1(BO) prefetcht0 B_PR1+64(BO) @@ -1324,12 +1324,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_M7 KERNEL8x3_E - jmp .L6_16 + jmp L(6_16) -.L6_13: +L(6_13): test $2, %rax - jz .L6_14 + jz L(6_14) KERNEL8x3_INIT KERNEL8x3_M2 @@ -1349,13 +1349,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_M7 KERNEL8x3_E - jmp .L6_16 + jmp L(6_16) -.L6_14: +L(6_14): test $1, %rax - jz .L6_15 + jz L(6_15) KERNEL8x3_INIT KERNEL8x3_M2 @@ -1367,50 +1367,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_E - jmp .L6_16 + jmp L(6_16) -.L6_15: +L(6_15): INIT8x3 -.L6_16: +L(6_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_19 + je L(6_19) ALIGN_4 -.L6_17: +L(6_17): KERNEL8x3_SUBN dec %rax - jne .L6_17 + jne L(6_17) ALIGN_4 -.L6_19: +L(6_19): SAVE8x3 decq I # i -- - jg .L6_11 + jg L(6_11) /************************************************************************** * Rest of M ***************************************************************************/ -.L6_20: +L(6_20): // Test rest of M testq $7, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N testq $4, M - jz .L6_30 + jz L(6_30) ALIGN_4 -.L6_21: +L(6_21): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1419,7 +1419,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_26 + je L(6_26) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1430,7 +1430,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_22: +L(6_22): KERNEL4x3_1(xxx) KERNEL4x3_2(xxx) @@ -1442,7 +1442,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L6_26 + je L(6_26) KERNEL4x3_1(xxx) KERNEL4x3_2(xxx) @@ -1454,16 +1454,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L6_26 + je L(6_26) - jmp .L6_22 + jmp L(6_22) ALIGN_4 -.L6_26: +L(6_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_29 + je L(6_29) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1475,16 +1475,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_27: +L(6_27): KERNEL4x3_SUB(xxx) addq $3, BI addq $4, %rax - jl .L6_27 + jl L(6_27) ALIGN_4 -.L6_29: +L(6_29): vmovddup ALPHA, %xmm0 @@ -1513,13 +1513,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L6_30: +L(6_30): testq $2, M - jz .L6_40 + jz L(6_40) ALIGN_4 -.L6_31: +L(6_31): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1529,7 +1529,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_36 + je L(6_36) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1540,7 +1540,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_32: +L(6_32): KERNEL2x3_1(xxx) KERNEL2x3_2(xxx) @@ -1552,7 +1552,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L6_36 + je L(6_36) KERNEL2x3_1(xxx) KERNEL2x3_2(xxx) @@ -1564,16 +1564,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L6_36 + je L(6_36) - jmp .L6_32 + jmp L(6_32) ALIGN_4 -.L6_36: +L(6_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_39 + je L(6_39) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1585,16 +1585,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_37: +L(6_37): KERNEL2x3_SUB(xxx) addq $3, BI addq $2, %rax - jl .L6_37 + jl L(6_37) ALIGN_4 -.L6_39: +L(6_39): vmovddup ALPHA, %xmm0 @@ -1612,13 +1612,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L6_40: +L(6_40): testq $1, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N ALIGN_4 -.L6_41: +L(6_41): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1627,7 +1627,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_46 + je L(6_46) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1637,7 +1637,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_42: +L(6_42): KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -1649,7 +1649,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L6_46 + je L(6_46) KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -1661,16 +1661,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L6_46 + je L(6_46) - jmp .L6_42 + jmp L(6_42) ALIGN_4 -.L6_46: +L(6_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_49 + je L(6_49) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1681,16 +1681,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_47: +L(6_47): KERNEL1x3_SUB(xxx) addq $3, BI addq $1, %rax - jl .L6_47 + jl L(6_47) ALIGN_4 -.L6_49: +L(6_49): vmovddup ALPHA, %xmm0 @@ -1712,7 +1712,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /***************************************************************************************************************/ -.L7_10: +L(7_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -1723,17 +1723,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $3, I // i = (m >> 3) - je .L7_20 + je L(7_20) ALIGN_4 -.L7_11: +L(7_11): leaq BUFFER2, BO // first buffer to BO addq $12 * SIZE, BO movq K, %rax sarq $3, %rax // K / 8 cmpq $3, %rax - jl .L7_13 + jl L(7_13) prefetcht0 B_PR1(BO) prefetcht0 B_PR1+64(BO) @@ -1751,7 +1751,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_5 -.L7_12: +L(7_12): prefetcht0 B_PR1-24(BO) prefetcht0 B_PR1+40(BO) @@ -1766,9 +1766,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_M8 dec %rax - jne .L7_12 + jne L(7_12) -.L7_12_E: +L(7_12_E): prefetcht0 B_PR1(BO) prefetcht0 B_PR1+64(BO) @@ -1781,14 +1781,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_M7 KERNEL8x3_E - jmp .L7_16 + jmp L(7_16) -.L7_13: +L(7_13): test $2, %rax - jz .L7_14 + jz L(7_14) KERNEL8x3_INIT KERNEL8x3_M2 @@ -1808,13 +1808,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_M7 KERNEL8x3_E - jmp .L7_16 + jmp L(7_16) -.L7_14: +L(7_14): test $1, %rax - jz .L7_15 + jz L(7_15) KERNEL8x3_INIT KERNEL8x3_M2 @@ -1825,51 +1825,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_M7 KERNEL8x3_E - jmp .L7_16 + jmp L(7_16) -.L7_15: +L(7_15): INIT8x3 -.L7_16: +L(7_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_19 + je L(7_19) ALIGN_4 -.L7_17: +L(7_17): KERNEL8x3_SUBN dec %rax - jne .L7_17 + jne L(7_17) ALIGN_4 -.L7_19: +L(7_19): SAVE8x3 decq I # i -- - jg .L7_11 + jg L(7_11) ALIGN_4 -.L7_20: +L(7_20): // Test rest of M testq $7, M - jz .L7_60 // to next 6 lines of N + jz L(7_60) // to next 6 lines of N testq $4, M - jz .L7_30 + jz L(7_30) ALIGN_4 -.L7_21: +L(7_21): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -1879,7 +1879,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_26 + je L(7_26) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1890,7 +1890,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_22: +L(7_22): KERNEL4x3_1(xxx) KERNEL4x3_2(xxx) @@ -1902,7 +1902,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L7_26 + je L(7_26) KERNEL4x3_1(xxx) KERNEL4x3_2(xxx) @@ -1914,16 +1914,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L7_26 + je L(7_26) - jmp .L7_22 + jmp L(7_22) ALIGN_4 -.L7_26: +L(7_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_29 + je L(7_29) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1935,16 +1935,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_27: +L(7_27): KERNEL4x3_SUB(xxx) addq $3, BI addq $4, %rax - jl .L7_27 + jl L(7_27) ALIGN_4 -.L7_29: +L(7_29): vmovddup ALPHA, %xmm0 @@ -1973,13 +1973,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L7_30: +L(7_30): testq $2, M - jz .L7_40 + jz L(7_40) ALIGN_4 -.L7_31: +L(7_31): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -1988,7 +1988,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_36 + je L(7_36) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1999,7 +1999,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_32: +L(7_32): KERNEL2x3_1(xxx) KERNEL2x3_2(xxx) @@ -2011,7 +2011,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L7_36 + je L(7_36) KERNEL2x3_1(xxx) KERNEL2x3_2(xxx) @@ -2023,16 +2023,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L7_36 + je L(7_36) - jmp .L7_32 + jmp L(7_32) ALIGN_4 -.L7_36: +L(7_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_39 + je L(7_39) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2044,16 +2044,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_37: +L(7_37): KERNEL2x3_SUB(xxx) addq $3, BI addq $2, %rax - jl .L7_37 + jl L(7_37) ALIGN_4 -.L7_39: +L(7_39): vmovddup ALPHA, %xmm0 @@ -2075,13 +2075,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L7_40: +L(7_40): testq $1, M - jz .L7_60 // to next 6 lines of N + jz L(7_60) // to next 6 lines of N ALIGN_4 -.L7_41: +L(7_41): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -2091,7 +2091,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L7_46 + je L(7_46) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2101,7 +2101,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_42: +L(7_42): KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -2113,7 +2113,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L7_46 + je L(7_46) KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -2125,16 +2125,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L7_46 + je L(7_46) - jmp .L7_42 + jmp L(7_42) ALIGN_4 -.L7_46: +L(7_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_49 + je L(7_49) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2145,16 +2145,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_47: +L(7_47): KERNEL1x3_SUB(xxx) addq $3, BI addq $1, %rax - jl .L7_47 + jl L(7_47) ALIGN_4 -.L7_49: +L(7_49): vmovddup ALPHA, %xmm0 @@ -2171,15 +2171,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $1 * SIZE, CO1 # coffset += 1 -.L7_60: +L(7_60): decq J // j -- - jg .L6_01 + jg L(6_01) -.L2_0: +L(2_0): cmpq $0, Nmod6 // N % 6 == 0 - je .L999 + je L(999) /************************************************************************************************ * Loop for Nmod6 / 2 > 0 @@ -2187,30 +2187,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J sarq $1, J // j = j / 2 - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -2220,11 +2220,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $3, I // i = (m >> 3) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2234,7 +2234,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2245,7 +2245,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_12: +L(2_12): KERNEL8x2_1(xxx) KERNEL8x2_2(xxx) @@ -2257,7 +2257,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_16 + je L(2_16) KERNEL8x2_1(xxx) KERNEL8x2_2(xxx) @@ -2269,16 +2269,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2290,16 +2290,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL8x2_SUB(xxx) addq $2, BI addq $8, %rax - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vmovddup ALPHA, %xmm0 @@ -2326,24 +2326,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $7, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2352,7 +2352,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2363,7 +2363,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_22: +L(2_22): KERNEL4x2_1(xxx) KERNEL4x2_2(xxx) @@ -2375,7 +2375,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) KERNEL4x2_1(xxx) KERNEL4x2_2(xxx) @@ -2387,16 +2387,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2408,16 +2408,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB(xxx) addq $2, BI addq $4, %rax - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): vmovddup ALPHA, %xmm0 @@ -2437,13 +2437,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2452,7 +2452,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2463,7 +2463,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_1(xxx) KERNEL2x2_2(xxx) @@ -2475,7 +2475,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) KERNEL2x2_1(xxx) KERNEL2x2_2(xxx) @@ -2487,16 +2487,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2508,16 +2508,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB(xxx) addq $2, BI addq $2, %rax - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): vmovddup ALPHA, %xmm0 @@ -2531,13 +2531,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2546,7 +2546,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2556,7 +2556,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -2568,7 +2568,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -2580,16 +2580,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2600,16 +2600,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) addq $2, BI addq $1, %rax - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vmovddup ALPHA, %xmm0 @@ -2625,14 +2625,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_60: +L(2_60): decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -2640,30 +2640,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -2672,11 +2672,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $3, I // i = (m >> 3) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -2685,7 +2685,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -2695,7 +2695,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_12: +L(1_12): KERNEL8x1_1(xxx) KERNEL8x1_2(xxx) @@ -2707,7 +2707,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_16 + je L(1_16) KERNEL8x1_1(xxx) KERNEL8x1_2(xxx) @@ -2719,16 +2719,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -2739,16 +2739,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL8x1_SUB(xxx) addq $1, BI addq $8, %rax - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vmovddup ALPHA, %xmm0 @@ -2764,24 +2764,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $7, M - jz .L999 + jz L(999) testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -2791,7 +2791,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -2801,7 +2801,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_22: +L(1_22): KERNEL4x1_1(xxx) KERNEL4x1_2(xxx) @@ -2813,7 +2813,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) KERNEL4x1_1(xxx) KERNEL4x1_2(xxx) @@ -2825,16 +2825,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -2845,16 +2845,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB(xxx) addq $1, BI addq $4, %rax - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): vmovddup ALPHA, %xmm0 @@ -2868,13 +2868,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -2884,7 +2884,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -2894,7 +2894,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_1(xxx) KERNEL2x1_2(xxx) @@ -2906,7 +2906,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) KERNEL2x1_1(xxx) KERNEL2x1_2(xxx) @@ -2918,16 +2918,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -2938,16 +2938,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB(xxx) addq $1, BI addq $2, %rax - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): vmovddup ALPHA, %xmm0 @@ -2959,13 +2959,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -2974,7 +2974,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, 8), AO @@ -2983,7 +2983,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -2995,7 +2995,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -3007,16 +3007,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -3026,16 +3026,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) addq $1, BI addq $1, %rax - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vmovddup ALPHA, %xmm0 @@ -3047,7 +3047,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp @@ -3135,13 +3135,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -3170,32 +3170,32 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv6, J cmpq $0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_0: +L(2_0): -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -3209,11 +3209,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $3, I // i = (m >> 3) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3251,7 +3251,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3262,7 +3262,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_12: +L(2_12): KERNEL8x2_1(xxx) KERNEL8x2_2(xxx) @@ -3274,7 +3274,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_16 + je L(2_16) KERNEL8x2_1(xxx) KERNEL8x2_2(xxx) @@ -3286,12 +3286,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3299,7 +3299,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3311,16 +3311,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL8x2_SUB(xxx) addq $2, BI addq $8, %rax - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vmovddup ALPHA, %xmm0 @@ -3378,24 +3378,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $7, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3432,7 +3432,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3443,7 +3443,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_22: +L(2_22): KERNEL4x2_1(xxx) KERNEL4x2_2(xxx) @@ -3455,7 +3455,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) KERNEL4x2_1(xxx) KERNEL4x2_2(xxx) @@ -3467,12 +3467,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -3480,7 +3480,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3492,16 +3492,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB(xxx) addq $2, BI addq $4, %rax - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): vmovddup ALPHA, %xmm0 @@ -3549,13 +3549,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3593,7 +3593,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3604,7 +3604,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_1(xxx) KERNEL2x2_2(xxx) @@ -3616,7 +3616,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) KERNEL2x2_1(xxx) KERNEL2x2_2(xxx) @@ -3628,12 +3628,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -3641,7 +3641,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3653,16 +3653,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB(xxx) addq $2, BI addq $2, %rax - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): vmovddup ALPHA, %xmm0 @@ -3700,13 +3700,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3741,7 +3741,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3751,7 +3751,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -3763,7 +3763,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -3775,12 +3775,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -3788,7 +3788,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3799,16 +3799,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) addq $2, BI addq $1, %rax - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vmovddup ALPHA, %xmm0 @@ -3847,17 +3847,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -3865,30 +3865,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -3902,11 +3902,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $3, I // i = (m >> 3) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3942,7 +3942,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -3952,7 +3952,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_12: +L(1_12): KERNEL8x1_1(xxx) KERNEL8x1_2(xxx) @@ -3964,7 +3964,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_16 + je L(1_16) KERNEL8x1_1(xxx) KERNEL8x1_2(xxx) @@ -3976,12 +3976,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3989,7 +3989,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -4000,16 +4000,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL8x1_SUB(xxx) addq $1, BI addq $8, %rax - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vmovddup ALPHA, %xmm0 @@ -4049,24 +4049,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $7, M - jz .L999 + jz L(999) testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4103,7 +4103,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -4113,7 +4113,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_22: +L(1_22): KERNEL4x1_1(xxx) KERNEL4x1_2(xxx) @@ -4125,7 +4125,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) KERNEL4x1_1(xxx) KERNEL4x1_2(xxx) @@ -4137,12 +4137,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4150,7 +4150,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -4161,16 +4161,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB(xxx) addq $1, BI addq $4, %rax - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): vmovddup ALPHA, %xmm0 @@ -4207,13 +4207,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4250,7 +4250,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -4260,7 +4260,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_1(xxx) KERNEL2x1_2(xxx) @@ -4272,7 +4272,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) KERNEL2x1_1(xxx) KERNEL2x1_2(xxx) @@ -4284,12 +4284,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -4297,7 +4297,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -4308,16 +4308,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB(xxx) addq $1, BI addq $2, %rax - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): vmovddup ALPHA, %xmm0 @@ -4350,13 +4350,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4391,7 +4391,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, 8), AO @@ -4400,7 +4400,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -4412,7 +4412,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -4424,12 +4424,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -4437,7 +4437,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -4447,16 +4447,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) addq $1, BI addq $1, %rax - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vmovddup ALPHA, %xmm0 @@ -4489,7 +4489,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/dgemm_ncopy_2.S b/kernel/x86_64/dgemm_ncopy_2.S index 532fddf75f..4a734f024f 100644 --- a/kernel/x86_64/dgemm_ncopy_2.S +++ b/kernel/x86_64/dgemm_ncopy_2.S @@ -112,20 +112,20 @@ cmovne %rax, MM testq $SIZE, LDA - jne .L50 + jne L(50) movq N, J sarq $1, J - jle .L30 + jle L(30) ALIGN_4 -.L21: +L(21): movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), A testq $SIZE, A - je .L22 + je L(22) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -139,13 +139,13 @@ subq $-2 * SIZE, B ALIGN_3 -.L22: +L(22): movq MM, I sarq $3, I - jle .L24 + jle L(24) ALIGN_4 -.L23: +L(23): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 2 * SIZE(AO1) #endif @@ -203,12 +203,12 @@ subq $-16 * SIZE, B decq I - jg .L23 + jg L(23) ALIGN_4 -.L24: +L(24): testq $4, MM - jle .L26 + jle L(26) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 0 * SIZE(AO2), %xmm1 @@ -233,9 +233,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L26: +L(26): testq $2, MM - jle .L28 + jle L(28) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 0 * SIZE(AO2), %xmm1 @@ -252,9 +252,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L28: +L(28): testq $1, MM - jle .L29 + jle L(29) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -265,27 +265,27 @@ subq $-2 * SIZE, B ALIGN_4 -.L29: +L(29): decq J - jg .L21 + jg L(21) ALIGN_4 -.L30: +L(30): testq $1, N - jle .L999 + jle L(999) -.L30x: +L(30x): movq A, AO1 testq $SIZE, A - jne .L35 + jne L(35) movq M, I sarq $3, I - jle .L32 + jle L(32) ALIGN_4 -.L31: +L(31): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 4 * SIZE(AO1) #endif @@ -308,12 +308,12 @@ addq $8 * SIZE, B decq I - jg .L31 + jg L(31) ALIGN_4 -.L32: +L(32): testq $4, M - jle .L33 + jle L(33) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 2 * SIZE(AO1), %xmm1 @@ -325,9 +325,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L33: +L(33): testq $2, M - jle .L34 + jle L(34) MOVAPS 0 * SIZE(AO1), %xmm0 @@ -337,24 +337,24 @@ subq $-2 * SIZE, B ALIGN_4 -.L34: +L(34): testq $1, M - jle .L999 + jle L(999) movsd 0 * SIZE(AO1), %xmm0 movlpd %xmm0, -16 * SIZE(B) - jmp .L999 + jmp L(999) ALIGN_4 -.L35: +L(35): movaps -1 * SIZE(AO1), %xmm0 movq M, I sarq $3, I - jle .L36 + jle L(36) ALIGN_4 -.L36: +L(36): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 4 * SIZE(AO1) #endif @@ -384,12 +384,12 @@ subq $-8 * SIZE, B decq I - jg .L36 + jg L(36) ALIGN_4 -.L37: +L(37): testq $4, M - jle .L38 + jle L(38) MOVAPS 1 * SIZE(AO1), %xmm1 MOVAPS 3 * SIZE(AO1), %xmm2 @@ -406,9 +406,9 @@ addq $4 * SIZE, B ALIGN_4 -.L38: +L(38): testq $2, M - jle .L39 + jle L(39) MOVAPS 1 * SIZE(AO1), %xmm1 @@ -422,27 +422,27 @@ subq $-2 * SIZE, B ALIGN_4 -.L39: +L(39): testq $1, M - jle .L999 + jle L(999) movhpd %xmm0, -16 * SIZE(B) - jmp .L999 + jmp L(999) ALIGN_4 -.L50: +L(50): movq N, J sarq $1, J - jle .L30 + jle L(30) ALIGN_4 -.L61: +L(61): movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), A testq $SIZE, A - je .L62 + je L(62) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -456,15 +456,15 @@ subq $-2 * SIZE, B ALIGN_3 -.L62: +L(62): MOVAPS -1 * SIZE(AO2), %xmm5 movq MM, I sarq $3, I - jle .L64 + jle L(64) ALIGN_4 -.L63: +L(63): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 2 * SIZE(AO1) #endif @@ -516,12 +516,12 @@ subq $-16 * SIZE, B decq I - jg .L63 + jg L(63) ALIGN_4 -.L64: +L(64): testq $4, MM - jle .L66 + jle L(66) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 1 * SIZE(AO2), %xmm1 @@ -545,9 +545,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L66: +L(66): testq $2, MM - jle .L68 + jle L(68) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 1 * SIZE(AO2), %xmm1 @@ -563,9 +563,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L68: +L(68): testq $1, MM - jle .L69 + jle L(69) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -576,15 +576,15 @@ subq $-2 * SIZE, B ALIGN_4 -.L69: +L(69): decq J - jg .L61 + jg L(61) testq $1, N - jne .L30 + jne L(30) ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 diff --git a/kernel/x86_64/dgemm_ncopy_4.S b/kernel/x86_64/dgemm_ncopy_4.S index 41eac95970..3e6ef0600f 100644 --- a/kernel/x86_64/dgemm_ncopy_4.S +++ b/kernel/x86_64/dgemm_ncopy_4.S @@ -133,20 +133,20 @@ cmovne %rax, MM testq $SIZE, LDA - jne .L50 + jne L(50) movq N, J sarq $2, J - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): movq A, AO1 leaq (A, LDA, 2), AO2 leaq (A, LDA, 4), A testq $SIZE, A - je .L12 + je L(12) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -164,13 +164,13 @@ subq $-4 * SIZE, B ALIGN_3 -.L12: +L(12): movq MM, I sarq $3, I - jle .L14 + jle L(14) ALIGN_4 -.L13: +L(13): #ifdef PREFETCH PREFETCH PREFETCHSIZE * SIZE(AO1) #endif @@ -280,12 +280,12 @@ subq $-32 * SIZE, B decq I - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): testq $4, MM - jle .L16 + jle L(16) movapd 0 * SIZE(AO1), %xmm0 movapd 0 * SIZE(AO1, LDA), %xmm1 @@ -328,9 +328,9 @@ subq $-16 * SIZE, B ALIGN_4 -.L16: +L(16): testq $2, MM - jle .L18 + jle L(18) movapd 0 * SIZE(AO1), %xmm0 movapd 0 * SIZE(AO1, LDA), %xmm1 @@ -355,9 +355,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L18: +L(18): testq $1, MM - jle .L19 + jle L(19) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -372,21 +372,21 @@ subq $-4 * SIZE, B ALIGN_4 -.L19: +L(19): decq J - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, N - jle .L30 + jle L(30) movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), A testq $SIZE, A - je .L22 + je L(22) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -400,13 +400,13 @@ subq $-2 * SIZE, B ALIGN_3 -.L22: +L(22): movq MM, I sarq $3, I - jle .L24 + jle L(24) ALIGN_4 -.L23: +L(23): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 2 * SIZE(AO1) #endif @@ -464,12 +464,12 @@ subq $-16 * SIZE, B decq I - jg .L23 + jg L(23) ALIGN_4 -.L24: +L(24): testq $4, MM - jle .L26 + jle L(26) movapd 0 * SIZE(AO1), %xmm0 movapd 0 * SIZE(AO2), %xmm1 @@ -494,9 +494,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L26: +L(26): testq $2, MM - jle .L28 + jle L(28) movapd 0 * SIZE(AO1), %xmm0 movapd 0 * SIZE(AO2), %xmm1 @@ -513,9 +513,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L28: +L(28): testq $1, MM - jle .L30 + jle L(30) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -526,21 +526,21 @@ subq $-2 * SIZE, B ALIGN_4 -.L30: +L(30): testq $1, N - jle .L999 + jle L(999) movq A, AO1 testq $SIZE, A - jne .L35 + jne L(35) movq MM, I sarq $3, I - jle .L32 + jle L(32) ALIGN_4 -.L31: +L(31): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 4 * SIZE(AO1) #endif @@ -563,12 +563,12 @@ subq $-8 * SIZE, B decq I - jg .L31 + jg L(31) ALIGN_4 -.L32: +L(32): testq $4, MM - jle .L33 + jle L(33) movapd 0 * SIZE(AO1), %xmm0 movapd 2 * SIZE(AO1), %xmm1 @@ -580,9 +580,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L33: +L(33): testq $2, MM - jle .L34 + jle L(34) movapd 0 * SIZE(AO1), %xmm0 @@ -592,25 +592,25 @@ subq $-2 * SIZE, B ALIGN_4 -.L34: +L(34): testq $1, MM - jle .L999 + jle L(999) movsd 0 * SIZE(AO1), %xmm0 movlpd %xmm0, -16 * SIZE(B) - jmp .L999 + jmp L(999) ALIGN_4 -.L35: +L(35): movapd -1 * SIZE(AO1), %xmm0 movq MM, I sarq $3, I - jle .L36 + jle L(36) ALIGN_4 -.L36: +L(36): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 4 * SIZE(AO1) #endif @@ -640,12 +640,12 @@ subq $-8 * SIZE, B decq I - jg .L36 + jg L(36) ALIGN_4 -.L37: +L(37): testq $4, MM - jle .L38 + jle L(38) movapd 1 * SIZE(AO1), %xmm1 movapd 3 * SIZE(AO1), %xmm2 @@ -662,9 +662,9 @@ addq $4 * SIZE, B ALIGN_4 -.L38: +L(38): testq $2, MM - jle .L39 + jle L(39) movapd 1 * SIZE(AO1), %xmm1 @@ -678,29 +678,29 @@ subq $-2 * SIZE, B ALIGN_4 -.L39: +L(39): testq $1, MM - jle .L999 + jle L(999) shufpd $1, %xmm0, %xmm0 movlpd %xmm0, -16 * SIZE(B) - jmp .L999 + jmp L(999) ALIGN_4 -.L50: +L(50): movq N, J sarq $2, J - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): movq A, AO1 leaq (A, LDA, 2), AO2 leaq (A, LDA, 4), A testq $SIZE, A - je .L52 + je L(52) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -718,16 +718,16 @@ subq $-4 * SIZE, B ALIGN_3 -.L52: +L(52): movapd -1 * SIZE(AO1, LDA), %xmm5 movapd -1 * SIZE(AO2, LDA), %xmm7 movq MM, I sarq $3, I - jle .L54 + jle L(54) ALIGN_4 -.L53: +L(53): #ifdef PREFETCH PREFETCH PREFETCHSIZE * SIZE(AO1) #endif @@ -825,12 +825,12 @@ subq $-32 * SIZE, B decq I - jg .L53 + jg L(53) ALIGN_4 -.L54: +L(54): testq $4, MM - jle .L56 + jle L(56) movapd 0 * SIZE(AO1), %xmm0 movapd 1 * SIZE(AO1, LDA), %xmm1 @@ -867,9 +867,9 @@ subq $-16 * SIZE, B ALIGN_4 -.L56: +L(56): testq $2, MM - jle .L58 + jle L(58) movapd 0 * SIZE(AO1), %xmm0 movapd 1 * SIZE(AO1, LDA), %xmm1 @@ -891,9 +891,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L58: +L(58): testq $1, MM - jle .L59 + jle L(59) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -908,21 +908,21 @@ subq $-4 * SIZE, B ALIGN_4 -.L59: +L(59): decq J - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, N - jle .L70 + jle L(70) movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), A testq $SIZE, A - je .L62 + je L(62) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -936,15 +936,15 @@ subq $-2 * SIZE, B ALIGN_3 -.L62: +L(62): movapd -1 * SIZE(AO2), %xmm5 movq MM, I sarq $3, I - jle .L64 + jle L(64) ALIGN_4 -.L63: +L(63): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 2 * SIZE(AO1) #endif @@ -996,12 +996,12 @@ subq $-16 * SIZE, B decq I - jg .L63 + jg L(63) ALIGN_4 -.L64: +L(64): testq $4, MM - jle .L66 + jle L(66) movapd 0 * SIZE(AO1), %xmm0 movapd 1 * SIZE(AO2), %xmm1 @@ -1025,9 +1025,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L66: +L(66): testq $2, MM - jle .L68 + jle L(68) movapd 0 * SIZE(AO1), %xmm0 movapd 1 * SIZE(AO2), %xmm1 @@ -1043,9 +1043,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L68: +L(68): testq $1, MM - jle .L70 + jle L(70) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -1056,21 +1056,21 @@ subq $-2 * SIZE, B ALIGN_4 -.L70: +L(70): testq $1, N - jle .L999 + jle L(999) movq A, AO1 testq $SIZE, A - jne .L75 + jne L(75) movq MM, I sarq $3, I - jle .L72 + jle L(72) ALIGN_4 -.L71: +L(71): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 4 * SIZE(AO1) #endif @@ -1093,12 +1093,12 @@ subq $-8 * SIZE, B decq I - jg .L71 + jg L(71) ALIGN_4 -.L72: +L(72): testq $4, MM - jle .L73 + jle L(73) movapd 0 * SIZE(AO1), %xmm0 movapd 2 * SIZE(AO1), %xmm2 @@ -1110,9 +1110,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L73: +L(73): testq $2, MM - jle .L74 + jle L(74) movapd 0 * SIZE(AO1), %xmm0 @@ -1122,25 +1122,25 @@ subq $-2 * SIZE, B ALIGN_4 -.L74: +L(74): testq $1, MM - jle .L999 + jle L(999) movsd 0 * SIZE(AO1), %xmm0 movlpd %xmm0, -16 * SIZE(B) - jmp .L999 + jmp L(999) ALIGN_4 -.L75: +L(75): movapd -1 * SIZE(AO1), %xmm0 movq MM, I sarq $3, I - jle .L76 + jle L(76) ALIGN_4 -.L76: +L(76): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 4 * SIZE(AO1) #endif @@ -1170,12 +1170,12 @@ subq $-8 * SIZE, B decq I - jg .L76 + jg L(76) ALIGN_4 -.L77: +L(77): testq $4, MM - jle .L78 + jle L(78) movapd 1 * SIZE(AO1), %xmm1 movapd 3 * SIZE(AO1), %xmm2 @@ -1192,9 +1192,9 @@ addq $4 * SIZE, B ALIGN_4 -.L78: +L(78): testq $2, MM - jle .L79 + jle L(79) movapd 1 * SIZE(AO1), %xmm1 @@ -1208,16 +1208,16 @@ subq $-2 * SIZE, B ALIGN_4 -.L79: +L(79): testq $1, MM - jle .L999 + jle L(999) shufpd $1, %xmm0, %xmm0 movlpd %xmm0, -16 * SIZE(B) ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 movups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/dgemm_ncopy_8.S b/kernel/x86_64/dgemm_ncopy_8.S index 7600c9a567..2b83ac6432 100644 --- a/kernel/x86_64/dgemm_ncopy_8.S +++ b/kernel/x86_64/dgemm_ncopy_8.S @@ -125,20 +125,20 @@ cmovne %rax, MM testq $SIZE, LDA - jne .L50 + jne L(50) movq N, J sarq $3, J - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): movq A, AO1 leaq (A, LDA, 4), AO2 leaq (A, LDA, 8), A testq $SIZE, A - je .L12 + je L(12) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -165,13 +165,13 @@ subq $-8 * SIZE, B ALIGN_3 -.L12: +L(12): movq MM, I sarq $3, I - jle .L14 + jle L(14) ALIGN_4 -.L13: +L(13): #ifdef PREFETCH PREFETCH PREFETCHSIZE * SIZE(AO1) #endif @@ -381,12 +381,12 @@ subq $-64 * SIZE, B decq I - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): testq $4, MM - jle .L16 + jle L(16) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 0 * SIZE(AO1, LDA), %xmm1 @@ -463,9 +463,9 @@ subq $-32 * SIZE, B ALIGN_4 -.L16: +L(16): testq $2, MM - jle .L18 + jle L(18) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 0 * SIZE(AO1, LDA), %xmm1 @@ -507,9 +507,9 @@ subq $-16 * SIZE, B ALIGN_4 -.L18: +L(18): testq $1, MM - jle .L19 + jle L(19) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -534,21 +534,21 @@ subq $-8 * SIZE, B ALIGN_4 -.L19: +L(19): decq J - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, N - jle .L30 + jle L(30) movq A, AO1 leaq (A, LDA, 2), AO2 leaq (A, LDA, 4), A testq $SIZE, A - je .L22 + je L(22) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -566,13 +566,13 @@ subq $-4 * SIZE, B ALIGN_3 -.L22: +L(22): movq MM, I sarq $3, I - jle .L24 + jle L(24) ALIGN_4 -.L23: +L(23): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 2 * SIZE(AO1) #endif @@ -682,12 +682,12 @@ subq $-32 * SIZE, B decq I - jg .L23 + jg L(23) ALIGN_4 -.L24: +L(24): testq $4, MM - jle .L26 + jle L(26) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 0 * SIZE(AO1, LDA), %xmm1 @@ -730,9 +730,9 @@ subq $-16 * SIZE, B ALIGN_4 -.L26: +L(26): testq $2, MM - jle .L28 + jle L(28) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 0 * SIZE(AO1, LDA), %xmm1 @@ -757,9 +757,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L28: +L(28): testq $1, MM - jle .L30 + jle L(30) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -774,16 +774,16 @@ subq $-4 * SIZE, B ALIGN_4 -.L30: +L(30): testq $2, N - jle .L40 + jle L(40) movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), A testq $SIZE, A - je .L32 + je L(32) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -797,13 +797,13 @@ subq $-2 * SIZE, B ALIGN_3 -.L32: +L(32): movq MM, I sarq $3, I - jle .L34 + jle L(34) ALIGN_4 -.L33: +L(33): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 4 * SIZE(AO1) #endif @@ -861,12 +861,12 @@ subq $-16 * SIZE, B decq I - jg .L33 + jg L(33) ALIGN_4 -.L34: +L(34): testq $4, MM - jle .L36 + jle L(36) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 0 * SIZE(AO2), %xmm1 @@ -891,9 +891,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L36: +L(36): testq $2, MM - jle .L38 + jle L(38) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 0 * SIZE(AO2), %xmm1 @@ -910,9 +910,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L38: +L(38): testq $1, MM - jle .L40 + jle L(40) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -923,21 +923,21 @@ subq $-2 * SIZE, B ALIGN_4 -.L40: +L(40): testq $1, N - jle .L999 + jle L(999) movq A, AO1 testq $SIZE, A - jne .L45 + jne L(45) movq MM, I sarq $3, I - jle .L42 + jle L(42) ALIGN_4 -.L41: +L(41): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 8 * SIZE(AO1) #endif @@ -960,12 +960,12 @@ subq $-8 * SIZE, B decq I - jg .L41 + jg L(41) ALIGN_4 -.L42: +L(42): testq $4, MM - jle .L43 + jle L(43) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 2 * SIZE(AO1), %xmm1 @@ -977,9 +977,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L43: +L(43): testq $2, MM - jle .L44 + jle L(44) MOVAPS 0 * SIZE(AO1), %xmm0 @@ -989,25 +989,25 @@ subq $-2 * SIZE, B ALIGN_4 -.L44: +L(44): testq $1, MM - jle .L999 + jle L(999) movsd 0 * SIZE(AO1), %xmm0 movlpd %xmm0, -16 * SIZE(B) - jmp .L999 + jmp L(999) ALIGN_4 -.L45: +L(45): MOVAPS -1 * SIZE(AO1), %xmm0 movq M, I sarq $3, I - jle .L46 + jle L(46) ALIGN_4 -.L46: +L(46): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 8 * SIZE(AO1) #endif @@ -1037,12 +1037,12 @@ subq $-8 * SIZE, B decq I - jg .L46 + jg L(46) ALIGN_4 -.L47: +L(47): testq $4, M - jle .L48 + jle L(48) MOVAPS 1 * SIZE(AO1), %xmm1 MOVAPS 3 * SIZE(AO1), %xmm2 @@ -1059,9 +1059,9 @@ addq $4 * SIZE, B ALIGN_4 -.L48: +L(48): testq $2, M - jle .L49 + jle L(49) MOVAPS 1 * SIZE(AO1), %xmm1 @@ -1075,29 +1075,29 @@ subq $-2 * SIZE, B ALIGN_4 -.L49: +L(49): testq $1, M - jle .L999 + jle L(999) shufpd $1, %xmm0, %xmm0 movlpd %xmm0, -16 * SIZE(B) - jmp .L999 + jmp L(999) ALIGN_4 -.L50: +L(50): movq N, J sarq $3, J - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): movq A, AO1 leaq (A, LDA, 4), AO2 leaq (A, LDA, 8), A testq $SIZE, A - je .L52 + je L(52) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -1123,7 +1123,7 @@ subq $-8 * SIZE, B ALIGN_3 -.L52: +L(52): MOVAPS -1 * SIZE(AO1, LDA), %xmm9 MOVAPS -1 * SIZE(AO1, LDA3), %xmm10 MOVAPS -1 * SIZE(AO2, LDA), %xmm11 @@ -1131,10 +1131,10 @@ movq MM, I sarq $3, I - jle .L54 + jle L(54) ALIGN_4 -.L53: +L(53): #ifdef PREFETCH PREFETCH PREFETCHSIZE * SIZE(AO1) #endif @@ -1323,12 +1323,12 @@ subq $-64 * SIZE, B decq I - jg .L53 + jg L(53) ALIGN_4 -.L54: +L(54): testq $4, MM - jle .L56 + jle L(56) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 1 * SIZE(AO1, LDA), %xmm1 @@ -1393,9 +1393,9 @@ subq $-32 * SIZE, B ALIGN_4 -.L56: +L(56): testq $2, MM - jle .L58 + jle L(58) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 1 * SIZE(AO1, LDA), %xmm1 @@ -1431,9 +1431,9 @@ subq $-16 * SIZE, B ALIGN_4 -.L58: +L(58): testq $1, MM - jle .L59 + jle L(59) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -1457,21 +1457,21 @@ subq $-8 * SIZE, B ALIGN_4 -.L59: +L(59): decq J - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $4, N - jle .L70 + jle L(70) movq A, AO1 leaq (A, LDA, 2), AO2 leaq (A, LDA, 4), A testq $SIZE, A - je .L62 + je L(62) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -1489,16 +1489,16 @@ subq $-4 * SIZE, B ALIGN_3 -.L62: +L(62): movaps -1 * SIZE(AO1, LDA), %xmm5 movaps -1 * SIZE(AO2, LDA), %xmm7 movq MM, I sarq $3, I - jle .L64 + jle L(64) ALIGN_4 -.L63: +L(63): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 2 * SIZE(AO1) #endif @@ -1596,12 +1596,12 @@ subq $-32 * SIZE, B decq I - jg .L63 + jg L(63) ALIGN_4 -.L64: +L(64): testq $4, MM - jle .L66 + jle L(66) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 1 * SIZE(AO1, LDA), %xmm1 @@ -1638,9 +1638,9 @@ subq $-16 * SIZE, B ALIGN_4 -.L66: +L(66): testq $2, MM - jle .L68 + jle L(68) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 1 * SIZE(AO1, LDA), %xmm1 @@ -1662,9 +1662,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L68: +L(68): testq $1, MM - jle .L70 + jle L(70) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -1679,16 +1679,16 @@ subq $-4 * SIZE, B ALIGN_4 -.L70: +L(70): testq $2, N - jle .L80 + jle L(80) movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), A testq $SIZE, A - je .L72 + je L(72) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -1702,15 +1702,15 @@ subq $-2 * SIZE, B ALIGN_3 -.L72: +L(72): MOVAPS -1 * SIZE(AO2), %xmm5 movq MM, I sarq $3, I - jle .L74 + jle L(74) ALIGN_4 -.L73: +L(73): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 4 * SIZE(AO1) #endif @@ -1762,12 +1762,12 @@ subq $-16 * SIZE, B decq I - jg .L73 + jg L(73) ALIGN_4 -.L74: +L(74): testq $4, MM - jle .L76 + jle L(76) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 1 * SIZE(AO2), %xmm1 @@ -1791,9 +1791,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L76: +L(76): testq $2, MM - jle .L78 + jle L(78) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 1 * SIZE(AO2), %xmm1 @@ -1809,9 +1809,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L78: +L(78): testq $1, MM - jle .L80 + jle L(80) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -1822,21 +1822,21 @@ subq $-2 * SIZE, B ALIGN_4 -.L80: +L(80): testq $1, N - jle .L999 + jle L(999) movq A, AO1 testq $SIZE, A - jne .L85 + jne L(85) movq MM, I sarq $3, I - jle .L82 + jle L(82) ALIGN_4 -.L81: +L(81): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 8 * SIZE(AO1) #endif @@ -1859,12 +1859,12 @@ subq $-8 * SIZE, B decq I - jg .L81 + jg L(81) ALIGN_4 -.L82: +L(82): testq $4, MM - jle .L83 + jle L(83) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 2 * SIZE(AO1), %xmm2 @@ -1876,9 +1876,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L83: +L(83): testq $2, MM - jle .L84 + jle L(84) MOVAPS 0 * SIZE(AO1), %xmm0 @@ -1888,25 +1888,25 @@ subq $-2 * SIZE, B ALIGN_4 -.L84: +L(84): testq $1, MM - jle .L999 + jle L(999) movsd 0 * SIZE(AO1), %xmm0 movlpd %xmm0, -16 * SIZE(B) - jmp .L999 + jmp L(999) ALIGN_4 -.L85: +L(85): MOVAPS -1 * SIZE(AO1), %xmm0 movq M, I sarq $3, I - jle .L86 + jle L(86) ALIGN_4 -.L86: +L(86): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 8 * SIZE(AO1) #endif @@ -1936,12 +1936,12 @@ subq $-8 * SIZE, B decq I - jg .L86 + jg L(86) ALIGN_4 -.L87: +L(87): testq $4, M - jle .L88 + jle L(88) MOVAPS 1 * SIZE(AO1), %xmm1 MOVAPS 3 * SIZE(AO1), %xmm2 @@ -1958,9 +1958,9 @@ addq $4 * SIZE, B ALIGN_4 -.L88: +L(88): testq $2, M - jle .L89 + jle L(89) MOVAPS 1 * SIZE(AO1), %xmm1 @@ -1974,16 +1974,16 @@ subq $-2 * SIZE, B ALIGN_4 -.L89: +L(89): testq $1, M - jle .L999 + jle L(999) shufpd $1, %xmm0, %xmm0 movlpd %xmm0, -16 * SIZE(B) ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 movups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/dgemm_ncopy_8_bulldozer.S b/kernel/x86_64/dgemm_ncopy_8_bulldozer.S index 43f9cd2e3c..dfccce8cfe 100644 --- a/kernel/x86_64/dgemm_ncopy_8_bulldozer.S +++ b/kernel/x86_64/dgemm_ncopy_8_bulldozer.S @@ -113,20 +113,20 @@ cmovne %rax, MM testq $SIZE, LDA - jne .L50 + jne L(50) movq N, J sarq $3, J - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): movq A, AO1 leaq (A, LDA, 4), AO2 leaq (A, LDA, 8), A testq $SIZE, A - je .L12 + je L(12) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO1, LDA), %xmm1 @@ -153,13 +153,13 @@ subq $-8 * SIZE, B ALIGN_3 -.L12: +L(12): movq MM, I sarq $3, I - jle .L14 + jle L(14) ALIGN_4 -.L13: +L(13): prefetchnta A_PRE(AO1) vmovups 0 * SIZE(AO1), %xmm0 @@ -338,12 +338,12 @@ subq $-64 * SIZE, B decq I - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): testq $4, MM - jle .L16 + jle L(16) vmovups 0 * SIZE(AO1), %xmm0 vmovups 0 * SIZE(AO1, LDA), %xmm1 @@ -420,9 +420,9 @@ subq $-32 * SIZE, B ALIGN_4 -.L16: +L(16): testq $2, MM - jle .L18 + jle L(18) vmovups 0 * SIZE(AO1), %xmm0 vmovups 0 * SIZE(AO1, LDA), %xmm1 @@ -464,9 +464,9 @@ subq $-16 * SIZE, B ALIGN_4 -.L18: +L(18): testq $1, MM - jle .L19 + jle L(19) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO1, LDA), %xmm1 @@ -491,21 +491,21 @@ subq $-8 * SIZE, B ALIGN_4 -.L19: +L(19): decq J - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, N - jle .L30 + jle L(30) movq A, AO1 leaq (A, LDA, 2), AO2 leaq (A, LDA, 4), A testq $SIZE, A - je .L22 + je L(22) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO1, LDA), %xmm1 @@ -523,13 +523,13 @@ subq $-4 * SIZE, B ALIGN_3 -.L22: +L(22): movq MM, I sarq $3, I - jle .L24 + jle L(24) ALIGN_4 -.L23: +L(23): vmovups 0 * SIZE(AO1), %xmm0 vmovups 0 * SIZE(AO1, LDA), %xmm1 @@ -615,12 +615,12 @@ subq $-32 * SIZE, B decq I - jg .L23 + jg L(23) ALIGN_4 -.L24: +L(24): testq $4, MM - jle .L26 + jle L(26) vmovups 0 * SIZE(AO1), %xmm0 vmovups 0 * SIZE(AO1, LDA), %xmm1 @@ -663,9 +663,9 @@ subq $-16 * SIZE, B ALIGN_4 -.L26: +L(26): testq $2, MM - jle .L28 + jle L(28) vmovups 0 * SIZE(AO1), %xmm0 vmovups 0 * SIZE(AO1, LDA), %xmm1 @@ -690,9 +690,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L28: +L(28): testq $1, MM - jle .L30 + jle L(30) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO1, LDA), %xmm1 @@ -707,16 +707,16 @@ subq $-4 * SIZE, B ALIGN_4 -.L30: +L(30): testq $2, N - jle .L40 + jle L(40) movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), A testq $SIZE, A - je .L32 + je L(32) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO2), %xmm1 @@ -730,13 +730,13 @@ subq $-2 * SIZE, B ALIGN_3 -.L32: +L(32): movq MM, I sarq $3, I - jle .L34 + jle L(34) ALIGN_4 -.L33: +L(33): vmovups 0 * SIZE(AO1), %xmm0 vmovups 0 * SIZE(AO2), %xmm1 @@ -782,12 +782,12 @@ subq $-16 * SIZE, B decq I - jg .L33 + jg L(33) ALIGN_4 -.L34: +L(34): testq $4, MM - jle .L36 + jle L(36) vmovups 0 * SIZE(AO1), %xmm0 vmovups 0 * SIZE(AO2), %xmm1 @@ -812,9 +812,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L36: +L(36): testq $2, MM - jle .L38 + jle L(38) vmovups 0 * SIZE(AO1), %xmm0 vmovups 0 * SIZE(AO2), %xmm1 @@ -831,9 +831,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L38: +L(38): testq $1, MM - jle .L40 + jle L(40) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO2), %xmm1 @@ -844,21 +844,21 @@ subq $-2 * SIZE, B ALIGN_4 -.L40: +L(40): testq $1, N - jle .L999 + jle L(999) movq A, AO1 testq $SIZE, A - jne .L45 + jne L(45) movq MM, I sarq $3, I - jle .L42 + jle L(42) ALIGN_4 -.L41: +L(41): vmovups 0 * SIZE(AO1), %xmm0 vmovups 2 * SIZE(AO1), %xmm1 @@ -875,12 +875,12 @@ subq $-8 * SIZE, B decq I - jg .L41 + jg L(41) ALIGN_4 -.L42: +L(42): testq $4, MM - jle .L43 + jle L(43) vmovups 0 * SIZE(AO1), %xmm0 vmovups 2 * SIZE(AO1), %xmm1 @@ -892,9 +892,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L43: +L(43): testq $2, MM - jle .L44 + jle L(44) vmovups 0 * SIZE(AO1), %xmm0 @@ -904,25 +904,25 @@ subq $-2 * SIZE, B ALIGN_4 -.L44: +L(44): testq $1, MM - jle .L999 + jle L(999) vmovsd 0 * SIZE(AO1), %xmm0 vmovlpd %xmm0, -16 * SIZE(B) - jmp .L999 + jmp L(999) ALIGN_4 -.L45: +L(45): vmovups -1 * SIZE(AO1), %xmm0 movq M, I sarq $3, I - jle .L46 + jle L(46) ALIGN_4 -.L46: +L(46): vmovups 1 * SIZE(AO1), %xmm1 vmovups 3 * SIZE(AO1), %xmm2 @@ -946,12 +946,12 @@ subq $-8 * SIZE, B decq I - jg .L46 + jg L(46) ALIGN_4 -.L47: +L(47): testq $4, M - jle .L48 + jle L(48) vmovups 1 * SIZE(AO1), %xmm1 vmovups 3 * SIZE(AO1), %xmm2 @@ -968,9 +968,9 @@ addq $4 * SIZE, B ALIGN_4 -.L48: +L(48): testq $2, M - jle .L49 + jle L(49) vmovups 1 * SIZE(AO1), %xmm1 @@ -984,29 +984,29 @@ subq $-2 * SIZE, B ALIGN_4 -.L49: +L(49): testq $1, M - jle .L999 + jle L(999) vshufpd $1, %xmm0, %xmm0 , %xmm0 vmovlpd %xmm0, -16 * SIZE(B) - jmp .L999 + jmp L(999) ALIGN_4 -.L50: +L(50): movq N, J sarq $3, J - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): movq A, AO1 leaq (A, LDA, 4), AO2 leaq (A, LDA, 8), A testq $SIZE, A - je .L52 + je L(52) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO1, LDA), %xmm1 @@ -1032,7 +1032,7 @@ subq $-8 * SIZE, B ALIGN_3 -.L52: +L(52): vmovups -1 * SIZE(AO1, LDA), %xmm9 vmovups -1 * SIZE(AO1, LDA3), %xmm10 vmovups -1 * SIZE(AO2, LDA), %xmm11 @@ -1040,10 +1040,10 @@ movq MM, I sarq $3, I - jle .L54 + jle L(54) ALIGN_4 -.L53: +L(53): vmovups 0 * SIZE(AO1), %xmm0 vmovups 1 * SIZE(AO1, LDA), %xmm1 @@ -1184,12 +1184,12 @@ subq $-64 * SIZE, B decq I - jg .L53 + jg L(53) ALIGN_4 -.L54: +L(54): testq $4, MM - jle .L56 + jle L(56) vmovups 0 * SIZE(AO1), %xmm0 vmovups 1 * SIZE(AO1, LDA), %xmm1 @@ -1254,9 +1254,9 @@ subq $-32 * SIZE, B ALIGN_4 -.L56: +L(56): testq $2, MM - jle .L58 + jle L(58) vmovups 0 * SIZE(AO1), %xmm0 vmovups 1 * SIZE(AO1, LDA), %xmm1 @@ -1292,9 +1292,9 @@ subq $-16 * SIZE, B ALIGN_4 -.L58: +L(58): testq $1, MM - jle .L59 + jle L(59) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO1, LDA), %xmm1 @@ -1318,21 +1318,21 @@ subq $-8 * SIZE, B ALIGN_4 -.L59: +L(59): decq J - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $4, N - jle .L70 + jle L(70) movq A, AO1 leaq (A, LDA, 2), AO2 leaq (A, LDA, 4), A testq $SIZE, A - je .L62 + je L(62) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO1, LDA), %xmm1 @@ -1350,16 +1350,16 @@ subq $-4 * SIZE, B ALIGN_3 -.L62: +L(62): vmovups -1 * SIZE(AO1, LDA), %xmm5 vmovups -1 * SIZE(AO2, LDA), %xmm7 movq MM, I sarq $3, I - jle .L64 + jle L(64) ALIGN_4 -.L63: +L(63): vmovups 0 * SIZE(AO1), %xmm0 vmovups 1 * SIZE(AO1, LDA), %xmm1 @@ -1433,12 +1433,12 @@ subq $-32 * SIZE, B decq I - jg .L63 + jg L(63) ALIGN_4 -.L64: +L(64): testq $4, MM - jle .L66 + jle L(66) vmovups 0 * SIZE(AO1), %xmm0 vmovups 1 * SIZE(AO1, LDA), %xmm1 @@ -1475,9 +1475,9 @@ subq $-16 * SIZE, B ALIGN_4 -.L66: +L(66): testq $2, MM - jle .L68 + jle L(68) vmovups 0 * SIZE(AO1), %xmm0 vmovups 1 * SIZE(AO1, LDA), %xmm1 @@ -1499,9 +1499,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L68: +L(68): testq $1, MM - jle .L70 + jle L(70) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO1, LDA), %xmm1 @@ -1516,16 +1516,16 @@ subq $-4 * SIZE, B ALIGN_4 -.L70: +L(70): testq $2, N - jle .L80 + jle L(80) movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), A testq $SIZE, A - je .L72 + je L(72) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO2), %xmm1 @@ -1539,15 +1539,15 @@ subq $-2 * SIZE, B ALIGN_3 -.L72: +L(72): vmovups -1 * SIZE(AO2), %xmm5 movq MM, I sarq $3, I - jle .L74 + jle L(74) ALIGN_4 -.L73: +L(73): vmovups 0 * SIZE(AO1), %xmm0 vmovups 1 * SIZE(AO2), %xmm1 @@ -1587,12 +1587,12 @@ subq $-16 * SIZE, B decq I - jg .L73 + jg L(73) ALIGN_4 -.L74: +L(74): testq $4, MM - jle .L76 + jle L(76) vmovups 0 * SIZE(AO1), %xmm0 vmovups 1 * SIZE(AO2), %xmm1 @@ -1616,9 +1616,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L76: +L(76): testq $2, MM - jle .L78 + jle L(78) vmovups 0 * SIZE(AO1), %xmm0 vmovups 1 * SIZE(AO2), %xmm1 @@ -1634,9 +1634,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L78: +L(78): testq $1, MM - jle .L80 + jle L(80) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO2), %xmm1 @@ -1647,21 +1647,21 @@ subq $-2 * SIZE, B ALIGN_4 -.L80: +L(80): testq $1, N - jle .L999 + jle L(999) movq A, AO1 testq $SIZE, A - jne .L85 + jne L(85) movq MM, I sarq $3, I - jle .L82 + jle L(82) ALIGN_4 -.L81: +L(81): vmovups 0 * SIZE(AO1), %xmm0 vmovups 2 * SIZE(AO1), %xmm2 @@ -1678,12 +1678,12 @@ subq $-8 * SIZE, B decq I - jg .L81 + jg L(81) ALIGN_4 -.L82: +L(82): testq $4, MM - jle .L83 + jle L(83) vmovups 0 * SIZE(AO1), %xmm0 vmovups 2 * SIZE(AO1), %xmm2 @@ -1695,9 +1695,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L83: +L(83): testq $2, MM - jle .L84 + jle L(84) vmovups 0 * SIZE(AO1), %xmm0 @@ -1707,25 +1707,25 @@ subq $-2 * SIZE, B ALIGN_4 -.L84: +L(84): testq $1, MM - jle .L999 + jle L(999) vmovsd 0 * SIZE(AO1), %xmm0 vmovlpd %xmm0, -16 * SIZE(B) - jmp .L999 + jmp L(999) ALIGN_4 -.L85: +L(85): vmovups -1 * SIZE(AO1), %xmm0 movq M, I sarq $3, I - jle .L86 + jle L(86) ALIGN_4 -.L86: +L(86): vmovups 1 * SIZE(AO1), %xmm1 vmovups 3 * SIZE(AO1), %xmm2 @@ -1749,12 +1749,12 @@ subq $-8 * SIZE, B decq I - jg .L86 + jg L(86) ALIGN_4 -.L87: +L(87): testq $4, M - jle .L88 + jle L(88) vmovups 1 * SIZE(AO1), %xmm1 vmovups 3 * SIZE(AO1), %xmm2 @@ -1771,9 +1771,9 @@ addq $4 * SIZE, B ALIGN_4 -.L88: +L(88): testq $2, M - jle .L89 + jle L(89) vmovups 1 * SIZE(AO1), %xmm1 @@ -1787,16 +1787,16 @@ subq $-2 * SIZE, B ALIGN_4 -.L89: +L(89): testq $1, M - jle .L999 + jle L(999) vshufpd $1, %xmm0, %xmm0 , %xmm0 vmovlpd %xmm0, -16 * SIZE(B) ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI vmovups 0(%rsp), %xmm6 vmovups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/dgemm_tcopy_2.S b/kernel/x86_64/dgemm_tcopy_2.S index 9881610d7d..33504368e1 100644 --- a/kernel/x86_64/dgemm_tcopy_2.S +++ b/kernel/x86_64/dgemm_tcopy_2.S @@ -142,10 +142,10 @@ leaq (, N, SIZE), M8 cmpq $2, N - jl .L40 + jl L(40) ALIGN_4 -.L31: +L(31): subq $2, N movq A, AO1 @@ -157,10 +157,10 @@ movq M, I sarq $3, I - jle .L34 + jle L(34) ALIGN_4 -.L33: +L(33): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 2 * SIZE(AO1) #endif @@ -201,12 +201,12 @@ addq $8 * SIZE, AO2 decq I - jg .L33 + jg L(33) ALIGN_4 -.L34: +L(34): testq $4, M - jle .L36 + jle L(36) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -223,9 +223,9 @@ leaq (B0, M8, 4), B0 ALIGN_4 -.L36: +L(36): testq $2, M - jle .L38 + jle L(38) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(0 * SIZE, AO2, %xmm1) @@ -238,9 +238,9 @@ leaq (B0, M8, 2), B0 ALIGN_4 -.L38: +L(38): testq $1, M - jle .L39 + jle L(39) movsd 0 * SIZE(AO1), %xmm0 movhpd 0 * SIZE(AO2), %xmm0 @@ -249,24 +249,24 @@ subq $-2 * SIZE, B3 ALIGN_4 -.L39: +L(39): cmpq $2, N - jge .L31 + jge L(31) ALIGN_4 -.L40: +L(40): cmpq $1, N - jl .L999 + jl L(999) movq A, AO1 movq B, B0 movq M, I sarq $3, I - jle .L44 + jle L(44) ALIGN_4 -.L43: +L(43): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 4 * SIZE(AO1) #endif @@ -290,12 +290,12 @@ leaq (B0, M8, 4), B0 decq I - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): testq $4, M - jle .L45 + jle L(45) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -307,9 +307,9 @@ leaq (B0, M8, 4), B0 ALIGN_4 -.L45: +L(45): testq $2, M - jle .L46 + jle L(46) MOVUPS_A1(0 * SIZE, AO1, %xmm0) @@ -318,16 +318,16 @@ addq $2 * SIZE, AO1 ALIGN_4 -.L46: +L(46): testq $1, M - jle .L999 + jle L(999) movsd 0 * SIZE(AO1), %xmm0 movlpd %xmm0, -16 * SIZE(B3) ALIGN_4 -.L999: +L(999): popq %rbp popq %r13 popq %r12 diff --git a/kernel/x86_64/dgemm_tcopy_4.S b/kernel/x86_64/dgemm_tcopy_4.S index 98ba6473c5..46c4e3bdd0 100644 --- a/kernel/x86_64/dgemm_tcopy_4.S +++ b/kernel/x86_64/dgemm_tcopy_4.S @@ -152,10 +152,10 @@ leaq (, N, SIZE), M8 cmpq $4, N - jl .L30 + jl L(30) ALIGN_4 -.L21: +L(21): subq $4, N movq A, AO1 @@ -167,10 +167,10 @@ movq M, I sarq $3, I - jle .L24 + jle L(24) ALIGN_4 -.L23: +L(23): #ifdef PREFETCH PREFETCH PREFETCHSIZE * SIZE(AO1) #endif @@ -248,12 +248,12 @@ leaq (B0, M8, 8), B0 decq I - jg .L23 + jg L(23) ALIGN_4 -.L24: +L(24): testq $4, M - jle .L26 + jle L(26) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -280,9 +280,9 @@ leaq (B0, M8, 4), B0 ALIGN_4 -.L26: +L(26): testq $2, M - jle .L28 + jle L(28) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A2(0 * SIZE, AO1, LDA, 1, %xmm1) @@ -299,9 +299,9 @@ subq $-8 * SIZE, B2 ALIGN_4 -.L28: +L(28): testq $1, M - jle .L29 + jle L(29) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -316,14 +316,14 @@ subq $-4 * SIZE, B3 ALIGN_4 -.L29: +L(29): cmpq $4, N - jge .L21 + jge L(21) ALIGN_4 -.L30: +L(30): cmpq $2, N - jl .L40 + jl L(40) subq $2, N @@ -336,10 +336,10 @@ movq M, I sarq $3, I - jle .L34 + jle L(34) ALIGN_4 -.L33: +L(33): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 2 * SIZE(AO1) #endif @@ -381,12 +381,12 @@ leaq (B0, M8, 8), B0 decq I - jg .L33 + jg L(33) ALIGN_4 -.L34: +L(34): testq $4, M - jle .L36 + jle L(36) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -403,9 +403,9 @@ leaq (B0, M8, 4), B0 ALIGN_4 -.L36: +L(36): testq $2, M - jle .L38 + jle L(38) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(0 * SIZE, AO2, %xmm1) @@ -418,9 +418,9 @@ subq $-4 * SIZE, B2 ALIGN_4 -.L38: +L(38): testq $1, M - jle .L40 + jle L(40) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -431,9 +431,9 @@ subq $-2 * SIZE, B3 ALIGN_4 -.L40: +L(40): cmpq $1, N - jl .L999 + jl L(999) movq A, AO1 @@ -441,10 +441,10 @@ movq M, I sarq $3, I - jle .L44 + jle L(44) ALIGN_4 -.L43: +L(43): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 4 * SIZE(AO1) #endif @@ -467,12 +467,12 @@ leaq (B0, M8, 8), B0 decq I - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): testq $4, M - jle .L45 + jle L(45) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -484,9 +484,9 @@ leaq (B0, M8, 4), B0 ALIGN_4 -.L45: +L(45): testq $2, M - jle .L46 + jle L(46) MOVUPS_A1(0 * SIZE, AO1, %xmm0) @@ -496,17 +496,17 @@ subq $-2 * SIZE, B2 ALIGN_4 -.L46: +L(46): testq $1, M - jle .L999 + jle L(999) movsd 0 * SIZE(AO1), %xmm0 movlpd %xmm0, -16 * SIZE(B3) - jmp .L999 + jmp L(999) ALIGN_4 -.L999: +L(999): popq %rbp popq %r12 popq %r13 diff --git a/kernel/x86_64/dgemm_tcopy_8.S b/kernel/x86_64/dgemm_tcopy_8.S index db97db7f18..42240dc404 100644 --- a/kernel/x86_64/dgemm_tcopy_8.S +++ b/kernel/x86_64/dgemm_tcopy_8.S @@ -139,10 +139,10 @@ leaq (, N, SIZE), M8 cmpq $8, N - jl .L20 + jl L(20) ALIGN_4 -.L11: +L(11): subq $8, N movq A, AO1 @@ -154,10 +154,10 @@ movq M, I sarq $3, I - jle .L14 + jle L(14) ALIGN_4 -.L13: +L(13): #ifdef PREFETCH PREFETCH PREFETCHSIZE * SIZE(AO1) #endif @@ -307,12 +307,12 @@ leaq (B0, M8, 8), B0 decq I - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): testq $4, M - jle .L16 + jle L(16) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -359,9 +359,9 @@ subq $-32 * SIZE, B1 ALIGN_4 -.L16: +L(16): testq $2, M - jle .L18 + jle L(18) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A2(0 * SIZE, AO1, LDA, 1, %xmm1) @@ -388,9 +388,9 @@ subq $-16 * SIZE, B2 ALIGN_4 -.L18: +L(18): testq $1, M - jle .L19 + jle L(19) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -417,14 +417,14 @@ subq $-8 * SIZE, B3 ALIGN_4 -.L19: +L(19): cmpq $8, N - jge .L11 + jge L(11) ALIGN_4 -.L20: +L(20): cmpq $4, N - jl .L30 + jl L(30) subq $4, N @@ -437,10 +437,10 @@ movq M, I sarq $3, I - jle .L24 + jle L(24) ALIGN_4 -.L23: +L(23): #ifdef PREFETCH PREFETCH PREFETCHSIZE * SIZE(AO1) #endif @@ -518,12 +518,12 @@ leaq (B0, M8, 8), B0 decq I - jg .L23 + jg L(23) ALIGN_4 -.L24: +L(24): testq $4, M - jle .L26 + jle L(26) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -550,9 +550,9 @@ subq $-16 * SIZE, B1 ALIGN_4 -.L26: +L(26): testq $2, M - jle .L28 + jle L(28) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A2(0 * SIZE, AO1, LDA, 1, %xmm1) @@ -569,9 +569,9 @@ subq $-8 * SIZE, B2 ALIGN_4 -.L28: +L(28): testq $1, M - jle .L30 + jle L(30) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -586,9 +586,9 @@ subq $-4 * SIZE, B3 ALIGN_4 -.L30: +L(30): cmpq $2, N - jl .L40 + jl L(40) subq $2, N @@ -601,10 +601,10 @@ movq M, I sarq $3, I - jle .L34 + jle L(34) ALIGN_4 -.L33: +L(33): #ifdef PREFETCH PREFETCH PREFETCHSIZE * SIZE(AO1) #endif @@ -646,12 +646,12 @@ leaq (B0, M8, 8), B0 decq I - jg .L33 + jg L(33) ALIGN_4 -.L34: +L(34): testq $4, M - jle .L36 + jle L(36) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -668,9 +668,9 @@ subq $-8 * SIZE, B1 ALIGN_4 -.L36: +L(36): testq $2, M - jle .L38 + jle L(38) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(0 * SIZE, AO2, %xmm1) @@ -683,9 +683,9 @@ subq $-4 * SIZE, B2 ALIGN_4 -.L38: +L(38): testq $1, M - jle .L40 + jle L(40) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -696,9 +696,9 @@ subq $-2 * SIZE, B3 ALIGN_4 -.L40: +L(40): cmpq $1, N - jl .L999 + jl L(999) movq A, AO1 @@ -706,10 +706,10 @@ movq M, I sarq $3, I - jle .L44 + jle L(44) ALIGN_4 -.L43: +L(43): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 8 * SIZE(AO1) #endif @@ -732,12 +732,12 @@ leaq (B0, M8, 8), B0 decq I - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): testq $4, M - jle .L45 + jle L(45) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -749,9 +749,9 @@ subq $-4 * SIZE, B1 ALIGN_4 -.L45: +L(45): testq $2, M - jle .L46 + jle L(46) MOVUPS_A1(0 * SIZE, AO1, %xmm0) @@ -761,17 +761,17 @@ subq $-2 * SIZE, B2 ALIGN_4 -.L46: +L(46): testq $1, M - jle .L999 + jle L(999) movsd 0 * SIZE(AO1), %xmm0 movlpd %xmm0, -16 * SIZE(B3) - jmp .L999 + jmp L(999) ALIGN_4 -.L999: +L(999): popq %rbp popq %r12 popq %r13 diff --git a/kernel/x86_64/dgemm_tcopy_8_bulldozer.S b/kernel/x86_64/dgemm_tcopy_8_bulldozer.S index a9dd25389f..1f4a8fb75f 100644 --- a/kernel/x86_64/dgemm_tcopy_8_bulldozer.S +++ b/kernel/x86_64/dgemm_tcopy_8_bulldozer.S @@ -122,10 +122,10 @@ leaq (, N, SIZE), M8 cmpq $8, N - jl .L20 + jl L(20) ALIGN_4 -.L11: +L(11): subq $8, N movq A, AO1 @@ -137,10 +137,10 @@ movq M, I sarq $3, I - jle .L14 + jle L(14) ALIGN_4 -.L13: +L(13): prefetchnta A_PRE(AO1) VMOVUPS_A1(0 * SIZE, AO1, %xmm0) @@ -239,12 +239,12 @@ leaq (B0, M8, 8), B0 decq I - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): testq $4, M - jle .L16 + jle L(16) VMOVUPS_A1(0 * SIZE, AO1, %xmm0) VMOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -291,9 +291,9 @@ subq $-32 * SIZE, B1 ALIGN_4 -.L16: +L(16): testq $2, M - jle .L18 + jle L(18) VMOVUPS_A1(0 * SIZE, AO1, %xmm0) VMOVUPS_A2(0 * SIZE, AO1, LDA, 1, %xmm1) @@ -320,9 +320,9 @@ subq $-16 * SIZE, B2 ALIGN_4 -.L18: +L(18): testq $1, M - jle .L19 + jle L(19) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO1, LDA), %xmm1 @@ -349,14 +349,14 @@ subq $-8 * SIZE, B3 ALIGN_4 -.L19: +L(19): cmpq $8, N - jge .L11 + jge L(11) ALIGN_4 -.L20: +L(20): cmpq $4, N - jl .L30 + jl L(30) subq $4, N @@ -369,10 +369,10 @@ movq M, I sarq $3, I - jle .L24 + jle L(24) ALIGN_4 -.L23: +L(23): VMOVUPS_A1(0 * SIZE, AO1, %xmm0) VMOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -420,12 +420,12 @@ leaq (B0, M8, 8), B0 decq I - jg .L23 + jg L(23) ALIGN_4 -.L24: +L(24): testq $4, M - jle .L26 + jle L(26) VMOVUPS_A1(0 * SIZE, AO1, %xmm0) VMOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -452,9 +452,9 @@ subq $-16 * SIZE, B1 ALIGN_4 -.L26: +L(26): testq $2, M - jle .L28 + jle L(28) VMOVUPS_A1(0 * SIZE, AO1, %xmm0) VMOVUPS_A2(0 * SIZE, AO1, LDA, 1, %xmm1) @@ -471,9 +471,9 @@ subq $-8 * SIZE, B2 ALIGN_4 -.L28: +L(28): testq $1, M - jle .L30 + jle L(30) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO1, LDA), %xmm1 @@ -488,9 +488,9 @@ subq $-4 * SIZE, B3 ALIGN_4 -.L30: +L(30): cmpq $2, N - jl .L40 + jl L(40) subq $2, N @@ -503,10 +503,10 @@ movq M, I sarq $3, I - jle .L34 + jle L(34) ALIGN_4 -.L33: +L(33): VMOVUPS_A1(0 * SIZE, AO1, %xmm0) VMOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -533,12 +533,12 @@ leaq (B0, M8, 8), B0 decq I - jg .L33 + jg L(33) ALIGN_4 -.L34: +L(34): testq $4, M - jle .L36 + jle L(36) VMOVUPS_A1(0 * SIZE, AO1, %xmm0) VMOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -555,9 +555,9 @@ subq $-8 * SIZE, B1 ALIGN_4 -.L36: +L(36): testq $2, M - jle .L38 + jle L(38) VMOVUPS_A1(0 * SIZE, AO1, %xmm0) VMOVUPS_A1(0 * SIZE, AO2, %xmm1) @@ -570,9 +570,9 @@ subq $-4 * SIZE, B2 ALIGN_4 -.L38: +L(38): testq $1, M - jle .L40 + jle L(40) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO2), %xmm1 @@ -583,9 +583,9 @@ subq $-2 * SIZE, B3 ALIGN_4 -.L40: +L(40): cmpq $1, N - jl .L999 + jl L(999) movq A, AO1 @@ -593,10 +593,10 @@ movq M, I sarq $3, I - jle .L44 + jle L(44) ALIGN_4 -.L43: +L(43): VMOVUPS_A1(0 * SIZE, AO1, %xmm0) VMOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -612,12 +612,12 @@ leaq (B0, M8, 8), B0 decq I - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): testq $4, M - jle .L45 + jle L(45) VMOVUPS_A1(0 * SIZE, AO1, %xmm0) VMOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -629,9 +629,9 @@ subq $-4 * SIZE, B1 ALIGN_4 -.L45: +L(45): testq $2, M - jle .L46 + jle L(46) VMOVUPS_A1(0 * SIZE, AO1, %xmm0) @@ -641,17 +641,17 @@ subq $-2 * SIZE, B2 ALIGN_4 -.L46: +L(46): testq $1, M - jle .L999 + jle L(999) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd %xmm0, -16 * SIZE(B3) - jmp .L999 + jmp L(999) ALIGN_4 -.L999: +L(999): popq %rbp popq %r12 popq %r13 diff --git a/kernel/x86_64/dgemv_n.S b/kernel/x86_64/dgemv_n.S index 58dd43bbb1..aa139a042a 100644 --- a/kernel/x86_64/dgemv_n.S +++ b/kernel/x86_64/dgemv_n.S @@ -164,20 +164,20 @@ movq LDA,LDAX movq X,XX -.L0t: +L(0t): xorq I,I addq $1,I salq $21,I subq I,MMM movq I,M - jge .L00t + jge L(00t) movq MMM,M addq M, I - jle .L999x + jle L(999x) movq I, M -.L00t: +L(00t): movq XX,X movq AA,A movq NN,N @@ -205,9 +205,9 @@ #endif testq N, N # if n <= 0 goto END - jle .L999 + jle L(999) testq M, M # if n <= 0 goto END - jle .L999 + jle L(999) #if !defined(COPY_FORCE) && !defined(ALIGNED_ACCESS) #ifndef NOCOPY_UNALIGNED @@ -217,7 +217,7 @@ #endif testq %rax, %rax cmoveq Y, BUFFER - je .L10 + je L(10) #endif movq BUFFER, Y1 @@ -229,7 +229,7 @@ sarq $4, %rax ALIGN_3 -.L01: +L(01): movapd %xmm4, 0 * SIZE(Y1) movapd %xmm4, 2 * SIZE(Y1) movapd %xmm4, 4 * SIZE(Y1) @@ -240,10 +240,10 @@ movapd %xmm4, 14 * SIZE(Y1) subq $-16 * SIZE, Y1 decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L10: +L(10): #ifdef ALIGNED_ACCESS leaq SIZE(BUFFER), %rax @@ -251,16 +251,16 @@ cmovne %rax, BUFFER testq $SIZE, LDA - jne .L50 + jne L(50) #endif #if GEMV_UNROLL >= 8 cmpq $8, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $8, N leaq 16 * SIZE(BUFFER), Y1 @@ -328,7 +328,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L1X + je L(1X) movsd -16 * SIZE(A1), %xmm4 movsd -16 * SIZE(A1, LDA), %xmm5 @@ -366,12 +366,12 @@ addq $SIZE, Y1 ALIGN_3 -.L1X: +L(1X): #endif movq MM, I sarq $3, I - jle .L15 + jle L(15) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm5) @@ -384,10 +384,10 @@ MOVUPS_YL1(-10 * SIZE, Y1, %xmm3) decq I - jle .L14 + jle L(14) ALIGN_3 -.L13: +L(13): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -552,10 +552,10 @@ subq $1, I BRANCH - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): mulpd %xmm8, %xmm4 addpd %xmm4, %xmm0 MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm4) @@ -673,9 +673,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L15: +L(15): testq $4, MM - je .L16 + je L(16) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm5) @@ -744,9 +744,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L16: +L(16): testq $2, MM - je .L17 + je L(17) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm5) @@ -785,9 +785,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L17: +L(17): testq $1, MM - je .L18 + je L(18) movsd -16 * SIZE(A1), %xmm4 movsd -16 * SIZE(A1, LDA), %xmm5 @@ -821,23 +821,23 @@ movsd %xmm0, -16 * SIZE(Y1) ALIGN_3 -.L18: +L(18): cmpq $8, N - jge .L11 + jge L(11) ALIGN_3 -.L20: +L(20): #endif #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 4 ALIGN_3 -.L21: +L(21): #endif subq $4, N @@ -883,7 +883,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L2X + je L(2X) movsd -16 * SIZE(A1), %xmm4 movsd -16 * SIZE(A1, LDA), %xmm5 @@ -908,12 +908,12 @@ addq $SIZE, Y1 ALIGN_3 -.L2X: +L(2X): #endif movq MM, I sarq $3, I - jle .L25 + jle L(25) MOVUPS_A1(-16 * SIZE, A1, %xmm0) MOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -931,10 +931,10 @@ MOVUPS_A2(-10 * SIZE, A1, LDA, 1, %xmm7) decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -1027,10 +1027,10 @@ subq $1, I BRANCH - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): mulpd %xmm12, %xmm0 addpd %xmm0, %xmm8 MOVUPS_A1(-16 * SIZE, A2, %xmm0) @@ -1088,9 +1088,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L25: +L(25): testq $4, MM - je .L26 + je L(26) MOVUPS_A1(-16 * SIZE, A1, %xmm0) MOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -1135,9 +1135,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L26: +L(26): testq $2, MM - je .L27 + je L(27) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm9) @@ -1162,12 +1162,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L27: +L(27): testq $1, MM #if GEMV_UNROLL == 4 - je .L28 + je L(28) #else - je .L30 + je L(30) #endif movsd -16 * SIZE(Y1), %xmm0 @@ -1190,25 +1190,25 @@ ALIGN_3 #if GEMV_UNROLL == 4 -.L28: +L(28): cmpq $4, N - jge .L21 + jge L(21) ALIGN_3 #endif -.L30: +L(30): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L40 + jl L(40) #if GEMV_UNROLL == 2 ALIGN_3 -.L31: +L(31): #endif subq $2, N @@ -1242,7 +1242,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L3X + je L(3X) movsd -16 * SIZE(A1), %xmm4 movsd -16 * SIZE(A2), %xmm5 @@ -1261,12 +1261,12 @@ addq $SIZE, Y1 ALIGN_3 -.L3X: +L(3X): #endif movq MM, I sarq $3, I - jle .L35 + jle L(35) MOVUPS_A1(-16 * SIZE, A1, %xmm0) MOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -1284,10 +1284,10 @@ MOVUPS_A1(-10 * SIZE, A2, %xmm7) decq I - jle .L34 + jle L(34) ALIGN_3 -.L33: +L(33): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -1344,10 +1344,10 @@ subq $1, I BRANCH - jg .L33 + jg L(33) ALIGN_3 -.L34: +L(34): mulpd %xmm12, %xmm0 addpd %xmm0, %xmm8 mulpd %xmm12, %xmm1 @@ -1375,9 +1375,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L35: +L(35): testq $4, MM - je .L36 + je L(36) MOVUPS_A1(-16 * SIZE, A1, %xmm0) @@ -1406,9 +1406,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L36: +L(36): testq $2, MM - je .L37 + je L(37) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A1(-16 * SIZE, A2, %xmm9) @@ -1427,12 +1427,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L37: +L(37): testq $1, MM #if GEMV_UNROLL == 2 - je .L38 + je L(38) #else - je .L40 + je L(40) #endif movsd -16 * SIZE(Y1), %xmm0 @@ -1449,16 +1449,16 @@ ALIGN_3 #if GEMV_UNROLL == 2 -.L38: +L(38): cmpq $2, N - jge .L31 + jge L(31) ALIGN_3 #endif -.L40: +L(40): cmpq $1, N - jl .L900 + jl L(900) #endif leaq 16 * SIZE(BUFFER), Y1 @@ -1482,7 +1482,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L4X + je L(4X) movsd -16 * SIZE(A1), %xmm4 movsd -16 * SIZE(Y1), %xmm0 @@ -1496,12 +1496,12 @@ addq $SIZE, Y1 ALIGN_3 -.L4X: +L(4X): #endif movq MM, I sarq $3, I - jle .L45 + jle L(45) MOVUPS_A1(-16 * SIZE, A1, %xmm0) MOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -1514,10 +1514,10 @@ MOVUPS_YL1(-10 * SIZE, Y1, %xmm11) decq I - jle .L44 + jle L(44) ALIGN_3 -.L43: +L(43): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -1555,10 +1555,10 @@ subq $1, I BRANCH - jg .L43 + jg L(43) ALIGN_3 -.L44: +L(44): mulpd %xmm12, %xmm0 addpd %xmm0, %xmm8 MOVUPS_YS1(-16 * SIZE, Y1, %xmm8) @@ -1576,9 +1576,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L45: +L(45): testq $4, MM - je .L46 + je L(46) MOVUPS_A1(-16 * SIZE, A1, %xmm0) MOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -1597,9 +1597,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L46: +L(46): testq $2, MM - je .L47 + je L(47) MOVUPS_A1(-16 * SIZE, A1, %xmm8) @@ -1614,9 +1614,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L47: +L(47): testq $1, MM - je .L900 + je L(900) movsd -16 * SIZE(Y1), %xmm0 movsd -16 * SIZE(A1), %xmm8 @@ -1628,17 +1628,17 @@ ALIGN_3 #ifdef ALIGNED_ACCESS - jmp .L900 + jmp L(900) ALIGN_3 -.L50: +L(50): #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L60 + jl L(60) ALIGN_3 -.L51: +L(51): subq $4, N @@ -1682,7 +1682,7 @@ mulpd %xmm0, %xmm15 testq $SIZE, A - je .L5X + je L(5X) movsd -16 * SIZE(A1), %xmm4 movsd -16 * SIZE(A1, LDA), %xmm5 @@ -1707,13 +1707,13 @@ addq $SIZE, Y1 ALIGN_3 -.L5X: +L(5X): movhpd -16 * SIZE(A1, LDA), %xmm8 movhpd -16 * SIZE(A2, LDA), %xmm9 movq MM, I sarq $3, I - jle .L55 + jle L(55) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm5) @@ -1725,10 +1725,10 @@ MOVUPS_YL1(-10 * SIZE, Y1, %xmm3) decq I - jle .L54 + jle L(54) ALIGN_3 -.L53: +L(53): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -1829,10 +1829,10 @@ subq $1, I BRANCH - jg .L53 + jg L(53) ALIGN_3 -.L54: +L(54): mulpd %xmm12, %xmm4 addpd %xmm4, %xmm0 MOVUPS_A1(-10 * SIZE, A1, %xmm7) @@ -1904,9 +1904,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L55: +L(55): testq $4, MM - je .L56 + je L(56) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm5) @@ -1957,9 +1957,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L56: +L(56): testq $2, MM - je .L57 + je L(57) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A2(-15 * SIZE, A1, LDA, 1, %xmm5) @@ -1988,9 +1988,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L57: +L(57): testq $1, MM - je .L58 + je L(58) movsd -16 * SIZE(Y1), %xmm0 @@ -2011,23 +2011,23 @@ movsd %xmm0, -16 * SIZE(Y1) ALIGN_3 -.L58: +L(58): cmpq $4, N - jge .L51 + jge L(51) ALIGN_3 -.L60: +L(60): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L70 + jl L(70) #if GEMV_UNROLL == 2 ALIGN_3 -.L61: +L(61): #endif subq $2, N @@ -2060,7 +2060,7 @@ mulpd %xmm0, %xmm13 testq $SIZE, A - je .L6X + je L(6X) movsd -16 * SIZE(A1), %xmm4 movsd -16 * SIZE(A2), %xmm5 @@ -2079,12 +2079,12 @@ addq $SIZE, Y1 ALIGN_3 -.L6X: +L(6X): movhpd -16 * SIZE(A2), %xmm8 movq MM, I sarq $3, I - jle .L65 + jle L(65) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm5) @@ -2096,10 +2096,10 @@ MOVUPS_YL1(-10 * SIZE, Y1, %xmm3) decq I - jle .L64 + jle L(64) ALIGN_3 -.L63: +L(63): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -2160,10 +2160,10 @@ subq $1, I BRANCH - jg .L63 + jg L(63) ALIGN_3 -.L64: +L(64): mulpd %xmm12, %xmm4 addpd %xmm4, %xmm0 MOVUPS_A1(-10 * SIZE, A1, %xmm7) @@ -2203,9 +2203,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L65: +L(65): testq $4, MM - je .L66 + je L(66) MOVUPS_A1(-16 * SIZE, A1, %xmm4) @@ -2238,9 +2238,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L66: +L(66): testq $2, MM - je .L67 + je L(67) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-15 * SIZE, A2, %xmm5) @@ -2261,12 +2261,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L67: +L(67): testq $1, MM #if GEMV_UNROLL == 2 - je .L68 + je L(68) #else - je .L70 + je L(70) #endif movsd -16 * SIZE(Y1), %xmm0 @@ -2283,16 +2283,16 @@ ALIGN_3 #if GEMV_UNROLL == 2 -.L68: +L(68): cmpq $2, N - jge .L61 + jge L(61) ALIGN_3 #endif -.L70: +L(70): cmpq $1, N - jl .L900 + jl L(900) #endif @@ -2316,7 +2316,7 @@ mulpd %xmm0, %xmm12 testq $SIZE, A - je .L7X + je L(7X) movsd -16 * SIZE(A1), %xmm4 movsd -16 * SIZE(Y1), %xmm0 @@ -2330,11 +2330,11 @@ addq $SIZE, Y1 ALIGN_3 -.L7X: +L(7X): movq MM, I sarq $3, I - jle .L75 + jle L(75) MOVUPS_A1(-16 * SIZE, A1, %xmm0) MOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -2347,10 +2347,10 @@ MOVUPS_YL1(-10 * SIZE, Y1, %xmm11) decq I - jle .L74 + jle L(74) ALIGN_3 -.L73: +L(73): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -2388,10 +2388,10 @@ subq $1, I BRANCH - jg .L73 + jg L(73) ALIGN_3 -.L74: +L(74): mulpd %xmm12, %xmm0 addpd %xmm0, %xmm8 MOVUPS_YS1(-16 * SIZE, Y1, %xmm8) @@ -2409,9 +2409,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L75: +L(75): testq $4, MM - je .L76 + je L(76) MOVUPS_A1(-16 * SIZE, A1, %xmm0) MOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -2430,9 +2430,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L76: +L(76): testq $2, MM - je .L77 + je L(77) MOVUPS_A1(-16 * SIZE, A1, %xmm8) @@ -2447,9 +2447,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L77: +L(77): testq $1, MM - je .L900 + je L(900) movsd -16 * SIZE(Y1), %xmm0 movsd -16 * SIZE(A1), %xmm8 @@ -2462,19 +2462,19 @@ ALIGN_3 -.L900: +L(900): #ifndef COPY_FORCE cmpq Y, BUFFER - je .L999 + je L(999) #endif movq M, TMP_M movq Y, Y1 cmpq $SIZE, INCY - jne .L950 + jne L(950) testq $SIZE, Y1 - je .L910 + je L(910) movsd (Y1), %xmm0 addsd (BUFFER), %xmm0 @@ -2484,19 +2484,19 @@ addq $SIZE, BUFFER decq TMP_M - jle .L999 + jle L(999) ALIGN_4 -.L910: +L(910): testq $SIZE, BUFFER - jne .L920 + jne L(920) movq TMP_M, %rax sarq $3, %rax - jle .L914 + jle L(914) ALIGN_3 -.L912: +L(912): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE) * 4 + PREOFFSET(Y1) #endif @@ -2529,15 +2529,15 @@ addq $8 * SIZE, BUFFER decq %rax - jg .L912 + jg L(912) ALIGN_3 -.L914: +L(914): testq $7, TMP_M - jle .L999 + jle L(999) testq $4, TMP_M - jle .L915 + jle L(915) movapd 0 * SIZE(Y1), %xmm0 movapd 2 * SIZE(Y1), %xmm1 @@ -2555,9 +2555,9 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L915: +L(915): testq $2, TMP_M - jle .L916 + jle L(916) movapd (Y1), %xmm0 @@ -2571,9 +2571,9 @@ addq $2 * SIZE, BUFFER ALIGN_3 -.L916: +L(916): testq $1, TMP_M - jle .L999 + jle L(999) movsd (Y1), %xmm0 @@ -2584,18 +2584,18 @@ movlpd %xmm0, (Y1) ALIGN_3 - jmp .L999 + jmp L(999) ALIGN_4 -.L920: +L(920): movapd -1 * SIZE(BUFFER), %xmm4 movq TMP_M, %rax sarq $3, %rax - jle .L924 + jle L(924) ALIGN_3 -.L922: +L(922): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE) * 4 + PREOFFSET(Y1) #endif @@ -2635,15 +2635,15 @@ addq $8 * SIZE, BUFFER decq %rax - jg .L922 + jg L(922) ALIGN_3 -.L924: +L(924): testq $7, TMP_M - jle .L999 + jle L(999) testq $4, TMP_M - jle .L925 + jle L(925) movapd 0 * SIZE(Y1), %xmm0 movapd 2 * SIZE(Y1), %xmm1 @@ -2666,9 +2666,9 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L925: +L(925): testq $2, TMP_M - jle .L926 + jle L(926) movapd (Y1), %xmm0 @@ -2686,9 +2686,9 @@ addq $2 * SIZE, BUFFER ALIGN_3 -.L926: +L(926): testq $1, TMP_M - jle .L999 + jle L(999) movsd (Y1), %xmm0 @@ -2699,12 +2699,12 @@ movlpd %xmm0, (Y1) ALIGN_3 - jmp .L999 + jmp L(999) ALIGN_4 -.L950: +L(950): testq $SIZE, BUFFER - je .L960 + je L(960) movsd (Y1), %xmm0 addsd (BUFFER), %xmm0 @@ -2714,18 +2714,18 @@ addq $SIZE, BUFFER decq TMP_M - jle .L999 + jle L(999) ALIGN_4 -.L960: +L(960): movq Y1, Y2 movq TMP_M, %rax sarq $3, %rax - jle .L964 + jle L(964) ALIGN_3 -.L962: +L(962): movsd (Y2), %xmm0 addq INCY, Y2 movhpd (Y2), %xmm0 @@ -2783,15 +2783,15 @@ addq $8 * SIZE, BUFFER decq %rax - jg .L962 + jg L(962) ALIGN_3 -.L964: +L(964): testq $7, TMP_M - jle .L999 + jle L(999) testq $4, TMP_M - jle .L965 + jle L(965) movsd (Y2), %xmm0 addq INCY, Y2 @@ -2822,9 +2822,9 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L965: +L(965): testq $2, TMP_M - jle .L966 + jle L(966) movsd (Y2), %xmm0 addq INCY, Y2 @@ -2843,9 +2843,9 @@ addq $2 * SIZE, BUFFER ALIGN_3 -.L966: +L(966): testq $1, TMP_M - jle .L999 + jle L(999) movsd (Y2), %xmm0 @@ -2856,16 +2856,16 @@ movlpd %xmm0, (Y1) ALIGN_3 -.L999: +L(999): leaq (, M, SIZE), %rax addq %rax,AA movq STACK_INCY, INCY imulq INCY, %rax addq %rax, Y - jmp .L0t + jmp L(0t) ALIGN_4 -.L999x: +L(999x): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/dgemv_n_atom.S b/kernel/x86_64/dgemv_n_atom.S index ed6a585795..1e0957b661 100644 --- a/kernel/x86_64/dgemv_n_atom.S +++ b/kernel/x86_64/dgemv_n_atom.S @@ -143,13 +143,13 @@ leaq (,LDA, SIZE), LDA testq N, N - jle .L999 + jle L(999) testq M, M - jle .L999 + jle L(999) cmpq $SIZE, INCY cmoveq Y, BUFFER - je .L10 + je L(10) movq BUFFER, Y1 xorps %xmm4, %xmm4 @@ -159,7 +159,7 @@ sarq $3, %rax ALIGN_3 -.L01: +L(01): movsd %xmm4, 0 * SIZE(Y1) movsd %xmm4, 1 * SIZE(Y1) movsd %xmm4, 2 * SIZE(Y1) @@ -171,16 +171,16 @@ addq $8 * SIZE, Y1 decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L10: +L(10): movq N, J sarq $1, J - jle .L20 + jle L(20) ALIGN_3 -.L11: +L(11): movq BUFFER, Y1 movq A, A1 @@ -199,7 +199,7 @@ movq M, I sarq $3, I - jle .L15 + jle L(15) movsd 0 * SIZE(A1), %xmm0 movsd 1 * SIZE(A1), %xmm1 @@ -221,10 +221,10 @@ mulsd %xmm14, %xmm3 decq I - jle .L14 + jle L(14) ALIGN_3 -.L13: +L(13): PREFETCH PREFETCH_SIZE * SIZE(A1) mulsd %xmm15, %xmm4 PREFETCH PREFETCH_SIZE * SIZE(A2) @@ -314,10 +314,10 @@ movsd 2 * SIZE(Y1), %xmm10 movsd %xmm11,-1 * SIZE(Y1) movsd 3 * SIZE(Y1), %xmm11 - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): mulsd %xmm15, %xmm4 addsd %xmm0, %xmm8 movsd 4 * SIZE(A1), %xmm0 @@ -393,9 +393,9 @@ movsd %xmm11,-1 * SIZE(Y1) ALIGN_3 -.L15: +L(15): testq $4, M - je .L17 + je L(17) movsd 0 * SIZE(A1), %xmm0 movsd 1 * SIZE(A1), %xmm1 @@ -440,9 +440,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L17: +L(17): testq $2, M - je .L18 + je L(18) movsd 0 * SIZE(A1), %xmm0 movsd 1 * SIZE(A1), %xmm1 @@ -470,9 +470,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L18: +L(18): testq $1, M - je .L19 + je L(19) movsd 0 * SIZE(Y1), %xmm8 @@ -488,14 +488,14 @@ movsd %xmm8, 0 * SIZE(Y1) ALIGN_3 -.L19: +L(19): decq J - jg .L11 + jg L(11) ALIGN_3 -.L20: +L(20): testq $1, N - je .L990 + je L(990) movq BUFFER, Y1 movq A, A1 @@ -505,7 +505,7 @@ movq M, I sarq $3, I - jle .L25 + jle L(25) movsd 0 * SIZE(A1), %xmm0 movsd 1 * SIZE(A1), %xmm1 @@ -527,10 +527,10 @@ mulsd %xmm14, %xmm3 decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): PREFETCH PREFETCH_SIZE * SIZE(A1) addsd %xmm0, %xmm8 @@ -583,10 +583,10 @@ addq $8 * SIZE, A1 decq I - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): addsd %xmm0, %xmm8 addsd %xmm1, %xmm9 addsd %xmm2, %xmm10 @@ -622,9 +622,9 @@ addq $8 * SIZE, A1 ALIGN_3 -.L25: +L(25): testq $4, M - je .L27 + je L(27) movsd 0 * SIZE(A1), %xmm0 movsd 1 * SIZE(A1), %xmm1 @@ -655,9 +655,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L27: +L(27): testq $2, M - je .L28 + je L(28) movsd 0 * SIZE(A1), %xmm0 movsd 1 * SIZE(A1), %xmm1 @@ -677,9 +677,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L28: +L(28): testq $1, M - je .L990 + je L(990) movsd 0 * SIZE(Y1), %xmm8 movsd 0 * SIZE(A1), %xmm0 @@ -690,18 +690,18 @@ movsd %xmm8, 0 * SIZE(Y1) ALIGN_3 -.L990: +L(990): cmpq $SIZE, INCY - je .L999 + je L(999) movq Y, Y1 movq M, %rax sarq $2, %rax - jle .L994 + jle L(994) ALIGN_3 -.L992: +L(992): movsd (Y), %xmm0 addq INCY, Y movsd (Y), %xmm1 @@ -727,12 +727,12 @@ addq INCY, Y1 decq %rax - jg .L992 + jg L(992) ALIGN_3 -.L994: +L(994): testq $2, M - jle .L996 + jle L(996) movsd (Y), %xmm0 addq INCY, Y @@ -749,9 +749,9 @@ addq INCY, Y1 ALIGN_3 -.L996: +L(996): testq $1, M - jle .L999 + jle L(999) movsd (Y), %xmm0 @@ -760,7 +760,7 @@ movsd %xmm0, (Y1) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/dgemv_n_bulldozer.S b/kernel/x86_64/dgemv_n_bulldozer.S index bc00d67fc5..24ea327a27 100644 --- a/kernel/x86_64/dgemv_n_bulldozer.S +++ b/kernel/x86_64/dgemv_n_bulldozer.S @@ -174,9 +174,9 @@ #endif testq N, N # if n <= 0 goto END - jle .L999 + jle L(999) testq M, M # if n <= 0 goto END - jle .L999 + jle L(999) #if !defined(COPY_FORCE) && !defined(ALIGNED_ACCESS) #ifndef NOCOPY_UNALIGNED @@ -186,7 +186,7 @@ #endif testq %rax, %rax cmoveq Y, BUFFER - je .L10 + je L(10) #endif movq BUFFER, Y1 @@ -198,7 +198,7 @@ sarq $4, %rax ALIGN_3 -.L01: +L(01): vmovups %xmm4, 0 * SIZE(Y1) vmovups %xmm4, 2 * SIZE(Y1) vmovups %xmm4, 4 * SIZE(Y1) @@ -209,10 +209,10 @@ vmovups %xmm4, 14 * SIZE(Y1) subq $-16 * SIZE, Y1 decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L10: +L(10): #ifdef ALIGNED_ACCESS leaq SIZE(BUFFER), %rax @@ -220,16 +220,16 @@ cmovne %rax, BUFFER testq $SIZE, LDA - jne .L50 + jne L(50) #endif #if GEMV_UNROLL >= 8 cmpq $8, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $8, N leaq 16 * SIZE(BUFFER), Y1 @@ -267,7 +267,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L1X + je L(1X) vmovsd -16 * SIZE(Y1), %xmm0 @@ -299,12 +299,12 @@ addq $SIZE, Y1 ALIGN_3 -.L1X: +L(1X): #endif movq MM, I sarq $3, I - jle .L15 + jle L(15) VMOVUPS_YL1(-16 * SIZE, Y1, %xmm0) VMOVUPS_YL1(-14 * SIZE, Y1, %xmm1) @@ -312,10 +312,10 @@ VMOVUPS_YL1(-10 * SIZE, Y1, %xmm3) decq I - jle .L14 + jle L(14) ALIGN_5 -.L13: +L(13): prefetchnta A_PRE(A1) @@ -387,10 +387,10 @@ subq $1, I BRANCH - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm8, %xmm0 vfmaddpd %xmm1 , -14 * SIZE(A1) , %xmm8, %xmm1 @@ -445,9 +445,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L15: +L(15): testq $4, MM - je .L16 + je L(16) VMOVUPS_A1(-16 * SIZE, A1, %xmm4) VMOVUPS_A1(-14 * SIZE, A1, %xmm5) @@ -501,9 +501,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L16: +L(16): testq $2, MM - je .L17 + je L(17) VMOVUPS_A1(-16 * SIZE, A1, %xmm4) VMOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm5) @@ -534,9 +534,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L17: +L(17): testq $1, MM - je .L18 + je L(18) vmovsd -16 * SIZE(A1), %xmm4 vmovsd -16 * SIZE(A1, LDA), %xmm5 @@ -565,23 +565,23 @@ vmovsd %xmm0, -16 * SIZE(Y1) ALIGN_3 -.L18: +L(18): cmpq $8, N - jge .L11 + jge L(11) ALIGN_3 -.L20: +L(20): #endif #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 4 ALIGN_3 -.L21: +L(21): #endif subq $4, N @@ -609,7 +609,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L2X + je L(2X) vmovsd -16 * SIZE(A1), %xmm4 vmovsd -16 * SIZE(A1, LDA), %xmm5 @@ -630,12 +630,12 @@ addq $SIZE, Y1 ALIGN_3 -.L2X: +L(2X): #endif movq MM, I sarq $3, I - jle .L25 + jle L(25) VMOVUPS_A1(-16 * SIZE, A1, %xmm0) VMOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -653,10 +653,10 @@ VMOVUPS_A2(-10 * SIZE, A1, LDA, 1, %xmm7) decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): @@ -722,10 +722,10 @@ subq $1, I BRANCH - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): vfmaddpd %xmm8 , %xmm12, %xmm0 , %xmm8 vfmaddpd %xmm9 , %xmm12, %xmm1 , %xmm9 @@ -767,9 +767,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L25: +L(25): testq $4, MM - je .L26 + je L(26) VMOVUPS_A1(-16 * SIZE, A1, %xmm0) VMOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -806,9 +806,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L26: +L(26): testq $2, MM - je .L27 + je L(27) VMOVUPS_A1(-16 * SIZE, A1, %xmm8) VMOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm9) @@ -829,12 +829,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L27: +L(27): testq $1, MM #if GEMV_UNROLL == 4 - je .L28 + je L(28) #else - je .L30 + je L(30) #endif vmovsd -16 * SIZE(Y1), %xmm0 @@ -853,25 +853,25 @@ ALIGN_3 #if GEMV_UNROLL == 4 -.L28: +L(28): cmpq $4, N - jge .L21 + jge L(21) ALIGN_3 #endif -.L30: +L(30): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L40 + jl L(40) #if GEMV_UNROLL == 2 ALIGN_3 -.L31: +L(31): #endif subq $2, N @@ -893,7 +893,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L3X + je L(3X) vmovsd -16 * SIZE(A1), %xmm4 vmovsd -16 * SIZE(A2), %xmm5 @@ -910,12 +910,12 @@ addq $SIZE, Y1 ALIGN_3 -.L3X: +L(3X): #endif movq MM, I sarq $3, I - jle .L35 + jle L(35) VMOVUPS_A1(-16 * SIZE, A1, %xmm0) VMOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -933,10 +933,10 @@ VMOVUPS_A1(-10 * SIZE, A2, %xmm7) decq I - jle .L34 + jle L(34) ALIGN_3 -.L33: +L(33): vfmaddpd %xmm8 , %xmm12 , %xmm0 , %xmm8 @@ -980,10 +980,10 @@ subq $1, I BRANCH - jg .L33 + jg L(33) ALIGN_3 -.L34: +L(34): vfmaddpd %xmm8 , %xmm12 , %xmm0 , %xmm8 vfmaddpd %xmm9 , %xmm12 , %xmm1 , %xmm9 vfmaddpd %xmm10, %xmm12 , %xmm2 , %xmm10 @@ -1004,9 +1004,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L35: +L(35): testq $4, MM - je .L36 + je L(36) VMOVUPS_A1(-16 * SIZE, A1, %xmm0) @@ -1032,9 +1032,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L36: +L(36): testq $2, MM - je .L37 + je L(37) VMOVUPS_A1(-16 * SIZE, A1, %xmm8) VMOVUPS_A1(-16 * SIZE, A2, %xmm9) @@ -1051,12 +1051,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L37: +L(37): testq $1, MM #if GEMV_UNROLL == 2 - je .L38 + je L(38) #else - je .L40 + je L(40) #endif vmovsd -16 * SIZE(Y1), %xmm0 @@ -1071,16 +1071,16 @@ ALIGN_3 #if GEMV_UNROLL == 2 -.L38: +L(38): cmpq $2, N - jge .L31 + jge L(31) ALIGN_3 #endif -.L40: +L(40): cmpq $1, N - jl .L900 + jl L(900) #endif leaq 16 * SIZE(BUFFER), Y1 @@ -1095,7 +1095,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L4X + je L(4X) vmovsd -16 * SIZE(A1), %xmm4 vmovsd -16 * SIZE(Y1), %xmm0 @@ -1108,12 +1108,12 @@ addq $SIZE, Y1 ALIGN_3 -.L4X: +L(4X): #endif movq MM, I sarq $3, I - jle .L45 + jle L(45) VMOVUPS_A1(-16 * SIZE, A1, %xmm0) VMOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -1126,10 +1126,10 @@ VMOVUPS_YL1(-10 * SIZE, Y1, %xmm11) decq I - jle .L44 + jle L(44) ALIGN_3 -.L43: +L(43): vfmaddpd %xmm8 , %xmm12 , %xmm0 , %xmm8 @@ -1158,10 +1158,10 @@ subq $1, I BRANCH - jg .L43 + jg L(43) ALIGN_3 -.L44: +L(44): vfmaddpd %xmm8 , %xmm12 , %xmm0 , %xmm8 vfmaddpd %xmm9 , %xmm12 , %xmm1 , %xmm9 @@ -1177,9 +1177,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L45: +L(45): testq $4, MM - je .L46 + je L(46) VMOVUPS_A1(-16 * SIZE, A1, %xmm0) VMOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -1197,9 +1197,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L46: +L(46): testq $2, MM - je .L47 + je L(47) VMOVUPS_A1(-16 * SIZE, A1, %xmm8) @@ -1213,9 +1213,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L47: +L(47): testq $1, MM - je .L900 + je L(900) vmovsd -16 * SIZE(Y1), %xmm0 vmovsd -16 * SIZE(A1), %xmm8 @@ -1226,17 +1226,17 @@ ALIGN_3 #ifdef ALIGNED_ACCESS - jmp .L900 + jmp L(900) ALIGN_3 -.L50: +L(50): #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L60 + jl L(60) ALIGN_3 -.L51: +L(51): subq $4, N @@ -1262,7 +1262,7 @@ vmulpd %xmm0, %xmm15 , %xmm15 testq $SIZE, A - je .L5X + je L(5X) vmovsd -16 * SIZE(A1), %xmm4 vmovsd -16 * SIZE(A1, LDA), %xmm5 @@ -1283,13 +1283,13 @@ addq $SIZE, Y1 ALIGN_3 -.L5X: +L(5X): vmovhpd -16 * SIZE(A1, LDA), %xmm8, %xmm8 vmovhpd -16 * SIZE(A2, LDA), %xmm9, %xmm9 movq MM, I sarq $3, I - jle .L55 + jle L(55) VMOVUPS_A1(-16 * SIZE, A1, %xmm4) VMOVUPS_A1(-14 * SIZE, A1, %xmm5) @@ -1301,10 +1301,10 @@ VMOVUPS_YL1(-10 * SIZE, Y1, %xmm3) decq I - jle .L54 + jle L(54) ALIGN_3 -.L53: +L(53): @@ -1381,11 +1381,11 @@ subq $1, I BRANCH - jg .L53 + jg L(53) ALIGN_3 -.L54: +L(54): vfmaddpd %xmm0 , %xmm12 , %xmm4 , %xmm0 VMOVUPS_A1(-10 * SIZE, A1, %xmm7) @@ -1442,9 +1442,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L55: +L(55): testq $4, MM - je .L56 + je L(56) VMOVUPS_A1(-16 * SIZE, A1, %xmm4) VMOVUPS_A1(-14 * SIZE, A1, %xmm5) @@ -1487,9 +1487,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L56: +L(56): testq $2, MM - je .L57 + je L(57) VMOVUPS_A1(-16 * SIZE, A1, %xmm4) VMOVUPS_A2(-15 * SIZE, A1, LDA, 1, %xmm5) @@ -1514,9 +1514,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L57: +L(57): testq $1, MM - je .L58 + je L(58) vmovsd -16 * SIZE(Y1), %xmm0 @@ -1533,23 +1533,23 @@ vmovsd %xmm0, -16 * SIZE(Y1) ALIGN_3 -.L58: +L(58): cmpq $4, N - jge .L51 + jge L(51) ALIGN_3 -.L60: +L(60): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L70 + jl L(70) #if GEMV_UNROLL == 2 ALIGN_3 -.L61: +L(61): #endif subq $2, N @@ -1570,7 +1570,7 @@ vmulpd %xmm0, %xmm13 , %xmm13 testq $SIZE, A - je .L6X + je L(6X) vmovsd -16 * SIZE(A1), %xmm4 vmovsd -16 * SIZE(A2), %xmm5 @@ -1587,12 +1587,12 @@ addq $SIZE, Y1 ALIGN_3 -.L6X: +L(6X): vmovhpd -16 * SIZE(A2), %xmm8, %xmm8 movq MM, I sarq $3, I - jle .L65 + jle L(65) VMOVUPS_A1(-16 * SIZE, A1, %xmm4) VMOVUPS_A1(-14 * SIZE, A1, %xmm5) @@ -1604,10 +1604,10 @@ VMOVUPS_YL1(-10 * SIZE, Y1, %xmm3) decq I - jle .L64 + jle L(64) ALIGN_3 -.L63: +L(63): vfmaddpd %xmm0 , %xmm12 , %xmm4 , %xmm0 VMOVUPS_A1(-10 * SIZE, A1, %xmm7) @@ -1654,10 +1654,10 @@ subq $1, I BRANCH - jg .L63 + jg L(63) ALIGN_3 -.L64: +L(64): vfmaddpd %xmm0 , %xmm12 , %xmm4 , %xmm0 VMOVUPS_A1(-10 * SIZE, A1, %xmm7) @@ -1690,9 +1690,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L65: +L(65): testq $4, MM - je .L66 + je L(66) VMOVUPS_A1(-16 * SIZE, A1, %xmm4) @@ -1721,9 +1721,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L66: +L(66): testq $2, MM - je .L67 + je L(67) VMOVUPS_A1(-16 * SIZE, A1, %xmm4) VMOVUPS_A1(-15 * SIZE, A2, %xmm5) @@ -1742,12 +1742,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L67: +L(67): testq $1, MM #if GEMV_UNROLL == 2 - je .L68 + je L(68) #else - je .L70 + je L(70) #endif vmovsd -16 * SIZE(Y1), %xmm0 @@ -1762,16 +1762,16 @@ ALIGN_3 #if GEMV_UNROLL == 2 -.L68: +L(68): cmpq $2, N - jge .L61 + jge L(61) ALIGN_3 #endif -.L70: +L(70): cmpq $1, N - jl .L900 + jl L(900) #endif @@ -1786,7 +1786,7 @@ vmulpd %xmm0, %xmm12 , %xmm12 testq $SIZE, A - je .L7X + je L(7X) vmovsd -16 * SIZE(A1), %xmm4 vmovsd -16 * SIZE(Y1), %xmm0 @@ -1799,11 +1799,11 @@ addq $SIZE, Y1 ALIGN_3 -.L7X: +L(7X): movq MM, I sarq $3, I - jle .L75 + jle L(75) VMOVUPS_A1(-16 * SIZE, A1, %xmm0) VMOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -1816,10 +1816,10 @@ VMOVUPS_YL1(-10 * SIZE, Y1, %xmm11) decq I - jle .L74 + jle L(74) ALIGN_3 -.L73: +L(73): vfmaddpd %xmm8 , %xmm12 , %xmm0 , %xmm8 VMOVUPS_A1( -8 * SIZE, A1, %xmm0) @@ -1849,10 +1849,10 @@ subq $1, I BRANCH - jg .L73 + jg L(73) ALIGN_3 -.L74: +L(74): vfmaddpd %xmm8 , %xmm12 , %xmm0 , %xmm8 VMOVUPS_YS1(-16 * SIZE, Y1, %xmm8) @@ -1867,9 +1867,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L75: +L(75): testq $4, MM - je .L76 + je L(76) VMOVUPS_A1(-16 * SIZE, A1, %xmm0) VMOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -1886,9 +1886,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L76: +L(76): testq $2, MM - je .L77 + je L(77) VMOVUPS_A1(-16 * SIZE, A1, %xmm8) @@ -1902,9 +1902,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L77: +L(77): testq $1, MM - je .L900 + je L(900) vmovsd -16 * SIZE(Y1), %xmm0 vmovsd -16 * SIZE(A1), %xmm8 @@ -1916,17 +1916,17 @@ ALIGN_3 -.L900: +L(900): #ifndef COPY_FORCE cmpq Y, BUFFER - je .L999 + je L(999) #endif cmpq $SIZE, INCY - jne .L950 + jne L(950) testq $SIZE, Y - je .L910 + je L(910) vmovsd (Y), %xmm0 vaddsd (BUFFER), %xmm0, %xmm0 @@ -1936,19 +1936,19 @@ addq $SIZE, BUFFER decq M - jle .L999 + jle L(999) ALIGN_4 -.L910: +L(910): testq $SIZE, BUFFER - jne .L920 + jne L(920) movq M, %rax sarq $3, %rax - jle .L914 + jle L(914) ALIGN_3 -.L912: +L(912): vmovups 0 * SIZE(Y), %xmm0 vmovups 2 * SIZE(Y), %xmm1 @@ -1975,15 +1975,15 @@ addq $8 * SIZE, BUFFER decq %rax - jg .L912 + jg L(912) ALIGN_3 -.L914: +L(914): testq $7, M - jle .L999 + jle L(999) testq $4, M - jle .L915 + jle L(915) vmovups 0 * SIZE(Y), %xmm0 vmovups 2 * SIZE(Y), %xmm1 @@ -2001,9 +2001,9 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L915: +L(915): testq $2, M - jle .L916 + jle L(916) vmovups (Y), %xmm0 @@ -2017,9 +2017,9 @@ addq $2 * SIZE, BUFFER ALIGN_3 -.L916: +L(916): testq $1, M - jle .L999 + jle L(999) vmovsd (Y), %xmm0 @@ -2030,18 +2030,18 @@ vmovsd %xmm0, (Y) ALIGN_3 - jmp .L999 + jmp L(999) ALIGN_4 -.L920: +L(920): vmovups -1 * SIZE(BUFFER), %xmm4 movq M, %rax sarq $3, %rax - jle .L924 + jle L(924) ALIGN_3 -.L922: +L(922): vmovups 0 * SIZE(Y), %xmm0 vmovups 2 * SIZE(Y), %xmm1 @@ -2075,15 +2075,15 @@ addq $8 * SIZE, BUFFER decq %rax - jg .L922 + jg L(922) ALIGN_3 -.L924: +L(924): testq $7, M - jle .L999 + jle L(999) testq $4, M - jle .L925 + jle L(925) vmovups 0 * SIZE(Y), %xmm0 vmovups 2 * SIZE(Y), %xmm1 @@ -2106,9 +2106,9 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L925: +L(925): testq $2, M - jle .L926 + jle L(926) vmovups (Y), %xmm0 @@ -2126,9 +2126,9 @@ addq $2 * SIZE, BUFFER ALIGN_3 -.L926: +L(926): testq $1, M - jle .L999 + jle L(999) vmovsd (Y), %xmm0 @@ -2139,12 +2139,12 @@ vmovsd %xmm0, (Y) ALIGN_3 - jmp .L999 + jmp L(999) ALIGN_4 -.L950: +L(950): testq $SIZE, BUFFER - je .L960 + je L(960) vmovsd (Y), %xmm0 vaddsd (BUFFER), %xmm0, %xmm0 @@ -2154,18 +2154,18 @@ addq $SIZE, BUFFER decq M - jle .L999 + jle L(999) ALIGN_4 -.L960: +L(960): movq Y, Y1 movq M, %rax sarq $3, %rax - jle .L964 + jle L(964) ALIGN_3 -.L962: +L(962): vmovsd (Y), %xmm0 addq INCY, Y vmovhpd (Y), %xmm0, %xmm0 @@ -2223,15 +2223,15 @@ addq $8 * SIZE, BUFFER decq %rax - jg .L962 + jg L(962) ALIGN_3 -.L964: +L(964): testq $7, M - jle .L999 + jle L(999) testq $4, M - jle .L965 + jle L(965) vmovsd (Y), %xmm0 addq INCY, Y @@ -2262,9 +2262,9 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L965: +L(965): testq $2, M - jle .L966 + jle L(966) vmovsd (Y), %xmm0 addq INCY, Y @@ -2283,9 +2283,9 @@ addq $2 * SIZE, BUFFER ALIGN_3 -.L966: +L(966): testq $1, M - jle .L999 + jle L(999) vmovsd (Y), %xmm0 @@ -2296,7 +2296,7 @@ vmovsd %xmm0, (Y1) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/dgemv_t.S b/kernel/x86_64/dgemv_t.S index 9277774161..5bdb1bfb0c 100644 --- a/kernel/x86_64/dgemv_t.S +++ b/kernel/x86_64/dgemv_t.S @@ -168,19 +168,19 @@ -.L0x: +L(0x): xorq M,M addq $1,M salq $21,M subq M,MMM - jge .L00 + jge L(00) movq MMM,%rax addq M,%rax - jle .L999x + jle L(999x) movq %rax,M -.L00: +L(00): movq LDAX,LDA movq NN,N movq AA,A @@ -200,15 +200,15 @@ subq $-16 * SIZE, A testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) movq BUFFER, X1 #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L01 + je L(01) movsd (X), %xmm0 addq INCX, X @@ -217,18 +217,18 @@ addq $1 * SIZE, BUFFER addq $2 * SIZE, X1 decq M - jle .L10 + jle L(10) ALIGN_4 -.L01: +L(01): #endif movq M, I sarq $3, I - jle .L05 + jle L(05) ALIGN_4 -.L02: +L(02): movsd (X), %xmm0 addq INCX, X movhpd (X), %xmm0 @@ -256,38 +256,38 @@ addq $8 * SIZE, X1 decq I - jg .L02 + jg L(02) ALIGN_4 -.L05: +L(05): movq M, I andq $7, I - jle .L10 + jle L(10) ALIGN_2 -.L06: +L(06): movsd (X), %xmm0 addq INCX, X movsd %xmm0, 0 * SIZE(X1) addq $SIZE, X1 decq I - jg .L06 + jg L(06) ALIGN_4 -.L10: +L(10): movq Y, Y1 #ifdef ALIGNED_ACCESS testq $SIZE, LDA - jne .L50 + jne L(50) #endif #if GEMV_UNROLL >= 8 cmpq $8, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $8, N leaq 16 * SIZE(BUFFER), X1 @@ -311,7 +311,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L1X + je L(1X) movsd -16 * SIZE(X1), %xmm12 @@ -345,12 +345,12 @@ addq $SIZE, X1 ALIGN_3 -.L1X: +L(1X): #endif movq M, I sarq $3, I - jle .L15 + jle L(15) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm9) @@ -361,10 +361,10 @@ MOVUPS_XL1(-14 * SIZE, X1, %xmm13) decq I - jle .L13 + jle L(13) ALIGN_4 -.L12: +L(12): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -515,10 +515,10 @@ addq $8 * SIZE, X1 decq I - jg .L12 + jg L(12) ALIGN_4 -.L13: +L(13): mulpd %xmm12, %xmm8 addpd %xmm8, %xmm0 MOVUPS_A1(-16 * SIZE, A2, %xmm8) @@ -626,9 +626,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L15: +L(15): testq $4, M - jle .L16 + jle L(16) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm9) @@ -691,9 +691,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L16: +L(16): testq $2, M - jle .L17 + jle L(17) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm9) @@ -729,9 +729,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L17: +L(17): testq $1, M - je .L18 + je L(18) movsd -16 * SIZE(X1), %xmm12 @@ -761,7 +761,7 @@ addsd %xmm11, %xmm7 ALIGN_4 -.L18: +L(18): #ifdef HAVE_SSE3 haddpd %xmm1, %xmm0 haddpd %xmm3, %xmm2 @@ -796,7 +796,7 @@ mulpd ALPHA, %xmm6 cmpq $SIZE, INCY - jne .L19 + jne L(19) movsd 0 * SIZE(Y), %xmm8 movhpd 1 * SIZE(Y), %xmm8 @@ -824,11 +824,11 @@ addq $8 * SIZE, Y1 cmpq $8, N - jge .L11 - jmp .L20 + jge L(11) + jmp L(20) ALIGN_4 -.L19: +L(19): movsd (Y), %xmm8 addq INCY, Y movhpd (Y), %xmm8 @@ -869,21 +869,21 @@ addq INCY, Y1 cmpq $8, N - jge .L11 + jge L(11) ALIGN_4 -.L20: +L(20): #endif #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 4 ALIGN_3 -.L21: +L(21): #endif subq $4, N @@ -904,7 +904,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L2X + je L(2X) movsd -16 * SIZE(X1), %xmm12 @@ -926,12 +926,12 @@ addq $SIZE, X1 ALIGN_3 -.L2X: +L(2X): #endif movq M, I sarq $3, I - jle .L25 + jle L(25) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm9) @@ -942,10 +942,10 @@ MOVUPS_XL1(-14 * SIZE, X1, %xmm13) decq I - jle .L23 + jle L(23) ALIGN_4 -.L22: +L(22): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -1028,10 +1028,10 @@ addq $8 * SIZE, X1 decq I - jg .L22 + jg L(22) ALIGN_4 -.L23: +L(23): mulpd %xmm12, %xmm8 addpd %xmm8, %xmm0 MOVUPS_A1(-14 * SIZE, A1, %xmm8) @@ -1087,9 +1087,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L25: +L(25): testq $4, M - jle .L26 + jle L(26) MOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -1126,9 +1126,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L26: +L(26): testq $2, M - jle .L27 + jle L(27) MOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -1150,9 +1150,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L27: +L(27): testq $1, M - je .L28 + je L(28) movsd -16 * SIZE(X1), %xmm12 @@ -1170,7 +1170,7 @@ addsd %xmm11, %xmm3 ALIGN_4 -.L28: +L(28): #ifdef HAVE_SSE3 haddpd %xmm1, %xmm0 haddpd %xmm3, %xmm2 @@ -1191,7 +1191,7 @@ mulpd ALPHA, %xmm2 cmpq $SIZE, INCY - jne .L29 + jne L(29) movsd 0 * SIZE(Y), %xmm4 movhpd 1 * SIZE(Y), %xmm4 @@ -1210,12 +1210,12 @@ #if GEMV_UNROLL == 4 cmpq $4, N - jge .L21 + jge L(21) #endif - jmp .L30 + jmp L(30) ALIGN_4 -.L29: +L(29): movsd (Y), %xmm4 addq INCY, Y movhpd (Y), %xmm4 @@ -1239,22 +1239,22 @@ #if GEMV_UNROLL == 4 cmpq $4, N - jge .L21 + jge L(21) #endif ALIGN_4 -.L30: +L(30): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L40 + jl L(40) #if GEMV_UNROLL == 2 ALIGN_3 -.L31: +L(31): #endif subq $2, N @@ -1276,7 +1276,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L3X + je L(3X) movsd -16 * SIZE(X1), %xmm12 @@ -1292,12 +1292,12 @@ addq $SIZE, X1 ALIGN_3 -.L3X: +L(3X): #endif movq M, I sarq $3, I - jle .L35 + jle L(35) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A1(-16 * SIZE, A2, %xmm9) @@ -1308,10 +1308,10 @@ MOVUPS_XL1(-14 * SIZE, X1, %xmm13) decq I - jle .L33 + jle L(33) ALIGN_4 -.L32: +L(32): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -1361,10 +1361,10 @@ addq $8 * SIZE, X1 decq I - jg .L32 + jg L(32) ALIGN_4 -.L33: +L(33): mulpd %xmm12, %xmm8 addpd %xmm8, %xmm0 MOVUPS_A1(-12 * SIZE, A1, %xmm8) @@ -1396,9 +1396,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L35: +L(35): testq $4, M - jle .L36 + jle L(36) MOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -1423,9 +1423,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L36: +L(36): testq $2, M - jle .L37 + jle L(37) MOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -1441,9 +1441,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L37: +L(37): testq $1, M - je .L38 + je L(38) movsd -16 * SIZE(X1), %xmm12 @@ -1455,7 +1455,7 @@ addsd %xmm9, %xmm1 ALIGN_4 -.L38: +L(38): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -1485,13 +1485,13 @@ #if GEMV_UNROLL == 2 cmpq $2, N - jge .L31 + jge L(31) #endif ALIGN_4 -.L40: +L(40): cmpq $1, N - jl .L999 + jl L(999) #endif @@ -1506,7 +1506,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L4X + je L(4X) movsd -16 * SIZE(X1), %xmm12 movsd -16 * SIZE(A1), %xmm8 @@ -1517,12 +1517,12 @@ addq $SIZE, X1 ALIGN_3 -.L4X: +L(4X): #endif movq M, I sarq $3, I - jle .L45 + jle L(45) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A1(-14 * SIZE, A1, %xmm9) @@ -1533,10 +1533,10 @@ MOVUPS_XL1(-14 * SIZE, X1, %xmm13) decq I - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -1567,10 +1567,10 @@ addq $8 * SIZE, X1 decq I - jg .L42 + jg L(42) ALIGN_4 -.L43: +L(43): mulpd %xmm12, %xmm8 MOVUPS_XL1(-12 * SIZE, X1, %xmm12) addpd %xmm8, %xmm0 @@ -1587,9 +1587,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L45: +L(45): testq $4, M - jle .L46 + jle L(46) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A1(-14 * SIZE, A1, %xmm9) @@ -1606,9 +1606,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L46: +L(46): testq $2, M - jle .L47 + jle L(47) MOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -1620,9 +1620,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L47: +L(47): testq $1, M - je .L48 + je L(48) movsd -16 * SIZE(X1), %xmm12 @@ -1631,7 +1631,7 @@ addsd %xmm8, %xmm0 ALIGN_4 -.L48: +L(48): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -1658,17 +1658,17 @@ addq INCY, Y1 #ifdef ALIGNED_ACCESS - jmp .L999 + jmp L(999) ALIGN_4 -.L50: +L(50): #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L60 + jl L(60) ALIGN_3 -.L51: +L(51): subq $4, N leaq 16 * SIZE(BUFFER), X1 @@ -1688,7 +1688,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L5X + je L(5X) movsd -16 * SIZE(X1), %xmm12 @@ -1710,7 +1710,7 @@ addq $SIZE, X1 ALIGN_3 -.L5X: +L(5X): #endif movhpd -16 * SIZE(A1, LDA), %xmm8 @@ -1718,7 +1718,7 @@ movq M, I sarq $3, I - jle .L55 + jle L(55) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A2(-15 * SIZE, A1, LDA, 1, %xmm5) @@ -1729,10 +1729,10 @@ MOVUPS_XL1(-14 * SIZE, X1, %xmm13) decq I - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -1825,10 +1825,10 @@ addq $8 * SIZE, X1 decq I - jg .L52 + jg L(52) ALIGN_4 -.L53: +L(53): mulpd %xmm12, %xmm4 addpd %xmm4, %xmm0 MOVUPS_A1(-14 * SIZE, A1, %xmm4) @@ -1895,9 +1895,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L55: +L(55): testq $4, M - jle .L56 + jle L(56) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A2(-15 * SIZE, A1, LDA, 1, %xmm5) @@ -1941,9 +1941,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L56: +L(56): testq $2, M - jle .L57 + jle L(57) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A2(-15 * SIZE, A1, LDA, 1, %xmm5) @@ -1970,9 +1970,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L57: +L(57): testq $1, M - je .L58 + je L(58) movsd -16 * SIZE(X1), %xmm12 @@ -1990,7 +1990,7 @@ addsd %xmm9, %xmm3 ALIGN_4 -.L58: +L(58): #ifdef HAVE_SSE3 haddpd %xmm1, %xmm0 haddpd %xmm3, %xmm2 @@ -2011,7 +2011,7 @@ mulpd ALPHA, %xmm2 cmpq $SIZE, INCY - jne .L59 + jne L(59) movsd 0 * SIZE(Y), %xmm4 movhpd 1 * SIZE(Y), %xmm4 @@ -2029,11 +2029,11 @@ addq $4 * SIZE, Y1 cmpq $4, N - jge .L51 - jmp .L60 + jge L(51) + jmp L(60) ALIGN_4 -.L59: +L(59): movsd (Y), %xmm4 addq INCY, Y movhpd (Y), %xmm4 @@ -2055,21 +2055,21 @@ movhpd %xmm2, (Y1) addq INCY, Y1 cmpq $4, N - jge .L51 + jge L(51) ALIGN_4 -.L60: +L(60): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L70 + jl L(70) #if GEMV_UNROLL == 2 ALIGN_3 -.L61: +L(61): #endif subq $2, N @@ -2091,7 +2091,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L6X + je L(6X) movsd -16 * SIZE(X1), %xmm12 @@ -2107,14 +2107,14 @@ addq $SIZE, X1 ALIGN_3 -.L6X: +L(6X): #endif movhpd -16 * SIZE(A2), %xmm8 movq M, I sarq $3, I - jle .L65 + jle L(65) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-15 * SIZE, A2, %xmm5) @@ -2125,10 +2125,10 @@ MOVUPS_XL1(-14 * SIZE, X1, %xmm13) decq I - jle .L63 + jle L(63) ALIGN_4 -.L62: +L(62): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -2182,10 +2182,10 @@ addq $8 * SIZE, X1 decq I - jg .L62 + jg L(62) ALIGN_4 -.L63: +L(63): mulpd %xmm12, %xmm4 addpd %xmm4, %xmm0 MOVUPS_A1(-12 * SIZE, A1, %xmm4) @@ -2221,9 +2221,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L65: +L(65): testq $4, M - jle .L66 + jle L(66) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-15 * SIZE, A2, %xmm5) @@ -2251,9 +2251,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L66: +L(66): testq $2, M - jle .L67 + jle L(67) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-15 * SIZE, A2, %xmm5) @@ -2272,9 +2272,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L67: +L(67): testq $1, M - je .L68 + je L(68) movsd -16 * SIZE(X1), %xmm12 @@ -2286,7 +2286,7 @@ addsd %xmm8, %xmm1 ALIGN_4 -.L68: +L(68): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -2316,13 +2316,13 @@ #if GEMV_UNROLL == 2 cmpq $2, N - jge .L61 + jge L(61) #endif ALIGN_4 -.L70: +L(70): cmpq $1, N - jl .L999 + jl L(999) #endif @@ -2337,7 +2337,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L7X + je L(7X) movsd -16 * SIZE(X1), %xmm12 @@ -2349,11 +2349,11 @@ addq $SIZE, X1 ALIGN_3 -.L7X: +L(7X): #endif movq M, I sarq $3, I - jle .L75 + jle L(75) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm5) @@ -2364,10 +2364,10 @@ MOVUPS_XL1(-14 * SIZE, X1, %xmm13) decq I - jle .L73 + jle L(73) ALIGN_4 -.L72: +L(72): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -2398,10 +2398,10 @@ addq $8 * SIZE, X1 decq I - jg .L72 + jg L(72) ALIGN_4 -.L73: +L(73): mulpd %xmm12, %xmm4 MOVUPS_XL1(-12 * SIZE, X1, %xmm12) addpd %xmm4, %xmm0 @@ -2418,9 +2418,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L75: +L(75): testq $4, M - jle .L76 + jle L(76) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm5) @@ -2437,9 +2437,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L76: +L(76): testq $2, M - jle .L77 + jle L(77) MOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -2451,9 +2451,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L77: +L(77): testq $1, M - je .L78 + je L(78) movsd -16 * SIZE(X1), %xmm12 @@ -2462,7 +2462,7 @@ addsd %xmm4, %xmm0 ALIGN_4 -.L78: +L(78): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -2490,13 +2490,13 @@ #endif ALIGN_4 -.L999: +L(999): leaq (, M, SIZE), %rax addq %rax,AA - jmp .L0x; + jmp L(0x); ALIGN_4 -.L999x: +L(999x): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/dgemv_t_atom.S b/kernel/x86_64/dgemv_t_atom.S index 1e63c427b3..c2cde272ea 100644 --- a/kernel/x86_64/dgemv_t_atom.S +++ b/kernel/x86_64/dgemv_t_atom.S @@ -142,22 +142,22 @@ movq Y, Y1 testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) cmpq $SIZE, INCX cmoveq X, BUFFER - je .L10 + je L(10) movq BUFFER, X1 movq M, I sarq $3, I - jle .L05 + jle L(05) ALIGN_3 -.L02: +L(02): movsd (X), %xmm0 addq INCX, X movsd (X), %xmm1 @@ -189,16 +189,16 @@ addq $8 * SIZE, X1 decq I - jg .L02 + jg L(02) ALIGN_3 -.L05: +L(05): movq M, I andq $7, I - jle .L10 + jle L(10) ALIGN_3 -.L06: +L(06): movsd (X), %xmm0 addq INCX, X @@ -206,16 +206,16 @@ addq $SIZE, X1 decq I - jg .L06 + jg L(06) ALIGN_3 -.L10: +L(10): movq N, J sarq $1, J - jle .L20 + jle L(20) ALIGN_3 -.L11: +L(11): movq A, A1 leaq (A, LDA, 1), A2 leaq (A, LDA, 2), A @@ -228,7 +228,7 @@ movq M, I sarq $3, I - jle .L14 + jle L(14) movsd 0 * SIZE(X1), %xmm4 movsd 0 * SIZE(A1), %xmm8 @@ -254,10 +254,10 @@ movsd 5 * SIZE(X1), %xmm5 decq I - jle .L13 + jle L(13) ALIGN_3 -.L12: +L(12): PREFETCH PREFETCH_SIZE * SIZE(A1) addsd %xmm8, %xmm0 PREFETCH PREFETCH_SIZE * SIZE(A2) @@ -328,10 +328,10 @@ mulsd %xmm5, %xmm13 movsd 5 * SIZE(X1), %xmm5 - jg .L12 + jg L(12) ALIGN_3 -.L13: +L(13): addsd %xmm8, %xmm0 movsd 4 * SIZE(A1), %xmm8 mulsd %xmm6, %xmm10 @@ -381,9 +381,9 @@ addsd %xmm15, %xmm1 ALIGN_4 -.L14: +L(14): testq $4, M - je .L16 + je L(16) movsd 0 * SIZE(X1), %xmm4 movsd 0 * SIZE(A1), %xmm8 @@ -426,9 +426,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L16: +L(16): testq $2, M - je .L17 + je L(17) movsd 0 * SIZE(X1), %xmm4 movsd 0 * SIZE(A1), %xmm8 @@ -454,9 +454,9 @@ ALIGN_4 -.L17: +L(17): testq $1, M - je .L19 + je L(19) movsd 0 * SIZE(X1), %xmm4 @@ -470,7 +470,7 @@ addsd %xmm12, %xmm1 ALIGN_4 -.L19: +L(19): mulsd ALPHA, %xmm0 addsd (Y), %xmm0 addq INCY, Y @@ -485,12 +485,12 @@ addq INCY, Y1 decq J - jg .L11 + jg L(11) ALIGN_3 -.L20: +L(20): testq $1, N - jle .L999 + jle L(999) movq A, A1 movq BUFFER, X1 @@ -500,7 +500,7 @@ movq M, I sarq $3, I - jle .L24 + jle L(24) movsd 0 * SIZE(X1), %xmm4 movsd 0 * SIZE(A1), %xmm8 @@ -522,10 +522,10 @@ movsd 7 * SIZE(X1), %xmm7 decq I - jle .L23 + jle L(23) ALIGN_3 -.L22: +L(22): PREFETCH PREFETCH_SIZE * SIZE(A1) addsd %xmm8, %xmm0 movsd 4 * SIZE(A1), %xmm8 @@ -566,10 +566,10 @@ addq $8 * SIZE, A1 addq $8 * SIZE, X1 decq I - jg .L22 + jg L(22) ALIGN_3 -.L23: +L(23): addsd %xmm8, %xmm0 movsd 4 * SIZE(A1), %xmm8 addsd %xmm9, %xmm1 @@ -592,9 +592,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L24: +L(24): testq $4, M - je .L26 + je L(26) movsd 0 * SIZE(X1), %xmm4 movsd 0 * SIZE(A1), %xmm8 @@ -619,9 +619,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L26: +L(26): testq $2, M - je .L27 + je L(27) movsd 0 * SIZE(X1), %xmm4 movsd 0 * SIZE(A1), %xmm8 @@ -636,9 +636,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L27: +L(27): testq $1, M - je .L29 + je L(29) movsd 0 * SIZE(X1), %xmm4 movsd 0 * SIZE(A1), %xmm8 @@ -647,7 +647,7 @@ addsd %xmm8, %xmm0 ALIGN_4 -.L29: +L(29): addsd %xmm1, %xmm0 mulsd ALPHA, %xmm0 @@ -656,7 +656,7 @@ movsd %xmm0, (Y1) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/dgemv_t_bulldozer.S b/kernel/x86_64/dgemv_t_bulldozer.S index 9cd44ee2f0..279332cf22 100644 --- a/kernel/x86_64/dgemv_t_bulldozer.S +++ b/kernel/x86_64/dgemv_t_bulldozer.S @@ -173,19 +173,19 @@ -.L0x: +L(0x): xorq M,M addq $1,M salq $22,M subq M,MMM - jge .L00 + jge L(00) movq MMM,%rax addq M,%rax - jle .L999x + jle L(999x) movq %rax,M -.L00: +L(00): movq LDAX,LDA movq NN,N movq AA,A @@ -205,18 +205,18 @@ subq $-16 * SIZE, A testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) movq BUFFER, X1 movq M, I sarq $3, I - jle .L05 + jle L(05) ALIGN_4 -.L02: +L(02): vmovsd (X), %xmm0 addq INCX, X vmovhpd (X), %xmm0 , %xmm0 @@ -244,34 +244,34 @@ addq $8 * SIZE, X1 decq I - jg .L02 + jg L(02) ALIGN_4 -.L05: +L(05): movq M, I andq $7, I - jle .L10 + jle L(10) ALIGN_2 -.L06: +L(06): vmovsd (X), %xmm0 addq INCX, X vmovsd %xmm0, 0 * SIZE(X1) addq $SIZE, X1 decq I - jg .L06 + jg L(06) ALIGN_4 -.L10: +L(10): movq Y, Y1 #if GEMV_UNROLL >= 8 cmpq $8, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $8, N leaq 16 * SIZE(BUFFER), X1 @@ -292,17 +292,17 @@ movq M, I sarq $3, I - jle .L15 + jle L(15) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) VMOVUPS_XL1(-14 * SIZE, X1, %xmm13) decq I - jle .L13 + jle L(13) ALIGN_4 -.L12: +L(12): prefetchnta A_PRE(A1) prefetchnta A_PRE(A2) @@ -367,10 +367,10 @@ addq $8 * SIZE, X1 decq I - jg .L12 + jg L(12) ALIGN_4 -.L13: +L(13): vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 vfmaddpd %xmm1 , -16 * SIZE(A1 , LDA , 1) , %xmm12 , %xmm1 @@ -423,9 +423,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L15: +L(15): testq $4, M - jle .L16 + jle L(16) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -458,9 +458,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L16: +L(16): testq $2, M - jle .L17 + jle L(17) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -479,9 +479,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L17: +L(17): testq $1, M - je .L18 + je L(18) vmovsd -16 * SIZE(X1), %xmm12 @@ -507,7 +507,7 @@ ALIGN_4 -.L18: +L(18): vhaddpd %xmm1, %xmm0 , %xmm0 vhaddpd %xmm3, %xmm2 , %xmm2 vhaddpd %xmm5, %xmm4 , %xmm4 @@ -519,7 +519,7 @@ vmulpd ALPHA, %xmm6 , %xmm6 cmpq $SIZE, INCY - jne .L19 + jne L(19) vaddpd 0 * SIZE(Y), %xmm0 , %xmm0 vaddpd 2 * SIZE(Y), %xmm2 , %xmm2 @@ -534,11 +534,11 @@ addq $8 * SIZE, Y1 cmpq $8, N - jge .L11 - jmp .L20 + jge L(11) + jmp L(20) ALIGN_4 -.L19: +L(19): vmovsd (Y), %xmm8 addq INCY, Y @@ -580,21 +580,21 @@ addq INCY, Y1 cmpq $8, N - jge .L11 + jge L(11) ALIGN_4 -.L20: +L(20): #endif #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 4 ALIGN_3 -.L21: +L(21): #endif subq $4, N @@ -612,16 +612,16 @@ movq M, I sarq $3, I - jle .L25 + jle L(25) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) VMOVUPS_XL1(-14 * SIZE, X1, %xmm13) decq I - jle .L23 + jle L(23) ALIGN_4 -.L22: +L(22): prefetchnta A_PRE(A1) prefetchnta A_PRE(A2) @@ -661,10 +661,10 @@ addq $8 * SIZE, X1 decq I - jg .L22 + jg L(22) ALIGN_4 -.L23: +L(23): vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 vfmaddpd %xmm1 , -16 * SIZE(A1 , LDA , 1) , %xmm12 , %xmm1 @@ -696,9 +696,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L25: +L(25): testq $4, M - jle .L26 + jle L(26) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) VMOVUPS_XL1(-14 * SIZE, X1, %xmm13) @@ -719,9 +719,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L26: +L(26): testq $2, M - jle .L27 + jle L(27) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -735,9 +735,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L27: +L(27): testq $1, M - je .L28 + je L(28) vmovsd -16 * SIZE(X1), %xmm12 @@ -753,7 +753,7 @@ ALIGN_4 -.L28: +L(28): vhaddpd %xmm1, %xmm0 , %xmm0 vhaddpd %xmm3, %xmm2 , %xmm2 @@ -761,7 +761,7 @@ vmulpd ALPHA, %xmm2 , %xmm2 cmpq $SIZE, INCY - jne .L29 + jne L(29) vmovups 0 * SIZE(Y), %xmm4 vmovups 2 * SIZE(Y), %xmm5 @@ -776,12 +776,12 @@ #if GEMV_UNROLL == 4 cmpq $4, N - jge .L21 + jge L(21) #endif - jmp .L30 + jmp L(30) ALIGN_4 -.L29: +L(29): vmovsd (Y), %xmm4 addq INCY, Y @@ -806,22 +806,22 @@ #if GEMV_UNROLL == 4 cmpq $4, N - jge .L21 + jge L(21) #endif ALIGN_4 -.L30: +L(30): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L40 + jl L(40) #if GEMV_UNROLL == 2 ALIGN_3 -.L31: +L(31): #endif subq $2, N @@ -840,16 +840,16 @@ movq M, I sarq $3, I - jle .L35 + jle L(35) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) VMOVUPS_XL1(-14 * SIZE, X1, %xmm13) decq I - jle .L33 + jle L(33) ALIGN_4 -.L32: +L(32): prefetchnta A_PRE(A1) prefetchnta A_PRE(A2) @@ -879,10 +879,10 @@ addq $8 * SIZE, X1 decq I - jg .L32 + jg L(32) ALIGN_4 -.L33: +L(33): vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 vfmaddpd %xmm1 , -16 * SIZE(A2) , %xmm12 , %xmm1 @@ -905,9 +905,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L35: +L(35): testq $4, M - jle .L36 + jle L(36) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) VMOVUPS_XL1(-14 * SIZE, X1, %xmm13) @@ -924,9 +924,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L36: +L(36): testq $2, M - jle .L37 + jle L(37) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -938,9 +938,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L37: +L(37): testq $1, M - je .L38 + je L(38) vmovsd -16 * SIZE(X1), %xmm12 @@ -952,7 +952,7 @@ ALIGN_4 -.L38: +L(38): vaddpd %xmm2, %xmm0 , %xmm0 vaddpd %xmm3, %xmm1 , %xmm1 @@ -974,13 +974,13 @@ #if GEMV_UNROLL == 2 cmpq $2, N - jge .L31 + jge L(31) #endif ALIGN_4 -.L40: +L(40): cmpq $1, N - jl .L999 + jl L(999) #endif @@ -997,17 +997,17 @@ movq M, I sarq $3, I - jle .L45 + jle L(45) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) VMOVUPS_XL1(-14 * SIZE, X1, %xmm13) decq I - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): prefetchnta A_PRE(A1) vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 @@ -1027,10 +1027,10 @@ addq $8 * SIZE, X1 decq I - jg .L42 + jg L(42) ALIGN_4 -.L43: +L(43): vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 vfmaddpd %xmm2 , -14 * SIZE(A1) , %xmm13 , %xmm2 @@ -1047,9 +1047,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L45: +L(45): testq $4, M - jle .L46 + jle L(46) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -1062,9 +1062,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L46: +L(46): testq $2, M - jle .L47 + jle L(47) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -1074,9 +1074,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L47: +L(47): testq $1, M - je .L48 + je L(48) vmovsd -16 * SIZE(X1), %xmm12 vmovsd -16 * SIZE(A1), %xmm8 @@ -1084,7 +1084,7 @@ vfmaddpd %xmm0, %xmm8 , %xmm12, %xmm0 ALIGN_4 -.L48: +L(48): vaddpd %xmm2, %xmm0 , %xmm0 vaddpd %xmm3, %xmm1 , %xmm1 @@ -1104,13 +1104,13 @@ ALIGN_4 -.L999: +L(999): leaq (, M, SIZE), %rax addq %rax,AA - jmp .L0x; + jmp L(0x); ALIGN_4 -.L999x: +L(999x): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/dot.S b/kernel/x86_64/dot.S index a11d25e5d1..d1dd3d58bf 100644 --- a/kernel/x86_64/dot.S +++ b/kernel/x86_64/dot.S @@ -63,16 +63,16 @@ fldz cmpq $SIZE, INCX - jne .L14 + jne L(14) cmpq $SIZE, INCY - jne .L14 + jne L(14) movq N, %rax sarq $2, %rax - jle .L15 + jle L(15) ALIGN_3 -.L16: +L(16): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -101,16 +101,16 @@ addq $4 * SIZE, X addq $4 * SIZE, Y decq %rax - jg .L16 + jg L(16) ALIGN_3 -.L15: +L(15): movq N, %rax andq $3, %rax - jle .L27 + jle L(27) ALIGN_3 -.L22: +L(22): FLD (X) addq $SIZE, X FLD (Y) @@ -118,18 +118,18 @@ addq $SIZE, Y faddp %st,%st(1) decq %rax - jg .L22 + jg L(22) - jmp .L27 + jmp L(27) ALIGN_3 -.L14: +L(14): movq N, %rax sarq $2, %rax - jle .L30 + jle L(30) ALIGN_3 -.L31: +L(31): FLD (X) addq INCX, X FLD (Y) @@ -159,16 +159,16 @@ faddp %st,%st(4) decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L30: +L(30): movq N, %rax andq $3, %rax - jle .L27 + jle L(27) ALIGN_3 -.L37: +L(37): FLD (X) addq INCX, X FLD (Y) @@ -176,10 +176,10 @@ addq INCY, Y faddp %st, %st(1) decq %rax - jg .L37 + jg L(37) ALIGN_3 -.L27: +L(27): faddp %st,%st(2) faddp %st,%st(2) faddp %st,%st(1) diff --git a/kernel/x86_64/dot_atom.S b/kernel/x86_64/dot_atom.S index 794cf14220..70cbbcdbc4 100644 --- a/kernel/x86_64/dot_atom.S +++ b/kernel/x86_64/dot_atom.S @@ -68,16 +68,16 @@ pxor %xmm2, %xmm2 cmpq $0, N pxor %xmm3, %xmm3 - jle .L999 + jle L(999) cmpq $SIZE, INCX - jne .L50 + jne L(50) cmpq $SIZE, INCY - jne .L50 + jne L(50) movq N, %rax sarq $3, %rax - jle .L14 + jle L(14) movsd 0 * SIZE(X), %xmm4 movsd 0 * SIZE(Y), %xmm8 @@ -94,11 +94,11 @@ mulsd %xmm11, %xmm7 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -147,10 +147,10 @@ addq $8 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): addsd %xmm4, %xmm0 movsd 4 * SIZE(X), %xmm4 addsd %xmm5, %xmm1 @@ -178,12 +178,12 @@ addq $ 8 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $7, N - jle .L999 + jle L(999) testq $4, N - jle .L16 + jle L(16) movsd 0 * SIZE(X), %xmm4 movsd 0 * SIZE(Y), %xmm8 @@ -208,9 +208,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $2, N - jle .L17 + jle L(17) movsd 0 * SIZE(X), %xmm4 movsd 0 * SIZE(Y), %xmm8 @@ -226,25 +226,25 @@ addq $2 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $1, N - jle .L999 + jle L(999) movsd 0 * SIZE(X), %xmm4 movsd 0 * SIZE(Y), %xmm8 mulsd %xmm8, %xmm4 addsd %xmm4, %xmm0 - jmp .L999 + jmp L(999) ALIGN_3 -.L50: +L(50): movq N, %rax sarq $2, %rax - jle .L55 + jle L(55) ALIGN_3 -.L53: +L(53): movsd 0 * SIZE(X), %xmm4 addq INCX, X mulsd 0 * SIZE(Y), %xmm4 @@ -268,26 +268,26 @@ addsd %xmm7, %xmm3 decq %rax - jg .L53 + jg L(53) ALIGN_3 -.L55: +L(55): movq N, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_3 -.L56: +L(56): movsd 0 * SIZE(X), %xmm4 addq INCX, X mulsd 0 * SIZE(Y), %xmm4 addq INCY, Y addsd %xmm4, %xmm0 decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L999: +L(999): addsd %xmm1, %xmm0 addsd %xmm3, %xmm2 addsd %xmm2, %xmm0 diff --git a/kernel/x86_64/dot_sse.S b/kernel/x86_64/dot_sse.S index 6886222598..69f45e46d0 100644 --- a/kernel/x86_64/dot_sse.S +++ b/kernel/x86_64/dot_sse.S @@ -69,21 +69,21 @@ xorps %xmm3, %xmm3 cmpq $0, N - jle .L999 + jle L(999) cmpq $SIZE, INCX - jne .L50 + jne L(50) cmpq $SIZE, INCY - jne .L50 + jne L(50) subq $-32 * SIZE, X subq $-32 * SIZE, Y cmpq $3, N - jle .L17 + jle L(17) testq $SIZE, Y - je .L05 + je L(05) movss -32 * SIZE(X), %xmm0 mulss -32 * SIZE(Y), %xmm0 @@ -92,9 +92,9 @@ decq N ALIGN_2 -.L05: +L(05): testq $2 * SIZE, Y - je .L10 + je L(10) #ifdef movsd xorps %xmm4, %xmm4 @@ -108,24 +108,24 @@ addq $2 * SIZE, X addq $2 * SIZE, Y subq $2, N - jle .L999 + jle L(999) ALIGN_2 -.L10: +L(10): #ifdef ALIGNED_ACCESS testq $2 * SIZE, X - jne .L30 + jne L(30) testq $SIZE, X - jne .L20 + jne L(20) #else testq $3 * SIZE, X - jne .L20 + jne L(20) #endif movq N, %rax sarq $5, %rax - jle .L14 + jle L(14) movaps -32 * SIZE(X), %xmm4 movaps -28 * SIZE(X), %xmm5 @@ -138,11 +138,11 @@ movaps -4 * SIZE(X), %xmm11 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -195,10 +195,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): mulps -32 * SIZE(Y), %xmm4 addps %xmm4, %xmm0 mulps -28 * SIZE(Y), %xmm5 @@ -221,12 +221,12 @@ subq $-32 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $31, N - jle .L999 + jle L(999) testq $16, N - jle .L15 + jle L(15) movaps -32 * SIZE(X), %xmm4 movaps -28 * SIZE(X), %xmm5 @@ -246,9 +246,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $8, N - jle .L16 + jle L(16) movaps -32 * SIZE(X), %xmm4 movaps -28 * SIZE(X), %xmm5 @@ -262,9 +262,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $4, N - jle .L17 + jle L(17) movaps -32 * SIZE(X), %xmm4 mulps -32 * SIZE(Y), %xmm4 @@ -275,9 +275,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $2, N - jle .L18 + jle L(18) #ifdef movsd xorps %xmm4, %xmm4 @@ -295,17 +295,17 @@ addq $2 * SIZE, Y ALIGN_3 -.L18: +L(18): testq $1, N - jle .L999 + jle L(999) movss -32 * SIZE(X), %xmm4 mulss -32 * SIZE(Y), %xmm4 addss %xmm4, %xmm0 - jmp .L999 + jmp L(999) ALIGN_3 -.L20: +L(20): #ifdef ALIGNED_ACCESS movaps -33 * SIZE(X), %xmm4 @@ -313,7 +313,7 @@ movq N, %rax sarq $5, %rax - jle .L24 + jle L(24) movaps -32 * SIZE(X), %xmm5 movaps -28 * SIZE(X), %xmm6 @@ -325,11 +325,11 @@ movaps -8 * SIZE(X), %xmm11 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -398,10 +398,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): movss %xmm5, %xmm4 pshufd $0x39, %xmm4, %xmm4 mulps -32 * SIZE(Y), %xmm4 @@ -447,12 +447,12 @@ subq $-32 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $31, N - jle .L999 + jle L(999) testq $16, N - jle .L25 + jle L(25) movaps -32 * SIZE(X), %xmm5 movaps -28 * SIZE(X), %xmm6 @@ -483,9 +483,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $8, N - jle .L26 + jle L(26) movaps -32 * SIZE(X), %xmm5 movaps -28 * SIZE(X), %xmm6 @@ -506,9 +506,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $4, N - jle .L27 + jle L(27) movaps -32 * SIZE(X), %xmm5 movss %xmm5, %xmm4 @@ -521,9 +521,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $2, N - jle .L28 + jle L(28) #ifdef movsd xorps %xmm8, %xmm8 @@ -540,26 +540,26 @@ addq $2 * SIZE, Y ALIGN_3 -.L28: +L(28): testq $1, N - jle .L999 + jle L(999) pshufd $0x39, %xmm4, %xmm4 mulss -32 * SIZE(Y), %xmm4 addss %xmm4, %xmm0 - jmp .L999 + jmp L(999) ALIGN_3 -.L30: +L(30): testq $SIZE, X - jne .L40 + jne L(40) movhps -32 * SIZE(X), %xmm4 addq $2 * SIZE, X movq N, %rax sarq $5, %rax - jle .L34 + jle L(34) movaps -32 * SIZE(X), %xmm5 movaps -28 * SIZE(X), %xmm6 @@ -571,11 +571,11 @@ movaps -8 * SIZE(X), %xmm11 decq %rax - jle .L32 + jle L(32) ALIGN_3 -.L31: +L(31): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -636,10 +636,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L32: +L(32): SHUFPD_1 %xmm5, %xmm4 mulps -32 * SIZE(Y), %xmm4 addps %xmm4, %xmm0 @@ -677,12 +677,12 @@ subq $-32 * SIZE, Y ALIGN_3 -.L34: +L(34): testq $31, N - jle .L999 + jle L(999) testq $16, N - jle .L35 + jle L(35) movaps -32 * SIZE(X), %xmm5 movaps -28 * SIZE(X), %xmm6 @@ -709,9 +709,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L35: +L(35): testq $8, N - jle .L36 + jle L(36) movaps -32 * SIZE(X), %xmm5 movaps -28 * SIZE(X), %xmm6 @@ -728,9 +728,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L36: +L(36): testq $4, N - jle .L37 + jle L(37) movaps -32 * SIZE(X), %xmm5 @@ -743,9 +743,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L37: +L(37): testq $2, N - jle .L38 + jle L(38) xorps %xmm5, %xmm5 movhlps %xmm4, %xmm5 @@ -758,23 +758,23 @@ addq $2 * SIZE, Y ALIGN_3 -.L38: +L(38): testq $1, N - jle .L999 + jle L(999) movss -34 * SIZE(X), %xmm4 mulss -32 * SIZE(Y), %xmm4 addss %xmm4, %xmm0 - jmp .L999 + jmp L(999) ALIGN_3 -.L40: +L(40): movaps -35 * SIZE(X), %xmm4 addq $SIZE, X movq N, %rax sarq $5, %rax - jle .L44 + jle L(44) movaps -32 * SIZE(X), %xmm5 movaps -28 * SIZE(X), %xmm6 @@ -786,11 +786,11 @@ movaps -8 * SIZE(X), %xmm11 decq %rax - jle .L42 + jle L(42) ALIGN_3 -.L41: +L(41): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -859,10 +859,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L42: +L(42): movss %xmm5, %xmm4 shufps $0x93, %xmm5, %xmm4 mulps -32 * SIZE(Y), %xmm4 @@ -908,12 +908,12 @@ subq $-32 * SIZE, Y ALIGN_3 -.L44: +L(44): testq $31, N - jle .L999 + jle L(999) testq $16, N - jle .L45 + jle L(45) movaps -32 * SIZE(X), %xmm5 movaps -28 * SIZE(X), %xmm6 @@ -944,9 +944,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L45: +L(45): testq $8, N - jle .L46 + jle L(46) movaps -32 * SIZE(X), %xmm5 movaps -28 * SIZE(X), %xmm6 @@ -967,9 +967,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L46: +L(46): testq $4, N - jle .L47 + jle L(47) movaps -32 * SIZE(X), %xmm5 movss %xmm5, %xmm4 @@ -982,9 +982,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L47: +L(47): testq $2, N - jle .L48 + jle L(48) movaps -32 * SIZE(X), %xmm5 #ifdef movsd @@ -1003,20 +1003,20 @@ addq $2 * SIZE, Y ALIGN_3 -.L48: +L(48): testq $1, N - jle .L999 + jle L(999) pshufd $0x93, %xmm4, %xmm4 mulss -32 * SIZE(Y), %xmm4 addss %xmm4, %xmm0 - jmp .L999 + jmp L(999) ALIGN_4 #else movq N, %rax sarq $5, %rax - jle .L24 + jle L(24) movlps -32 * SIZE(X), %xmm4 movhps -30 * SIZE(X), %xmm4 @@ -1037,11 +1037,11 @@ movhps -2 * SIZE(X), %xmm11 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -1102,10 +1102,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): mulps -32 * SIZE(Y), %xmm4 addps %xmm4, %xmm0 mulps -28 * SIZE(Y), %xmm5 @@ -1128,12 +1128,12 @@ subq $-32 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $31, N - jle .L999 + jle L(999) testq $16, N - jle .L25 + jle L(25) movlps -32 * SIZE(X), %xmm4 movhps -30 * SIZE(X), %xmm4 @@ -1157,9 +1157,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $8, N - jle .L26 + jle L(26) movlps -32 * SIZE(X), %xmm4 movhps -30 * SIZE(X), %xmm4 @@ -1175,9 +1175,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $4, N - jle .L27 + jle L(27) movlps -32 * SIZE(X), %xmm4 movhps -30 * SIZE(X), %xmm4 @@ -1189,9 +1189,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $2, N - jle .L28 + jle L(28) #ifdef movsd xorps %xmm4, %xmm4 @@ -1209,25 +1209,25 @@ addq $2 * SIZE, Y ALIGN_3 -.L28: +L(28): testq $1, N - jle .L999 + jle L(999) movss -32 * SIZE(X), %xmm4 mulss -32 * SIZE(Y), %xmm4 addss %xmm4, %xmm0 - jmp .L999 + jmp L(999) ALIGN_3 #endif -.L50: +L(50): movq N, %rax sarq $2, %rax - jle .L55 + jle L(55) ALIGN_3 -.L53: +L(53): movss 0 * SIZE(X), %xmm4 addq INCX, X mulss 0 * SIZE(Y), %xmm4 @@ -1251,26 +1251,26 @@ addss %xmm7, %xmm3 decq %rax - jg .L53 + jg L(53) ALIGN_3 -.L55: +L(55): movq N, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_3 -.L56: +L(56): movss 0 * SIZE(X), %xmm4 addq INCX, X mulss 0 * SIZE(Y), %xmm4 addq INCY, Y addss %xmm4, %xmm0 decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L999: +L(999): addps %xmm1, %xmm0 addps %xmm3, %xmm2 addps %xmm2, %xmm0 diff --git a/kernel/x86_64/dot_sse2.S b/kernel/x86_64/dot_sse2.S index ceb2d0c29b..49f0246205 100644 --- a/kernel/x86_64/dot_sse2.S +++ b/kernel/x86_64/dot_sse2.S @@ -69,18 +69,18 @@ xorps %xmm3, %xmm3 cmpq $0, N - jle .L999 + jle L(999) cmpq $SIZE, INCX - jne .L50 + jne L(50) cmpq $SIZE, INCY - jne .L50 + jne L(50) subq $-16 * SIZE, X subq $-16 * SIZE, Y testq $SIZE, Y - je .L10 + je L(10) movsd -16 * SIZE(X), %xmm0 mulsd -16 * SIZE(Y), %xmm0 @@ -89,13 +89,13 @@ decq N ALIGN_2 -.L10: +L(10): testq $SIZE, X - jne .L20 + jne L(20) movq N, %rax sarq $4, %rax - jle .L14 + jle L(14) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -108,11 +108,11 @@ movaps -2 * SIZE(X), %xmm11 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -165,10 +165,10 @@ subq $-16 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): mulpd -16 * SIZE(Y), %xmm4 addpd %xmm4, %xmm0 mulpd -14 * SIZE(Y), %xmm5 @@ -191,12 +191,12 @@ subq $-16 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $15, N - jle .L999 + jle L(999) testq $8, N - jle .L15 + jle L(15) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -216,9 +216,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $4, N - jle .L16 + jle L(16) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -232,9 +232,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $2, N - jle .L17 + jle L(17) movaps -16 * SIZE(X), %xmm4 @@ -245,17 +245,17 @@ addq $2 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $1, N - jle .L999 + jle L(999) movsd -16 * SIZE(X), %xmm4 mulsd -16 * SIZE(Y), %xmm4 addsd %xmm4, %xmm0 - jmp .L999 + jmp L(999) ALIGN_3 -.L20: +L(20): #ifdef ALIGNED_ACCESS @@ -264,7 +264,7 @@ movq N, %rax sarq $4, %rax - jle .L24 + jle L(24) movaps -16 * SIZE(X), %xmm5 movaps -14 * SIZE(X), %xmm6 @@ -276,11 +276,11 @@ movaps -4 * SIZE(X), %xmm11 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -341,10 +341,10 @@ subq $-16 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): SHUFPD_1 %xmm5, %xmm4 mulpd -16 * SIZE(Y), %xmm4 addpd %xmm4, %xmm0 @@ -382,12 +382,12 @@ subq $-16 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $15, N - jle .L999 + jle L(999) testq $8, N - jle .L25 + jle L(25) movaps -16 * SIZE(X), %xmm5 movaps -14 * SIZE(X), %xmm6 @@ -414,9 +414,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $4, N - jle .L26 + jle L(26) movaps -16 * SIZE(X), %xmm5 movaps -14 * SIZE(X), %xmm6 @@ -433,9 +433,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $2, N - jle .L27 + jle L(27) movaps -16 * SIZE(X), %xmm5 @@ -448,21 +448,21 @@ addq $2 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $1, N - jle .L999 + jle L(999) SHUFPD_1 %xmm4, %xmm4 mulsd -16 * SIZE(Y), %xmm4 addsd %xmm4, %xmm0 - jmp .L999 + jmp L(999) ALIGN_3 #else movq N, %rax sarq $4, %rax - jle .L24 + jle L(24) movlps -16 * SIZE(X), %xmm4 movhps -15 * SIZE(X), %xmm4 @@ -483,11 +483,11 @@ movhps -1 * SIZE(X), %xmm11 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -548,10 +548,10 @@ subq $-16 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): mulpd -16 * SIZE(Y), %xmm4 addpd %xmm4, %xmm0 mulpd -14 * SIZE(Y), %xmm5 @@ -574,12 +574,12 @@ subq $-16 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $15, N - jle .L999 + jle L(999) testq $8, N - jle .L25 + jle L(25) movlps -16 * SIZE(X), %xmm4 movhps -15 * SIZE(X), %xmm4 @@ -603,9 +603,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $4, N - jle .L26 + jle L(26) movlps -16 * SIZE(X), %xmm4 movhps -15 * SIZE(X), %xmm4 @@ -621,9 +621,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $2, N - jle .L27 + jle L(27) movlps -16 * SIZE(X), %xmm4 movhps -15 * SIZE(X), %xmm4 @@ -635,24 +635,24 @@ addq $2 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $1, N - jle .L999 + jle L(999) movsd -16 * SIZE(X), %xmm4 mulsd -16 * SIZE(Y), %xmm4 addsd %xmm4, %xmm0 - jmp .L999 + jmp L(999) ALIGN_3 #endif -.L50: +L(50): movq N, %rax sarq $2, %rax - jle .L55 + jle L(55) ALIGN_3 -.L53: +L(53): movsd 0 * SIZE(X), %xmm4 addq INCX, X mulsd 0 * SIZE(Y), %xmm4 @@ -676,26 +676,26 @@ addsd %xmm7, %xmm3 decq %rax - jg .L53 + jg L(53) ALIGN_3 -.L55: +L(55): movq N, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_3 -.L56: +L(56): movsd 0 * SIZE(X), %xmm4 addq INCX, X mulsd 0 * SIZE(Y), %xmm4 addq INCY, Y addsd %xmm4, %xmm0 decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L999: +L(999): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 diff --git a/kernel/x86_64/dtrsm_kernel_LT_8x2_bulldozer.S b/kernel/x86_64/dtrsm_kernel_LT_8x2_bulldozer.S index bccf1c9080..14a6fc75ab 100644 --- a/kernel/x86_64/dtrsm_kernel_LT_8x2_bulldozer.S +++ b/kernel/x86_64/dtrsm_kernel_LT_8x2_bulldozer.S @@ -667,10 +667,10 @@ movq N, J sarq $1, J # j = (n >> 1) - jle .L80 + jle L(80) ALIGN_4 -.L01: +L(01): movq A, AO @@ -683,10 +683,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L50_A + jle L(50_A) ALIGN_4 /*********************************************************************************/ -.L51: +L(51): movq B, BO @@ -707,10 +707,10 @@ leaq (BO, %rax, 2), BO negq %rax - je .L56 + je L(56) ALIGN_4 -.L52: +L(52): prefetcht0 A_PR1(AO,%rax,8) prefetcht0 B_PR1(BO,%rax,2) KERNEL8x2_SUB @@ -721,13 +721,13 @@ prefetcht0 A_PR1(AO,%rax,8) KERNEL8x2_SUB - jl .L52 + jl L(52) ALIGN_4 -.L56: +L(56): movq KK, %rax andq $3, %rax # if (k & 1) - je .L59 + je L(59) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 8), AO @@ -735,13 +735,13 @@ negq %rax ALIGN_4 -.L57: +L(57): KERNEL8x2_SUB - jl .L57 + jl L(57) ALIGN_4 -.L59: +L(59): SOLVE_8x2 @@ -757,16 +757,16 @@ addq $8, KK decq I # i -- - jg .L51 + jg L(51) ALIGN_4 /*********************************************************************************/ -.L50_A: +L(50_A): testq $4, M - je .L60 + je L(60) -.L51_A: +L(51_A): movq B, BO @@ -782,23 +782,23 @@ leaq (BO, %rax, 2), BO negq %rax - je .L56_A + je L(56_A) ALIGN_4 -.L52_A: +L(52_A): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB - jl .L52_A + jl L(52_A) ALIGN_4 -.L56_A: +L(56_A): movq KK, %rax andq $3, %rax # if (k & 1) - je .L59_A + je L(59_A) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -806,14 +806,14 @@ negq %rax ALIGN_4 -.L57_A: +L(57_A): KERNEL4x2_SUB - jl .L57_A + jl L(57_A) ALIGN_4 -.L59_A: +L(59_A): SOLVE_4x2 @@ -833,11 +833,11 @@ /*********************************************************************************/ -.L60: +L(60): testq $2, M - je .L70 + je L(70) -.L61: +L(61): movq B, BO vxorpd %xmm8, %xmm8 , %xmm8 @@ -850,23 +850,23 @@ leaq (BO, %rax, 2), BO negq %rax - je .L66 + je L(66) ALIGN_4 -.L62: +L(62): KERNEL2x2_SUB KERNEL2x2_SUB KERNEL2x2_SUB KERNEL2x2_SUB - jl .L62 + jl L(62) ALIGN_4 -.L66: +L(66): movq KK, %rax andq $3, %rax # if (k & 1) - je .L69 + je L(69) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -874,14 +874,14 @@ negq %rax ALIGN_4 -.L67: +L(67): KERNEL2x2_SUB - jl .L67 + jl L(67) ALIGN_4 -.L69: +L(69): SOLVE_2x2 @@ -898,12 +898,12 @@ ALIGN_4 /********************************************************************************/ -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): movq B, BO vxorpd %xmm8, %xmm8 , %xmm8 @@ -915,23 +915,23 @@ leaq (BO, %rax, 2), BO negq %rax - je .L76 + je L(76) ALIGN_4 -.L72: +L(72): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB - jl .L72 + jl L(72) ALIGN_4 -.L76: +L(76): movq KK, %rax andq $3, %rax # if (k & 1) - je .L78 + je L(78) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -939,14 +939,14 @@ negq %rax ALIGN_4 -.L77: +L(77): KERNEL1x2_SUB - jl .L77 + jl L(77) ALIGN_4 -.L78: +L(78): SOLVE_1x2 @@ -963,17 +963,17 @@ ALIGN_4 -.L79: +L(79): movq BO, B decq J # j -- - jg .L01 + jg L(01) ALIGN_4 /***************************************************************************************/ -.L80: +L(80): testq $1, N - je .L999 + je L(999) movq A, AO movq C, CO1 # coffset1 = c @@ -983,10 +983,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L90_A + jle L(90_A) ALIGN_4 /*************************************************************************************/ -.L91: +L(91): movq B, BO @@ -1003,22 +1003,22 @@ leaq (BO, %rax, 1), BO negq %rax - je .L96 + je L(96) ALIGN_4 -.L92: +L(92): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB - jl .L92 + jl L(92) ALIGN_4 -.L96: +L(96): movq KK, %rax andq $3, %rax # if (k & 1) - je .L99 + je L(99) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 8), AO @@ -1026,12 +1026,12 @@ negq %rax ALIGN_4 -.L97: +L(97): KERNEL8x1_SUB - jl .L97 + jl L(97) ALIGN_4 -.L99: +L(99): SOLVE_8x1 @@ -1047,15 +1047,15 @@ decq I # i -- - jg .L91 + jg L(91) ALIGN_4 /*****************************************************************************/ -.L90_A: +L(90_A): testq $4, M - je .L100 + je L(100) -.L91_A: +L(91_A): movq B, BO vxorpd %xmm8, %xmm8 , %xmm8 @@ -1068,23 +1068,23 @@ leaq (BO, %rax, 1), BO negq %rax - je .L96_A + je L(96_A) ALIGN_4 -.L92_A: +L(92_A): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB - jl .L92_A + jl L(92_A) ALIGN_4 -.L96_A: +L(96_A): movq KK, %rax andq $3, %rax # if (k & 1) - je .L99_A + je L(99_A) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -1092,13 +1092,13 @@ negq %rax ALIGN_4 -.L97_A: +L(97_A): KERNEL4x1_SUB - jl .L97_A + jl L(97_A) ALIGN_4 -.L99_A: +L(99_A): SOLVE_4x1 @@ -1116,9 +1116,9 @@ ALIGN_4 /*************************************************************************************/ -.L100: +L(100): testq $2, M - je .L110 + je L(110) @@ -1133,23 +1133,23 @@ leaq (BO, %rax, 1), BO negq %rax - je .L106 + je L(106) ALIGN_4 -.L102: +L(102): KERNEL2x1_SUB KERNEL2x1_SUB KERNEL2x1_SUB KERNEL2x1_SUB - jl .L102 + jl L(102) ALIGN_4 -.L106: +L(106): movq KK, %rax andq $3, %rax # if (k & 1) - je .L109 + je L(109) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -1157,14 +1157,14 @@ negq %rax ALIGN_4 -.L107: +L(107): KERNEL2x1_SUB - jl .L107 + jl L(107) ALIGN_4 -.L109: +L(109): SOLVE_2x1 @@ -1180,12 +1180,12 @@ ALIGN_4 -.L110: +L(110): testq $1, M - je .L119 + je L(119) ALIGN_4 -.L111: +L(111): movq B, BO vxorpd %xmm8, %xmm8 , %xmm8 @@ -1197,23 +1197,23 @@ leaq (BO, %rax, 1), BO negq %rax - je .L116 + je L(116) ALIGN_4 -.L112: +L(112): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB - jl .L112 + jl L(112) ALIGN_4 -.L116: +L(116): movq KK, %rax andq $3, %rax # if (k & 1) - je .L118 + je L(118) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -1221,14 +1221,14 @@ negq %rax ALIGN_4 -.L117: +L(117): KERNEL1x1_SUB - jl .L117 + jl L(117) ALIGN_4 -.L118: +L(118): SOLVE_1x1 @@ -1244,7 +1244,7 @@ ALIGN_4 -.L119: +L(119): movq BO, B @@ -1252,7 +1252,7 @@ ALIGN_4 -.L999: +L(999): movq (%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/dtrsm_kernel_RN_8x2_bulldozer.S b/kernel/x86_64/dtrsm_kernel_RN_8x2_bulldozer.S index 9f693f852e..855bbda292 100644 --- a/kernel/x86_64/dtrsm_kernel_RN_8x2_bulldozer.S +++ b/kernel/x86_64/dtrsm_kernel_RN_8x2_bulldozer.S @@ -480,10 +480,10 @@ movq N, J sarq $1, J # j = (n >> 1) - jle .L80 + jle L(80) ALIGN_4 -.L01: +L(01): movq A, AO @@ -493,10 +493,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L50_A + jle L(50_A) ALIGN_4 /*********************************************************************************/ -.L51: +L(51): movq B, BO @@ -517,10 +517,10 @@ leaq (BO, %rax, 2), BO negq %rax - je .L56 + je L(56) ALIGN_4 -.L52: +L(52): prefetcht0 A_PR1(AO,%rax,8) prefetcht0 B_PR1(BO,%rax,2) KERNEL8x2_SUB @@ -531,13 +531,13 @@ prefetcht0 A_PR1(AO,%rax,8) KERNEL8x2_SUB - jl .L52 + jl L(52) ALIGN_4 -.L56: +L(56): movq KK, %rax andq $3, %rax # if (k & 1) - je .L59 + je L(59) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 8), AO @@ -545,13 +545,13 @@ negq %rax ALIGN_4 -.L57: +L(57): KERNEL8x2_SUB - jl .L57 + jl L(57) ALIGN_4 -.L59: +L(59): SOLVE_8x2 @@ -565,16 +565,16 @@ leaq (BO, %rax, 2), BO decq I # i -- - jg .L51 + jg L(51) ALIGN_4 /*********************************************************************************/ -.L50_A: +L(50_A): testq $4, M - je .L60 + je L(60) -.L51_A: +L(51_A): movq B, BO @@ -590,23 +590,23 @@ leaq (BO, %rax, 2), BO negq %rax - je .L56_A + je L(56_A) ALIGN_4 -.L52_A: +L(52_A): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB - jl .L52_A + jl L(52_A) ALIGN_4 -.L56_A: +L(56_A): movq KK, %rax andq $3, %rax # if (k & 1) - je .L59_A + je L(59_A) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -614,14 +614,14 @@ negq %rax ALIGN_4 -.L57_A: +L(57_A): KERNEL4x2_SUB - jl .L57_A + jl L(57_A) ALIGN_4 -.L59_A: +L(59_A): SOLVE_4x2 @@ -639,11 +639,11 @@ /*********************************************************************************/ -.L60: +L(60): testq $2, M - je .L70 + je L(70) -.L61: +L(61): movq B, BO vxorpd %xmm8, %xmm8 , %xmm8 @@ -656,23 +656,23 @@ leaq (BO, %rax, 2), BO negq %rax - je .L66 + je L(66) ALIGN_4 -.L62: +L(62): KERNEL2x2_SUB KERNEL2x2_SUB KERNEL2x2_SUB KERNEL2x2_SUB - jl .L62 + jl L(62) ALIGN_4 -.L66: +L(66): movq KK, %rax andq $3, %rax # if (k & 1) - je .L69 + je L(69) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -680,14 +680,14 @@ negq %rax ALIGN_4 -.L67: +L(67): KERNEL2x2_SUB - jl .L67 + jl L(67) ALIGN_4 -.L69: +L(69): SOLVE_2x2 @@ -702,12 +702,12 @@ ALIGN_4 /********************************************************************************/ -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): movq B, BO vxorpd %xmm8, %xmm8 , %xmm8 @@ -719,23 +719,23 @@ leaq (BO, %rax, 2), BO negq %rax - je .L76 + je L(76) ALIGN_4 -.L72: +L(72): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB - jl .L72 + jl L(72) ALIGN_4 -.L76: +L(76): movq KK, %rax andq $3, %rax # if (k & 1) - je .L78 + je L(78) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -743,14 +743,14 @@ negq %rax ALIGN_4 -.L77: +L(77): KERNEL1x2_SUB - jl .L77 + jl L(77) ALIGN_4 -.L78: +L(78): SOLVE_1x2 @@ -765,29 +765,29 @@ ALIGN_4 -.L79: +L(79): addq $2, KK // number of values in B # only for RN Kernel movq BO, B decq J # j -- - jg .L01 + jg L(01) ALIGN_4 /***************************************************************************************/ -.L80: +L(80): testq $1, N - je .L999 + je L(999) movq A, AO movq C, CO1 # coffset1 = c movq M, I sarq $3, I # i = (m >> 3) - jle .L90_A + jle L(90_A) ALIGN_4 /*************************************************************************************/ -.L91: +L(91): movq B, BO @@ -804,22 +804,22 @@ leaq (BO, %rax, 1), BO negq %rax - je .L96 + je L(96) ALIGN_4 -.L92: +L(92): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB - jl .L92 + jl L(92) ALIGN_4 -.L96: +L(96): movq KK, %rax andq $3, %rax # if (k & 1) - je .L99 + je L(99) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 8), AO @@ -827,12 +827,12 @@ negq %rax ALIGN_4 -.L97: +L(97): KERNEL8x1_SUB - jl .L97 + jl L(97) ALIGN_4 -.L99: +L(99): SOLVE_8x1 @@ -846,15 +846,15 @@ decq I # i -- - jg .L91 + jg L(91) ALIGN_4 /*****************************************************************************/ -.L90_A: +L(90_A): testq $4, M - je .L100 + je L(100) -.L91_A: +L(91_A): movq B, BO vxorpd %xmm8, %xmm8 , %xmm8 @@ -867,23 +867,23 @@ leaq (BO, %rax, 1), BO negq %rax - je .L96_A + je L(96_A) ALIGN_4 -.L92_A: +L(92_A): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB - jl .L92_A + jl L(92_A) ALIGN_4 -.L96_A: +L(96_A): movq KK, %rax andq $3, %rax # if (k & 1) - je .L99_A + je L(99_A) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -891,13 +891,13 @@ negq %rax ALIGN_4 -.L97_A: +L(97_A): KERNEL4x1_SUB - jl .L97_A + jl L(97_A) ALIGN_4 -.L99_A: +L(99_A): SOLVE_4x1 @@ -912,9 +912,9 @@ ALIGN_4 /*************************************************************************************/ -.L100: +L(100): testq $2, M - je .L110 + je L(110) @@ -929,23 +929,23 @@ leaq (BO, %rax, 1), BO negq %rax - je .L106 + je L(106) ALIGN_4 -.L102: +L(102): KERNEL2x1_SUB KERNEL2x1_SUB KERNEL2x1_SUB KERNEL2x1_SUB - jl .L102 + jl L(102) ALIGN_4 -.L106: +L(106): movq KK, %rax andq $3, %rax # if (k & 1) - je .L109 + je L(109) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -953,14 +953,14 @@ negq %rax ALIGN_4 -.L107: +L(107): KERNEL2x1_SUB - jl .L107 + jl L(107) ALIGN_4 -.L109: +L(109): SOLVE_2x1 @@ -974,12 +974,12 @@ ALIGN_4 -.L110: +L(110): testq $1, M - je .L119 + je L(119) ALIGN_4 -.L111: +L(111): movq B, BO vxorpd %xmm8, %xmm8 , %xmm8 @@ -991,23 +991,23 @@ leaq (BO, %rax, 1), BO negq %rax - je .L116 + je L(116) ALIGN_4 -.L112: +L(112): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB - jl .L112 + jl L(112) ALIGN_4 -.L116: +L(116): movq KK, %rax andq $3, %rax # if (k & 1) - je .L118 + je L(118) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -1015,14 +1015,14 @@ negq %rax ALIGN_4 -.L117: +L(117): KERNEL1x1_SUB - jl .L117 + jl L(117) ALIGN_4 -.L118: +L(118): SOLVE_1x1 @@ -1036,7 +1036,7 @@ ALIGN_4 -.L119: +L(119): addq $1 , KK // number of values in B # only for RN Kernel @@ -1046,7 +1046,7 @@ ALIGN_4 -.L999: +L(999): movq (%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/gemm_beta.S b/kernel/x86_64/gemm_beta.S index 09df2b79d9..9c48feff04 100644 --- a/kernel/x86_64/gemm_beta.S +++ b/kernel/x86_64/gemm_beta.S @@ -93,28 +93,28 @@ pxor %xmm1, %xmm1 test M, M - jle .L999 + jle L(999) test N, N - jle .L999 + jle L(999) #ifdef DOUBLE ucomisd %xmm1, %xmm0 #else ucomiss %xmm1, %xmm0 #endif - jne .L201 + jne L(201) ALIGN_4 -.L101: +L(101): movq C, C1 leaq (C, LDC, SIZE), C movq M, I sarq $3, I - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): #ifdef OPTERON prefetchw 32 * SIZE(C1) #endif @@ -129,37 +129,37 @@ MOVSD %xmm0, 7 * SIZE(C1) addq $8 * SIZE, C1 decq I - jg .L102 + jg L(102) ALIGN_4 -.L103: +L(103): movq M, I andq $7, I - jle .L105 + jle L(105) ALIGN_4 -.L104: +L(104): MOVSD %xmm0, 0 * SIZE(C1) addq $SIZE, C1 decq I - jg .L104 + jg L(104) ALIGN_4 -.L105: +L(105): decq N - jg .L101 - jmp .L999 + jg L(101) + jmp L(999) ALIGN_3 -.L201: +L(201): movq C, C1 # c_offset = c leaq (C, LDC, SIZE), C # c += ldc movq M, I sarq $3, I - jle .L203 + jle L(203) ALIGN_4 -.L202: +L(202): #ifdef OPTERON prefetchw 32 * SIZE(C1) #endif @@ -193,30 +193,30 @@ addq $8 * SIZE, C1 decq I - jg .L202 + jg L(202) ALIGN_4 -.L203: +L(203): movq M, I andq $7, I - jle .L205 + jle L(205) ALIGN_4 -.L204: +L(204): MOVSD 0 * SIZE(C1), %xmm8 MULSD %xmm0, %xmm8 MOVSD %xmm8, 0 * SIZE(C1) addq $SIZE, C1 decq I - jg .L204 + jg L(204) ALIGN_4 -.L205: +L(205): decq N - jg .L201 + jg L(201) ALIGN_3 -.L999: +L(999): xorq %rax, %rax #ifdef WINDOWS_ABI diff --git a/kernel/x86_64/gemm_kernel_2x8_nehalem.S b/kernel/x86_64/gemm_kernel_2x8_nehalem.S index 7e4b0d863b..52caf7d2d1 100644 --- a/kernel/x86_64/gemm_kernel_2x8_nehalem.S +++ b/kernel/x86_64/gemm_kernel_2x8_nehalem.S @@ -163,10 +163,10 @@ movq N, J sarq $3, J NOBRANCH - jle .L30 + jle L(30) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -183,10 +183,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): prefetcht2 -16 * SIZE(BB) subq $-8 * SIZE, BB @@ -253,10 +253,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm12 @@ -382,10 +382,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -395,10 +395,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addpd %xmm1, %xmm12 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm13 @@ -434,10 +434,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): addpd %xmm1, %xmm12 movaps %xmm8, %xmm0 shufpd $2, %xmm9, %xmm8 @@ -470,7 +470,7 @@ orq LDC, %rax testq $15, %rax NOBRANCH - jne .L18x + jne L(18x) leaq (LDC, LDC, 2), %rax @@ -522,11 +522,11 @@ addq $2 * SIZE, CO2 decq I BRANCH - jg .L11 - jmp .L20 + jg L(11) + jmp L(20) ALIGN_4 -.L18x: +L(18x): leaq (LDC, LDC, 2), %rax #ifndef TRMMKERNEL @@ -576,13 +576,13 @@ addq $2 * SIZE, CO2 decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M BRANCH - jle .L29 + jle L(29) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -623,10 +623,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -688,10 +688,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -701,10 +701,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -724,10 +724,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): mulpd %xmm7, %xmm8 mulpd %xmm7, %xmm9 mulpd %xmm7, %xmm10 @@ -775,7 +775,7 @@ #endif ALIGN_4 -.L29: +L(29): #if defined(TRMMKERNEL) && !defined(LEFT) addq $8, KK #endif @@ -786,12 +786,12 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L30: +L(30): testq $4, N - jle .L50 + jle L(50) ALIGN_4 #if defined(TRMMKERNEL) && defined(LEFT) @@ -806,10 +806,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L31: +L(31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -856,10 +856,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -931,10 +931,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -944,10 +944,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): addpd %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm9 @@ -969,10 +969,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): addpd %xmm1, %xmm8 addpd %xmm2, %xmm9 addpd %xmm3, %xmm10 @@ -1036,13 +1036,13 @@ addq $2 * SIZE, CO2 decq I BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L40: +L(40): testq $1, M BRANCH - jle .L49 + jle L(49) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1083,10 +1083,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1124,10 +1124,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1137,10 +1137,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1154,10 +1154,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): addpd %xmm10, %xmm8 mulpd %xmm7, %xmm8 addpd %xmm11, %xmm9 @@ -1192,7 +1192,7 @@ #endif ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif @@ -1202,9 +1202,9 @@ leaq (C, LDC, 4), C ALIGN_4 -.L50: +L(50): testq $2, N - jle .L70 + jle L(70) ALIGN_4 #if defined(TRMMKERNEL) && defined(LEFT) @@ -1219,10 +1219,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1265,10 +1265,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -1308,13 +1308,13 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 ALIGN_3 -.L55: +L(55): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1324,10 +1324,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addpd %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm9 @@ -1341,10 +1341,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): addpd %xmm1, %xmm8 addpd %xmm2, %xmm9 @@ -1386,13 +1386,13 @@ addq $2 * SIZE, CO2 decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L69 + jle L(69) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1431,10 +1431,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): mulpd %xmm0, %xmm1 movddup -15 * SIZE(AO), %xmm0 addpd %xmm1, %xmm8 @@ -1460,10 +1460,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1473,10 +1473,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): mulpd %xmm0, %xmm1 movddup -15 * SIZE(AO), %xmm0 addpd %xmm1, %xmm8 @@ -1487,10 +1487,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addpd %xmm9, %xmm8 mulpd %xmm7, %xmm8 @@ -1518,7 +1518,7 @@ #endif ALIGN_4 -.L69: +L(69): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -1528,9 +1528,9 @@ leaq (C, LDC, 2), C ALIGN_4 -.L70: +L(70): testq $1, N - jle .L999 + jle L(999) ALIGN_4 #if defined(TRMMKERNEL) && defined(LEFT) @@ -1544,10 +1544,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L80 + jle L(80) ALIGN_4 -.L71: +L(71): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1589,10 +1589,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -1620,12 +1620,12 @@ subq $1, %rax BRANCH - jg .L72 + jg L(72) addpd %xmm9, %xmm8 ALIGN_3 -.L75: +L(75): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1635,10 +1635,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addpd %xmm1, %xmm8 movddup -16 * SIZE(BO), %xmm1 mulpd %xmm0, %xmm1 @@ -1649,10 +1649,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addpd %xmm1, %xmm8 mulpd %xmm7, %xmm8 @@ -1682,13 +1682,13 @@ addq $2 * SIZE, CO1 decq I BRANCH - jg .L71 + jg L(71) ALIGN_4 -.L80: +L(80): testq $1, M BRANCH - jle .L999 + jle L(999) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1736,10 +1736,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): mulpd %xmm0, %xmm1 #ifndef TRMMKERNEL movaps -14 * SIZE(AO), %xmm0 @@ -1775,12 +1775,12 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) addpd %xmm9, %xmm8 ALIGN_3 -.L85: +L(85): movsd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1790,10 +1790,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): mulsd %xmm0, %xmm1 movsd -15 * SIZE(AO), %xmm0 addsd %xmm1, %xmm8 @@ -1804,10 +1804,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): haddpd %xmm8, %xmm8 mulsd %xmm7, %xmm8 @@ -1820,7 +1820,7 @@ movsd %xmm8, (CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/gemm_kernel_4x2_atom.S b/kernel/x86_64/gemm_kernel_4x2_atom.S index e5f2e9105b..56ef8f20c5 100644 --- a/kernel/x86_64/gemm_kernel_4x2_atom.S +++ b/kernel/x86_64/gemm_kernel_4x2_atom.S @@ -147,10 +147,10 @@ movq N, J sarq $1, J - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -168,10 +168,10 @@ movq M, I sarq $2, I - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -225,10 +225,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addsd %xmm2, %xmm13 movaps %xmm0, %xmm2 @@ -370,10 +370,10 @@ mulsd %xmm3, %xmm6 movsd 1 * SIZE(BO), %xmm3 - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else @@ -382,10 +382,10 @@ andq $3, %rax BRANCH BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): addsd %xmm2, %xmm13 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -424,10 +424,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): movsd ALPHA, %xmm5 addsd %xmm2, %xmm13 @@ -482,12 +482,12 @@ addq $4 * SIZE, CO2 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M - jle .L30 + jle L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -533,10 +533,10 @@ #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addsd %xmm2, %xmm9 movaps %xmm0, %xmm2 @@ -614,10 +614,10 @@ addq $8 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -628,10 +628,10 @@ andq $3, %rax BRANCH BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): addsd %xmm2, %xmm9 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -654,10 +654,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): addsd %xmm2, %xmm9 mulsd %xmm7, %xmm8 addsd %xmm6, %xmm11 @@ -696,9 +696,9 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -740,10 +740,10 @@ #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): addsd %xmm5, %xmm8 movsd 2 * SIZE(BO), %xmm5 mulsd %xmm0, %xmm1 @@ -785,10 +785,10 @@ addq $8 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -801,10 +801,10 @@ andq $3, %rax BRANCH BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulsd %xmm0, %xmm1 addq $2 * SIZE, BO mulsd %xmm0, %xmm3 @@ -818,10 +818,10 @@ addq $1 * SIZE, AO decq %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): mulsd %xmm7, %xmm8 mulsd %xmm7, %xmm9 @@ -847,19 +847,19 @@ #endif ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif movq BO, B decq J # j -- - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $1, N - je .L999 + je L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -873,10 +873,10 @@ movq M, I sarq $2, I - jle .L50 + jle L(50) ALIGN_4 -.L41: +L(41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -923,10 +923,10 @@ #endif sarq $2, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): addsd %xmm9, %xmm8 movsd 4 * SIZE(AO), %xmm9 mulsd %xmm4, %xmm0 @@ -1000,10 +1000,10 @@ mulsd %xmm5, %xmm15 movsd 1 * SIZE(BO), %xmm5 - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1020,10 +1020,10 @@ andq $3, %rax BRANCH BRANCH - je .L49 + je L(49) ALIGN_4 -.L46: +L(46): mulsd %xmm4, %xmm0 mulsd %xmm4, %xmm1 mulsd %xmm4, %xmm2 @@ -1043,10 +1043,10 @@ addq $1 * SIZE, BO decq %rax BRANCH - jg .L46 + jg L(46) ALIGN_4 -.L49: +L(49): mulsd %xmm7, %xmm8 mulsd %xmm7, %xmm10 mulsd %xmm7, %xmm12 @@ -1080,12 +1080,12 @@ addq $4 * SIZE, CO1 decq I # i -- - jg .L41 + jg L(41) ALIGN_4 -.L50: +L(50): testq $2, M - jle .L60 + jle L(60) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1126,10 +1126,10 @@ #endif sarq $2, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): addsd %xmm2, %xmm8 movsd 2 * SIZE(AO), %xmm2 mulsd %xmm4, %xmm0 @@ -1170,10 +1170,10 @@ mulsd %xmm5, %xmm3 movsd 1 * SIZE(BO), %xmm5 - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1187,10 +1187,10 @@ andq $3, %rax BRANCH BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): mulsd %xmm4, %xmm0 mulsd %xmm4, %xmm1 movsd 1 * SIZE(BO), %xmm4 @@ -1204,10 +1204,10 @@ addq $1 * SIZE, BO decq %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): mulsd %xmm7, %xmm8 mulsd %xmm7, %xmm10 @@ -1235,9 +1235,9 @@ addq $2 * SIZE, CO1 ALIGN_4 -.L60: +L(60): testq $1, M - je .L999 + je L(999) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1281,10 +1281,10 @@ #endif sarq $2, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): addsd %xmm5, %xmm8 movsd 2 * SIZE(BO), %xmm5 mulsd %xmm0, %xmm1 @@ -1309,13 +1309,13 @@ addq $4 * SIZE, BO decq %rax - jne .L62 + jne L(62) addsd %xmm5, %xmm8 addsd %xmm7, %xmm9 ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1326,10 +1326,10 @@ andq $3, %rax BRANCH BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): movsd 0 * SIZE(AO), %xmm0 movsd 0 * SIZE(BO), %xmm1 @@ -1341,10 +1341,10 @@ decq %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addsd %xmm9, %xmm8 mulsd %xmm7, %xmm8 @@ -1356,7 +1356,7 @@ movsd %xmm8, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/gemm_kernel_4x4_barcelona.S b/kernel/x86_64/gemm_kernel_4x4_barcelona.S index 9a29a800d8..4f634ddba9 100644 --- a/kernel/x86_64/gemm_kernel_4x4_barcelona.S +++ b/kernel/x86_64/gemm_kernel_4x4_barcelona.S @@ -462,10 +462,10 @@ #endif movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): movq C, CO1 # coffset1 = c leaq (C, LDC, 2), CO2 # coffset2 = c + ldc @@ -484,10 +484,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -543,10 +543,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -556,7 +556,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -566,7 +566,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -576,7 +576,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -586,7 +586,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -596,7 +596,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -606,7 +606,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -616,7 +616,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -625,10 +625,10 @@ KERNEL6(16 * 0) KERNEL7(16 * 0) KERNEL8(16 * 0) - jl .L12 + jl L(12) ALIGN_4 -.L15: +L(15): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -637,7 +637,7 @@ movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) ALIGN_4 KERNEL_SUB1(16 * 0) @@ -649,14 +649,14 @@ subq $-16 * SIZE, AO ALIGN_4 -.L16: +L(16): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L19 + je L(19) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -664,7 +664,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -691,10 +691,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L17 + jl L(17) ALIGN_4 -.L19: +L(19): prefetch -8 * SIZE(BB) subq $-16 * SIZE, BB @@ -783,18 +783,18 @@ addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $3, M - je .L39 + je L(39) testq $2, M - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -838,10 +838,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L26 + je L(26) ALIGN_4 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -897,10 +897,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L22 + jl L(22) ALIGN_4 -.L26: +L(26): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -909,7 +909,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L29 + je L(29) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -917,7 +917,7 @@ negq %rax ALIGN_4 -.L27: +L(27): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -933,10 +933,10 @@ movddup -11 * SIZE(BO, %rax, 4), %xmm5 addq $SIZE, %rax - jl .L27 + jl L(27) ALIGN_4 -.L29: +L(29): #ifndef TRMMKERNEL movupd (CO1), %xmm0 movupd (CO1, LDC), %xmm2 @@ -983,9 +983,9 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1030,10 +1030,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L36 + je L(36) ALIGN_4 -.L32: +L(32): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO, %rax, 4), %xmm0 addpd %xmm1, %xmm8 @@ -1061,10 +1061,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L32 + jl L(32) ALIGN_4 -.L36: +L(36): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1073,7 +1073,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L38 + je L(38) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -1081,7 +1081,7 @@ negq %rax ALIGN_4 -.L37: +L(37): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO, %rax, 4), %xmm0 addpd %xmm1, %xmm8 @@ -1090,10 +1090,10 @@ movddup -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L37 + jl L(37) ALIGN_4 -.L38: +L(38): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1131,7 +1131,7 @@ #endif ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif @@ -1139,18 +1139,18 @@ movq BO, B decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $3, N - je .L999 + je L(999) testq $2, N - je .L80 + je L(80) ALIGN_4 -.L41: +L(41): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1166,10 +1166,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1219,10 +1219,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L56 + je L(56) ALIGN_4 -.L52: +L(52): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -1274,10 +1274,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L52 + jl L(52) ALIGN_4 -.L56: +L(56): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1286,7 +1286,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L59 + je L(59) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -1294,7 +1294,7 @@ negq %rax ALIGN_4 -.L57: +L(57): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -1309,10 +1309,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L57 + jl L(57) ALIGN_4 -.L59: +L(59): #ifndef TRMMKERNEL movupd (CO1), %xmm0 movupd 2 * SIZE(CO1), %xmm1 @@ -1358,15 +1358,15 @@ addq $4 * SIZE, CO1 # coffset += 4 addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1409,10 +1409,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L66 + je L(66) ALIGN_4 -.L62: +L(62): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -1444,10 +1444,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L62 + jl L(62) ALIGN_4 -.L66: +L(66): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1456,7 +1456,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L69 + je L(69) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -1464,7 +1464,7 @@ negq %rax ALIGN_4 -.L67: +L(67): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -1474,10 +1474,10 @@ movddup -13 * SIZE(BO, %rax, 2), %xmm3 addq $SIZE, %rax - jl .L67 + jl L(67) ALIGN_4 -.L69: +L(69): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1516,12 +1516,12 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1564,10 +1564,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L76 + je L(76) ALIGN_4 -.L72: +L(72): mulpd -16 * SIZE(BO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -12 * SIZE(AO, %rax, 1), %xmm0 @@ -1586,10 +1586,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L72 + jl L(72) ALIGN_4 -.L76: +L(76): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1598,7 +1598,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L78 + je L(78) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -1606,16 +1606,16 @@ negq %rax ALIGN_4 -.L77: +L(77): mulpd -16 * SIZE(BO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L77 + jl L(77) ALIGN_4 -.L78: +L(78): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -1648,7 +1648,7 @@ #endif ALIGN_4 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -1658,12 +1658,12 @@ leaq (C, LDC, 2), C ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L81: +L(81): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1674,10 +1674,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1723,10 +1723,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L96 + je L(96) ALIGN_4 -.L92: +L(92): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -1754,10 +1754,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L92 + jl L(92) ALIGN_4 -.L96: +L(96): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1766,7 +1766,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L99 + je L(99) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -1774,7 +1774,7 @@ negq %rax ALIGN_4 -.L97: +L(97): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -1783,10 +1783,10 @@ movddup -15 * SIZE(BO, %rax, 1), %xmm1 addq $SIZE, %rax - jl .L97 + jl L(97) ALIGN_4 -.L99: +L(99): addpd %xmm9, %xmm8 addpd %xmm13, %xmm12 @@ -1823,15 +1823,15 @@ addq $4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1874,10 +1874,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L106 + je L(106) ALIGN_4 -.L102: +L(102): mulpd -16 * SIZE(AO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -12 * SIZE(BO, %rax, 1), %xmm0 @@ -1896,10 +1896,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L102 + jl L(102) ALIGN_4 -.L106: +L(106): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1908,7 +1908,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L109 + je L(109) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -1916,16 +1916,16 @@ negq %rax ALIGN_4 -.L107: +L(107): movddup -16 * SIZE(BO, %rax, 1), %xmm0 mulpd -16 * SIZE(AO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 addq $SIZE, %rax - jl .L107 + jl L(107) ALIGN_4 -.L109: +L(109): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -1960,12 +1960,12 @@ ALIGN_4 -.L110: +L(110): testq $1, M - je .L999 + je L(999) ALIGN_4 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2004,10 +2004,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L116 + je L(116) ALIGN_4 -.L112: +L(112): mulpd -16 * SIZE(BO, %rax, 1), %xmm0 addpd %xmm0, %xmm8 movapd -12 * SIZE(AO, %rax, 1), %xmm0 @@ -2018,10 +2018,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L112 + jl L(112) ALIGN_4 -.L116: +L(116): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2030,7 +2030,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L118 + je L(118) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -2038,16 +2038,16 @@ negq %rax ALIGN_4 -.L117: +L(117): mulsd -16 * SIZE(BO, %rax, 1), %xmm0 addsd %xmm0, %xmm8 movsd -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L117 + jl L(117) ALIGN_4 -.L118: +L(118): addpd %xmm9, %xmm8 haddpd %xmm8, %xmm8 @@ -2064,7 +2064,7 @@ movsd %xmm8, (CO1) ALIGN_4 -.L999: +L(999): movq (%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/gemm_kernel_4x4_core2.S b/kernel/x86_64/gemm_kernel_4x4_core2.S index 2f2ddc875b..2dd1092fe0 100644 --- a/kernel/x86_64/gemm_kernel_4x4_core2.S +++ b/kernel/x86_64/gemm_kernel_4x4_core2.S @@ -162,10 +162,10 @@ movq N, J sarq $2, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): /* Copying to Sub Buffer */ leaq 16 * SIZE + BUFFER, BO @@ -180,10 +180,10 @@ movq K, %rax sarq $2, %rax NOBRANCH - jle .L05 + jle L(05) ALIGN_3 -.L02: +L(02): prefetcht0 (PREFETCH_R + 0) * SIZE(B) prefetcht0 (PREFETCH_R + 8) * SIZE(B) @@ -242,18 +242,18 @@ subq $-16 * SIZE, B decq %rax BRANCH - jne .L02 + jne L(02) ALIGN_3 -.L05: +L(05): movq K, %rax andq $3, %rax BRANCH BRANCH - jle .L10 + jle L(10) ALIGN_3 -.L06: +L(06): movapd -14 * SIZE(B), %xmm1 movddup %xmm0, %xmm8 @@ -272,10 +272,10 @@ addq $8 * SIZE, BO decq %rax BRANCH - jne .L06 + jne L(06) ALIGN_4 -.L10: +L(10): leaq (PREFETCH_R + 0) * SIZE(B), BB movq C, CO1 # coffset1 = c @@ -285,10 +285,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -346,10 +346,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): PADDING; addpd %xmm2, %xmm10 movaps -15 * SIZE(BO), %xmm2 @@ -479,10 +479,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): prefetcht2 -8 * SIZE(BB) #ifndef TRMMKERNEL @@ -492,10 +492,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addpd %xmm2, %xmm10 movaps -15 * SIZE(BO), %xmm2 addpd %xmm3, %xmm14 @@ -530,10 +530,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): movddup ALPHA, %xmm7 addpd %xmm2, %xmm10 @@ -554,7 +554,7 @@ orq LDC, %rax testq $15, %rax NOBRANCH - jne .L18x + jne L(18x) #ifndef TRMMKERNEL addpd 0 * SIZE(CO1), %xmm8 @@ -595,11 +595,11 @@ addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- BRANCH - jg .L11 - jmp .L20 + jg L(11) + jmp L(20) ALIGN_4 -.L18x: +L(18x): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 @@ -664,13 +664,13 @@ addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -713,10 +713,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_4 -.L21: +L(21): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm2, %xmm8 movapd -16 * SIZE(BO), %xmm2 @@ -778,10 +778,10 @@ subq $-32 * SIZE, BO subq $1, %rax BRANCH - jg .L21 + jg L(21) ALIGN_4 -.L25: +L(25): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -791,10 +791,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): addpd %xmm2, %xmm8 movapd -16 * SIZE(BO), %xmm2 mulpd %xmm0, %xmm2 @@ -813,10 +813,10 @@ addq $8 * SIZE, BO subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): addpd %xmm2, %xmm8 addpd %xmm3, %xmm9 addpd %xmm4, %xmm10 @@ -873,10 +873,10 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $1, M BRANCH - jle .L39 + jle L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -920,10 +920,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_4 -.L31: +L(31): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addsd %xmm2, %xmm8 movsd -16 * SIZE(BO), %xmm2 @@ -985,10 +985,10 @@ subq $-32 * SIZE, BO subq $1, %rax BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L35: +L(35): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -998,10 +998,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): addsd %xmm2, %xmm8 movsd -16 * SIZE(BO), %xmm2 mulsd %xmm0, %xmm2 @@ -1020,10 +1020,10 @@ addq $8 * SIZE, BO subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): addsd %xmm2, %xmm8 addsd %xmm3, %xmm9 addsd %xmm4, %xmm10 @@ -1068,7 +1068,7 @@ #endif ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif @@ -1076,16 +1076,16 @@ leaq (C, LDC, 4), C subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $2, N BRANCH - jle .L80 + jle L(80) ALIGN_4 -.L41: +L(41): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -1096,12 +1096,12 @@ movq K, %rax sarq $3, %rax - jle .L43 + jle L(43) addq %rax, %rax ALIGN_4 -.L42: +L(42): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 movddup -14 * SIZE(B), %xmm10 @@ -1124,17 +1124,17 @@ addq $16 * SIZE, BO subq $1, %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): movq K, %rax andq $7, %rax BRANCH - jle .L45 + jle L(45) ALIGN_4 -.L44: +L(44): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 @@ -1144,20 +1144,20 @@ addq $2 * SIZE, B addq $4 * SIZE, BO subq $1, %rax - jne .L44 + jne L(44) ALIGN_4 -.L45: +L(45): movq C, CO1 leaq (C, LDC, 1), CO2 movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1204,10 +1204,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L55 + jle L(55) ALIGN_4 -.L51: +L(51): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm2, %xmm8 @@ -1275,10 +1275,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L51 + jg L(51) ALIGN_4 -.L55: +L(55): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1287,10 +1287,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): addpd %xmm2, %xmm8 movapd -16 * SIZE(BO), %xmm2 addpd %xmm3, %xmm12 @@ -1310,10 +1310,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): addpd %xmm2, %xmm8 addpd %xmm3, %xmm12 addpd %xmm4, %xmm9 @@ -1368,12 +1368,12 @@ addq $4 * SIZE, CO1 addq $4 * SIZE, CO2 subq $1, I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): testq $2, M - jle .L70 + jle L(70) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1417,10 +1417,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L65 + jle L(65) ALIGN_4 -.L61: +L(61): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm2, %xmm8 @@ -1456,10 +1456,10 @@ subq $ -8 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L61 + jg L(61) ALIGN_4 -.L65: +L(65): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1468,10 +1468,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): addpd %xmm2, %xmm8 movapd -16 * SIZE(BO), %xmm2 mulpd %xmm0, %xmm2 @@ -1483,10 +1483,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addpd %xmm2, %xmm8 addpd %xmm3, %xmm9 addpd %xmm4, %xmm10 @@ -1532,9 +1532,9 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L70: +L(70): testq $1, M - jle .L79 + jle L(79) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1577,10 +1577,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L75 + jle L(75) ALIGN_4 -.L71: +L(71): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addsd %xmm2, %xmm8 @@ -1616,10 +1616,10 @@ subq $ -4 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L71 + jg L(71) ALIGN_4 -.L75: +L(75): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1628,10 +1628,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): addsd %xmm2, %xmm8 movsd -16 * SIZE(BO), %xmm2 mulsd %xmm0, %xmm2 @@ -1643,10 +1643,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addsd %xmm2, %xmm8 addsd %xmm3, %xmm9 addsd %xmm4, %xmm10 @@ -1685,7 +1685,7 @@ movsd %xmm9, 0 * SIZE(CO2) ALIGN_4 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -1693,13 +1693,13 @@ leaq (C, LDC, 2), C ALIGN_4 -.L80: +L(80): testq $1, N BRANCH - jle .L999 + jle L(999) ALIGN_4 -.L81: +L(81): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -1710,12 +1710,12 @@ movq K, %rax sarq $4, %rax - jle .L83 + jle L(83) addq %rax, %rax ALIGN_4 -.L82: +L(82): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 movddup -14 * SIZE(B), %xmm10 @@ -1737,17 +1737,17 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO subq $1, %rax - jne .L82 + jne L(82) ALIGN_4 -.L83: +L(83): movq K, %rax andq $15, %rax BRANCH - jle .L85 + jle L(85) ALIGN_4 -.L84: +L(84): movddup -16 * SIZE(B), %xmm8 movapd %xmm8, 0 * SIZE(BO) @@ -1755,19 +1755,19 @@ addq $1 * SIZE, B addq $2 * SIZE, BO subq $1, %rax - jne .L84 + jne L(84) ALIGN_4 -.L85: +L(85): movq C, CO1 movq A, AO movq M, I sarq $2, I - jle .L100 + jle L(100) ALIGN_4 -.L90: +L(90): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1812,10 +1812,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L95 + jle L(95) ALIGN_4 -.L91: +L(91): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm0, %xmm8 @@ -1852,10 +1852,10 @@ subq $-16 * SIZE, AO subq $ -8 * SIZE, BO subq $1, %rax - jg .L91 + jg L(91) ALIGN_4 -.L95: +L(95): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1864,10 +1864,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): addpd %xmm0, %xmm8 movapd -16 * SIZE(AO), %xmm0 mulpd %xmm4, %xmm0 @@ -1879,10 +1879,10 @@ addq $4 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): addpd %xmm0, %xmm8 addpd %xmm1, %xmm12 addpd %xmm2, %xmm9 @@ -1926,12 +1926,12 @@ addq $4 * SIZE, CO1 subq $1, I - jg .L90 + jg L(90) ALIGN_4 -.L100: +L(100): testq $2, M - jle .L110 + jle L(110) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1976,10 +1976,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L105 + jle L(105) ALIGN_4 -.L101: +L(101): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm0, %xmm8 @@ -2002,10 +2002,10 @@ subq $-8 * SIZE, AO subq $-8 * SIZE, BO subq $1, %rax - jg .L101 + jg L(101) ALIGN_4 -.L105: +L(105): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2014,10 +2014,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L108 + je L(108) ALIGN_4 -.L106: +L(106): addpd %xmm0, %xmm8 movapd -16 * SIZE(AO), %xmm0 mulpd %xmm4, %xmm0 @@ -2026,10 +2026,10 @@ addq $2 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L106 + jg L(106) ALIGN_4 -.L108: +L(108): addpd %xmm0, %xmm8 addpd %xmm1, %xmm9 addpd %xmm2, %xmm10 @@ -2069,9 +2069,9 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L110: +L(110): testq $1, M - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2116,10 +2116,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L115 + jle L(115) ALIGN_4 -.L111: +L(111): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm0, %xmm8 @@ -2142,10 +2142,10 @@ subq $-4 * SIZE, AO subq $-8 * SIZE, BO subq $1, %rax - jg .L111 + jg L(111) ALIGN_4 -.L115: +L(115): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2154,10 +2154,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): addsd %xmm0, %xmm8 movsd -16 * SIZE(AO), %xmm0 mulsd %xmm4, %xmm0 @@ -2166,10 +2166,10 @@ addq $1 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addsd %xmm0, %xmm8 addsd %xmm1, %xmm9 addsd %xmm2, %xmm10 @@ -2190,7 +2190,7 @@ movsd %xmm8, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/gemm_kernel_4x4_penryn.S b/kernel/x86_64/gemm_kernel_4x4_penryn.S index 56611e5c58..beb36b5b11 100644 --- a/kernel/x86_64/gemm_kernel_4x4_penryn.S +++ b/kernel/x86_64/gemm_kernel_4x4_penryn.S @@ -187,10 +187,10 @@ movq N, J sarq $2, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -207,10 +207,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -267,10 +267,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm3, %xmm11 movaps -15 * SIZE(BO), %xmm3 @@ -398,10 +398,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): PREFETCHB -8 * SIZE(BB) #ifdef DUNNINGTON PREFETCHB 0 * SIZE(BB) @@ -415,10 +415,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addpd %xmm3, %xmm11 movaps -15 * SIZE(BO), %xmm3 addpd %xmm4, %xmm15 @@ -454,10 +454,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): movddup ALPHA, %xmm1 #ifndef DUNNINGTON @@ -499,7 +499,7 @@ orq LDC, %rax testq $15, %rax NOBRANCH - jne .L18x + jne L(18x) #ifndef TRMMKERNEL addpd 0 * SIZE(CO1), %xmm8 @@ -541,11 +541,11 @@ decq I # i -- BRANCH - jg .L11 - jmp .L20 + jg L(11) + jmp L(20) ALIGN_4 -.L18x: +L(18x): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 @@ -610,13 +610,13 @@ addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -664,10 +664,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_4 -.L22: +L(22): addpd %xmm3, %xmm11 movaps -15 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -733,10 +733,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -744,10 +744,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): addpd %xmm3, %xmm11 movaps -15 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -768,10 +768,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): addpd %xmm3, %xmm11 addpd %xmm5, %xmm10 @@ -833,10 +833,10 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $1, M BRANCH - jle .L39 + jle L(39) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -879,10 +879,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) shufps $0x44, %xmm0, %xmm0 @@ -929,10 +929,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -940,10 +940,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): shufps $0x44, %xmm0, %xmm0 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm3 @@ -959,10 +959,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): movddup ALPHA, %xmm3 addpd %xmm10, %xmm8 @@ -1004,7 +1004,7 @@ #endif ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif @@ -1015,13 +1015,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $2, N BRANCH - jle .L80 + jle L(80) movq C, CO1 leaq (C, LDC, 1), CO2 @@ -1039,10 +1039,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1088,10 +1088,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_4 -.L52: +L(52): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 @@ -1167,10 +1167,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_4 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1178,10 +1178,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -1204,10 +1204,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): movddup ALPHA, %xmm3 movaps %xmm8, %xmm0 @@ -1264,13 +1264,13 @@ addq $4 * SIZE, CO2 decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M BRANCH - jle .L70 + jle L(70) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1311,10 +1311,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x4e, %xmm2, %xmm7 @@ -1357,10 +1357,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1368,10 +1368,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm7 @@ -1386,10 +1386,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1433,10 +1433,10 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L70: +L(70): testq $1, M BRANCH - jle .L79 + jle L(79) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1476,10 +1476,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_4 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) shufps $0x44, %xmm0, %xmm0 @@ -1510,10 +1510,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -1521,10 +1521,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): shufps $0x44, %xmm0, %xmm0 mulpd %xmm0, %xmm2 movsd -15 * SIZE(AO), %xmm0 @@ -1536,10 +1536,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): movddup ALPHA, %xmm3 addpd %xmm9, %xmm8 @@ -1572,7 +1572,7 @@ #endif ALIGN_4 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -1581,10 +1581,10 @@ movq BO, B ALIGN_4 -.L80: +L(80): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1597,10 +1597,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1642,10 +1642,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_4 -.L92: +L(92): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm2, %xmm3 @@ -1702,10 +1702,10 @@ subq $ -4 * SIZE, BO subq $1, %rax BRANCH - jg .L92 + jg L(92) ALIGN_4 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -1713,10 +1713,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): pshufd $0x44, %xmm2, %xmm3 pshufd $0x44, %xmm2, %xmm4 movsd -16 * SIZE(BO), %xmm2 @@ -1734,10 +1734,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): movddup ALPHA, %xmm3 #ifndef TRMMKERNEL @@ -1776,13 +1776,13 @@ addq $4 * SIZE, CO1 decq I BRANCH - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M BRANCH - jle .L110 + jle L(110) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1821,10 +1821,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_4 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm2, %xmm3 @@ -1859,10 +1859,10 @@ subq $-4 * SIZE, BO subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_4 -.L105: +L(105): #ifndef TRMMKERNEL movq K, %rax #else @@ -1870,10 +1870,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L108 + je L(108) ALIGN_4 -.L106: +L(106): pshufd $0x44, %xmm2, %xmm3 movsd -16 * SIZE(BO), %xmm2 @@ -1886,10 +1886,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_4 -.L108: +L(108): addpd %xmm9, %xmm8 movddup ALPHA, %xmm3 @@ -1925,10 +1925,10 @@ addq $2 * SIZE, CO1 ALIGN_4 -.L110: +L(110): testq $1, M BRANCH - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1967,10 +1967,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_4 -.L112: +L(112): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulsd %xmm0, %xmm2 @@ -1997,10 +1997,10 @@ subq $-4 * SIZE, BO subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_4 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2008,10 +2008,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd %xmm0, %xmm2 addsd %xmm2, %xmm8 movsd -15 * SIZE(AO), %xmm0 @@ -2022,10 +2022,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): movddup ALPHA, %xmm3 addpd %xmm9, %xmm8 @@ -2043,7 +2043,7 @@ movlpd %xmm8, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/gemm_kernel_4x4_sse2.S b/kernel/x86_64/gemm_kernel_4x4_sse2.S index bc317da8e4..1d0af3eecb 100644 --- a/kernel/x86_64/gemm_kernel_4x4_sse2.S +++ b/kernel/x86_64/gemm_kernel_4x4_sse2.S @@ -402,10 +402,10 @@ #endif movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_3 -.L01: +L(01): /* Copying to Sub Buffer */ leaq 16 * SIZE + BUFFER, BO movq C, CO1 # coffset1 = c @@ -418,14 +418,14 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_3 #define RPREFETCHSIZE (8 * 7 + 4) #define WPREFETCHSIZE (8 * 8 + 4) -.L02: +L(02): PREFETCH (RPREFETCHSIZE + 0) * SIZE(B) movq 0 * SIZE(B), %mm0 @@ -498,17 +498,17 @@ subq $-16 * SIZE, B subq $1, %rax - jne .L02 + jne L(02) ALIGN_3 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_3 -.L04: +L(04): movq 0 * SIZE(B), %mm0 movq %mm0, -16 * SIZE(BO) movq %mm0, -15 * SIZE(BO) @@ -526,20 +526,20 @@ addq $4 * SIZE, B addq $8 * SIZE, BO subq $1, %rax - jne .L04 + jne L(04) ALIGN_3 -.L10: +L(10): movq A, AO # aoffset = a leaq (RPREFETCHSIZE + 0) * SIZE(B), BB movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_3 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -601,10 +601,10 @@ leaq (BO, %rax, 8), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -625,7 +625,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -647,7 +647,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -669,7 +669,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -691,7 +691,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -713,7 +713,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -735,7 +735,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -757,7 +757,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -779,17 +779,17 @@ addq $8 * SIZE, %rax BRANCH - jl .L12 + jl L(12) ALIGN_3 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_3 @@ -809,10 +809,10 @@ #else sarq $2, %rax NOBRANCH - jle .L16 + jle L(16) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -826,10 +826,10 @@ subq $-16 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) #endif -.L16: +L(16): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -838,7 +838,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L19 + je L(19) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -846,7 +846,7 @@ negq %rax ALIGN_3 -.L17: +L(17): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movapd -14 * SIZE(BO, %rax, 8), %xmm1 @@ -873,10 +873,10 @@ movapd -10 * SIZE(AO, %rax, 4), %xmm2 addq $SIZE, %rax - jl .L17 + jl L(17) ALIGN_3 -.L19: +L(19): PREFETCH 8 * SIZE(BB) subq $-12 * SIZE, BB @@ -963,18 +963,18 @@ addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_3 -.L20: +L(20): testq $3, M - je .L39 + je L(39) testq $2, M - je .L30 + je L(30) ALIGN_3 -.L21: +L(21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1016,10 +1016,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_3 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1129,10 +1129,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_3 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1141,10 +1141,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_3 -.L26: +L(26): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movapd 2 * SIZE(BO), %xmm1 @@ -1161,10 +1161,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_3 -.L29: +L(29): #ifndef TRMMKERNEL movlpd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 @@ -1214,12 +1214,12 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_3 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_3 -.L31: +L(31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1261,10 +1261,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_3 -.L32: +L(32): mulsd %xmm0, %xmm1 addsd %xmm1, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1373,10 +1373,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_3 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1385,10 +1385,10 @@ movsd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): mulsd %xmm0, %xmm1 addsd %xmm1, %xmm8 movsd 2 * SIZE(BO), %xmm1 @@ -1405,10 +1405,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movsd 0 * SIZE(CO2), %xmm2 @@ -1447,25 +1447,25 @@ #endif ALIGN_3 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C # c += 4 * ldc decq J # j -- - jg .L01 + jg L(01) ALIGN_3 -.L40: +L(40): testq $3, N - je .L999 + je L(999) testq $2, N - je .L80 + je L(80) ALIGN_4 -.L41: +L(41): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -1476,10 +1476,10 @@ movq K, %rax sarq $2, %rax - jle .L43 + jle L(43) ALIGN_3 -.L42: +L(42): PREFETCH 56 * SIZE(B) movq 0 * SIZE(B), %mm0 @@ -1512,17 +1512,17 @@ movq %mm7, -1 * SIZE(BO) decq %rax - jne .L42 + jne L(42) ALIGN_3 -.L43: +L(43): movq K, %rax andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_3 -.L44: +L(44): movq 0 * SIZE(B), %mm0 movq 1 * SIZE(B), %mm1 @@ -1534,20 +1534,20 @@ addq $2 * SIZE, B addq $4 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_3 -.L50: +L(50): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_3 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1594,10 +1594,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_3 -.L52: +L(52): mulpd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(BO), %xmm0 @@ -1709,10 +1709,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_3 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1721,10 +1721,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_3 -.L56: +L(56): movapd 0 * SIZE(BO), %xmm1 mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 @@ -1741,10 +1741,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L59: +L(59): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 @@ -1793,15 +1793,15 @@ addq $4 * SIZE, CO1 # coffset += 4 addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L51 + jg L(51) ALIGN_3 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_3 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1843,10 +1843,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_3 -.L62: +L(62): mulpd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(BO), %xmm0 @@ -1908,10 +1908,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_3 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1920,10 +1920,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_3 -.L66: +L(66): mulpd %xmm0, %xmm1 mulpd 2 * SIZE(BO), %xmm0 addpd %xmm1, %xmm8 @@ -1934,10 +1934,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_3 -.L69: +L(69): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 @@ -1978,12 +1978,12 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_3 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_3 -.L71: +L(71): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2025,10 +2025,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_3 -.L72: +L(72): mulsd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulsd 2 * SIZE(BO), %xmm0 @@ -2089,10 +2089,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_3 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2101,10 +2101,10 @@ movsd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): mulsd %xmm0, %xmm1 mulsd 2 * SIZE(BO), %xmm0 addsd %xmm1, %xmm8 @@ -2115,10 +2115,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_3 -.L78: +L(78): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movsd 0 * SIZE(CO2), %xmm2 @@ -2152,19 +2152,19 @@ #endif ALIGN_3 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif leaq (C, LDC, 2), C ALIGN_3 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L81: +L(81): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -2175,10 +2175,10 @@ movq K, %rax sarq $3, %rax - jle .L83 + jle L(83) ALIGN_3 -.L82: +L(82): PREFETCH 56 * SIZE(B) movq 0 * SIZE(B), %mm0 @@ -2211,17 +2211,17 @@ movq %mm7, -1 * SIZE(BO) decq %rax - jne .L82 + jne L(82) ALIGN_3 -.L83: +L(83): movq K, %rax andq $7, %rax BRANCH - jle .L90 + jle L(90) ALIGN_3 -.L84: +L(84): movq 0 * SIZE(B), %mm0 movq %mm0, 0 * SIZE(BO) @@ -2230,19 +2230,19 @@ addq $1 * SIZE, B addq $2 * SIZE, BO decq %rax - jne .L84 + jne L(84) ALIGN_3 -.L90: +L(90): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_3 -.L91: +L(91): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2286,10 +2286,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_3 -.L92: +L(92): mulpd %xmm1, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd -14 * SIZE(AO), %xmm1 @@ -2346,10 +2346,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_3 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2358,10 +2358,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_3 -.L96: +L(96): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO), %xmm1 addpd %xmm0, %xmm8 @@ -2372,10 +2372,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_3 -.L99: +L(99): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 @@ -2414,15 +2414,15 @@ addq $4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L91 + jg L(91) ALIGN_3 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_3 -.L101: +L(101): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2461,10 +2461,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L105 + je L(105) ALIGN_3 -.L102: +L(102): mulpd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -14 * SIZE(AO), %xmm0 @@ -2497,10 +2497,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_3 -.L105: +L(105): #ifndef TRMMKERNEL movq K, %rax #else @@ -2509,10 +2509,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_3 -.L106: +L(106): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movapd -14 * SIZE(AO), %xmm0 @@ -2521,10 +2521,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L106 + jg L(106) ALIGN_3 -.L109: +L(109): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -2556,12 +2556,12 @@ #endif ALIGN_3 -.L110: +L(110): testq $1, M - je .L999 + je L(999) ALIGN_3 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2600,10 +2600,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_3 -.L112: +L(112): mulsd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -15 * SIZE(AO), %xmm0 @@ -2635,10 +2635,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_3 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2647,10 +2647,10 @@ movsd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_3 -.L116: +L(116): mulsd %xmm0, %xmm1 movsd -15 * SIZE(AO), %xmm0 addsd %xmm1, %xmm8 @@ -2659,10 +2659,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_3 -.L118: +L(118): addsd %xmm10, %xmm8 addsd %xmm11, %xmm9 addsd %xmm9, %xmm8 @@ -2674,7 +2674,7 @@ movsd %xmm8, 0 * SIZE(CO1) ALIGN_3 -.L999: +L(999): movq %rbx, %rsp EMMS diff --git a/kernel/x86_64/gemm_kernel_4x4_sse3.S b/kernel/x86_64/gemm_kernel_4x4_sse3.S index ae153fe8b7..43b49601e8 100644 --- a/kernel/x86_64/gemm_kernel_4x4_sse3.S +++ b/kernel/x86_64/gemm_kernel_4x4_sse3.S @@ -390,10 +390,10 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -409,10 +409,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -471,9 +471,9 @@ andq $-8, %rax salq $4, %rax NOBRANCH - je .L15 + je L(15) -.L1X: +L(1X): KERNEL1 (16 * 0) KERNEL2 (16 * 0) KERNEL3 (16 * 0) @@ -492,7 +492,7 @@ KERNEL16(16 * 0) cmpq $128 * 1, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 1) KERNEL2 (16 * 1) KERNEL3 (16 * 1) @@ -511,7 +511,7 @@ KERNEL16(16 * 1) cmpq $128 * 2, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 2) KERNEL2 (16 * 2) KERNEL3 (16 * 2) @@ -530,7 +530,7 @@ KERNEL16(16 * 2) cmpq $128 * 3, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 3) KERNEL2 (16 * 3) KERNEL3 (16 * 3) @@ -549,7 +549,7 @@ KERNEL16(16 * 3) cmpq $128 * 4, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 4) KERNEL2 (16 * 4) KERNEL3 (16 * 4) @@ -568,7 +568,7 @@ KERNEL16(16 * 4) cmpq $128 * 5, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 5) KERNEL2 (16 * 5) KERNEL3 (16 * 5) @@ -587,7 +587,7 @@ KERNEL16(16 * 5) cmpq $128 * 6, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 6) KERNEL2 (16 * 6) KERNEL3 (16 * 6) @@ -606,7 +606,7 @@ KERNEL16(16 * 6) cmpq $128 * 7, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 7) KERNEL2 (16 * 7) KERNEL3 (16 * 7) @@ -628,18 +628,18 @@ addq $32 * 8 * SIZE, BO subq $128 * 8, %rax BRANCH - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 2), BO # * 64 #else sarq $3, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -866,11 +866,11 @@ addq $32 * SIZE, AO decq %rax BRANCH - jne .L12 + jne L(12) #endif ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else @@ -880,10 +880,10 @@ andq $7, %rax # if (k & 1) BRANCH BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 addpd %xmm9, %xmm0 @@ -915,10 +915,10 @@ addq $4 * SIZE, BO # boffset1 += 8 decq %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): mulpd %xmm15, %xmm0 mulpd %xmm15, %xmm4 mulpd %xmm15, %xmm1 @@ -926,10 +926,10 @@ testq $15, CO1 NOBRANCH - jne .L19x + jne L(19x) testq $15, LDC NOBRANCH - jne .L19x + jne L(19x) mulpd %xmm15, %xmm2 mulpd %xmm15, %xmm3 @@ -975,11 +975,11 @@ addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 - jmp .L20 + jg L(11) + jmp L(20) ALIGN_4 -.L19x: +L(19x): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhpd 1 * SIZE(CO1), %xmm8 @@ -1056,16 +1056,16 @@ addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1103,10 +1103,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1216,10 +1216,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1228,10 +1228,10 @@ movddup ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -1249,10 +1249,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhpd 1 * SIZE(CO1), %xmm8 @@ -1302,12 +1302,12 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_4 -.L31: +L(31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1345,10 +1345,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1410,10 +1410,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1422,10 +1422,10 @@ movddup ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -1437,10 +1437,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhpd 0 * SIZE(CO2), %xmm8 @@ -1474,7 +1474,7 @@ #endif ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif @@ -1482,12 +1482,12 @@ leaq (C, LDC, 4), C # c += 4 * ldc movq BO, B decq J # j -- - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $2, N - je .L80 + je L(80) ALIGN_4 #if defined(TRMMKERNEL) && defined(LEFT) @@ -1505,10 +1505,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1557,10 +1557,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1679,10 +1679,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1691,10 +1691,10 @@ movddup ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 addpd %xmm9, %xmm0 @@ -1713,10 +1713,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhpd 1 * SIZE(CO1), %xmm8 @@ -1765,15 +1765,15 @@ addq $4 * SIZE, CO1 # coffset += 4 addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1811,10 +1811,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1876,10 +1876,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1888,10 +1888,10 @@ movddup ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_4 -.L66: +L(66): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -1903,10 +1903,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L69: +L(69): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhpd 1 * SIZE(CO1), %xmm8 @@ -1946,12 +1946,12 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1989,10 +1989,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movddup 1 * SIZE(AO), %xmm8 @@ -2024,10 +2024,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2036,10 +2036,10 @@ movddup ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulpd %xmm8, %xmm9 movddup 1 * SIZE(AO), %xmm8 addpd %xmm9, %xmm0 @@ -2048,10 +2048,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhpd 0 * SIZE(CO2), %xmm8 @@ -2083,7 +2083,7 @@ #endif ALIGN_4 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -2091,9 +2091,9 @@ movq BO, B ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 #if defined(TRMMKERNEL) && defined(LEFT) @@ -2106,10 +2106,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2153,10 +2153,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(AO), %xmm9 @@ -2211,10 +2211,10 @@ addq $32 * SIZE, AO addq $8 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2223,10 +2223,10 @@ movddup ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_4 -.L96: +L(96): mulpd %xmm9, %xmm8 mulpd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -2237,10 +2237,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L99: +L(99): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhpd 1 * SIZE(CO1), %xmm8 @@ -2279,15 +2279,15 @@ addq $4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2325,10 +2325,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L105 + je L(105) ALIGN_4 -.L102: +L(102): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movddup 1 * SIZE(BO), %xmm9 @@ -2360,10 +2360,10 @@ addq $16 * SIZE, AO addq $ 8 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L105: +L(105): #ifndef TRMMKERNEL movq K, %rax #else @@ -2372,10 +2372,10 @@ movddup ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_4 -.L106: +L(106): mulpd %xmm9, %xmm8 movddup 1 * SIZE(BO), %xmm9 addpd %xmm8, %xmm0 @@ -2384,10 +2384,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L106 + jg L(106) ALIGN_4 -.L109: +L(109): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhpd 1 * SIZE(CO1), %xmm8 @@ -2421,12 +2421,12 @@ addq $2 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L110: +L(110): testq $1, M - je .L999 + je L(999) ALIGN_4 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2469,10 +2469,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulpd %xmm9, %xmm8 movapd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -2491,10 +2491,10 @@ addq $8 * SIZE, AO addq $8 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2503,10 +2503,10 @@ movddup ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd 0 * SIZE(BO), %xmm9 addsd %xmm9, %xmm0 movsd 1 * SIZE(AO), %xmm9 @@ -2514,10 +2514,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 #endif @@ -2532,7 +2532,7 @@ movsd %xmm0, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/gemm_kernel_4x8_nano.S b/kernel/x86_64/gemm_kernel_4x8_nano.S index e29520fa1b..a835d2fe08 100644 --- a/kernel/x86_64/gemm_kernel_4x8_nano.S +++ b/kernel/x86_64/gemm_kernel_4x8_nano.S @@ -160,9 +160,9 @@ movq N, J sarq $3, J - jle .L40 + jle L(40) -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -177,10 +177,10 @@ movq K, %rax sarq $1, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): PREFETCH (RPREFETCHSIZE + 0) * SIZE(B) pshufd $0x50, %xmm1, %xmm0 @@ -215,14 +215,14 @@ addq $32 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $1, %rax BRANCH - jle .L10 + jle L(10) pshufd $0x50, %xmm1, %xmm0 movaps %xmm0, -32 * SIZE(BO) @@ -238,7 +238,7 @@ subq $-16 * SIZE, BO ALIGN_4 -.L10: +L(10): movq C, CO1 leaq (C, LDC, 4), CO2 movq A, AO @@ -247,10 +247,10 @@ movq M, I sarq $2, I - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -307,10 +307,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L16 + jle L(16) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0)(AO) pshufd $0x4e, %xmm1, %xmm3 @@ -434,9 +434,9 @@ addq $ 64 * SIZE, BO decq %rax BRANCH - jg .L12 + jg L(12) -.L16: +L(16): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -445,10 +445,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L18 + je L(18) ALIGN_4 -.L17: +L(17): pshufd $0x4e, %xmm1, %xmm3 mulps %xmm0, %xmm1 addps %xmm1, %xmm8 @@ -481,10 +481,10 @@ addq $ 4 * SIZE, AO subq $-16 * SIZE, BO decq %rax - jg .L17 + jg L(17) ALIGN_4 -.L18: +L(18): leaq (LDC, LDC, 2), %rax mulps %xmm7, %xmm8 @@ -567,12 +567,12 @@ addq $4 * SIZE, CO1 addq $4 * SIZE, CO2 decq I - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -611,10 +611,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L26 + jle L(26) ALIGN_3 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0)(AO) mulps %xmm0, %xmm1 @@ -673,9 +673,9 @@ addq $64 * SIZE, BO decq %rax BRANCH - jg .L22 + jg L(22) -.L26: +L(26): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -684,10 +684,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L28 + je L(28) ALIGN_4 -.L27: +L(27): mulps %xmm0, %xmm1 addps %xmm1, %xmm8 movaps -28 * SIZE(BO), %xmm1 @@ -704,10 +704,10 @@ addq $ 2 * SIZE, AO subq $-16 * SIZE, BO decq %rax - jg .L27 + jg L(27) ALIGN_4 -.L28: +L(28): leaq (LDC, LDC, 2), %rax mulps %xmm7, %xmm8 @@ -761,9 +761,9 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -803,10 +803,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L36 + jle L(36) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0)(AO) shufps $0, %xmm0, %xmm0 @@ -869,9 +869,9 @@ addq $64 * SIZE, BO decq %rax BRANCH - jg .L32 + jg L(32) -.L36: +L(36): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -880,10 +880,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L38 + je L(38) ALIGN_4 -.L37: +L(37): shufps $0, %xmm0, %xmm0 mulps %xmm0, %xmm1 addps %xmm1, %xmm8 @@ -901,10 +901,10 @@ addq $ 1 * SIZE, AO subq $-16 * SIZE, BO decq %rax - jg .L37 + jg L(37) ALIGN_4 -.L38: +L(38): leaq (LDC, LDC, 2), %rax mulps %xmm7, %xmm8 @@ -956,18 +956,18 @@ #endif ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $8, KK #endif leaq (C, LDC, 8), C decq J - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $4, N - jle .L80 + jle L(80) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -983,10 +983,10 @@ movq K, %rax sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): PREFETCH (RPREFETCHSIZE + 0) * SIZE(B) pshufd $0x50, %xmm1, %xmm0 @@ -1021,17 +1021,17 @@ addq $32 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): movq K, %rax andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L45: +L(45): pshufd $0x50, %xmm1, %xmm0 movaps %xmm0, -32 * SIZE(BO) pshufd $0xfa, %xmm1, %xmm1 @@ -1041,20 +1041,20 @@ addq $ 4 * SIZE, B subq $-8 * SIZE, BO decq %rax - jne .L45 + jne L(45) ALIGN_4 -.L50: +L(50): movq C, CO1 leaq (C, LDC, 2), CO2 movq A, AO movq M, I sarq $2, I - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1097,10 +1097,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L56 + jle L(56) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0)(AO) pshufd $0x4e, %xmm1, %xmm3 @@ -1167,9 +1167,9 @@ subq $-32 * SIZE, BO decq %rax BRANCH - jg .L52 + jg L(52) -.L56: +L(56): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1178,10 +1178,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L58 + je L(58) ALIGN_4 -.L57: +L(57): pshufd $0x4e, %xmm1, %xmm3 mulps %xmm0, %xmm1 addps %xmm1, %xmm8 @@ -1200,10 +1200,10 @@ addq $ 4 * SIZE, AO subq $-8 * SIZE, BO decq %rax - jg .L57 + jg L(57) ALIGN_4 -.L58: +L(58): mulps %xmm7, %xmm8 mulps %xmm7, %xmm9 mulps %xmm7, %xmm10 @@ -1252,12 +1252,12 @@ addq $4 * SIZE, CO1 addq $4 * SIZE, CO2 decq I - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1296,10 +1296,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L66 + jle L(66) ALIGN_3 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0)(AO) mulps %xmm0, %xmm1 @@ -1334,9 +1334,9 @@ subq $-32 * SIZE, BO decq %rax BRANCH - jg .L62 + jg L(62) -.L66: +L(66): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1345,10 +1345,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L68 + je L(68) ALIGN_4 -.L67: +L(67): mulps %xmm0, %xmm1 mulps -28 * SIZE(BO), %xmm0 addps %xmm1, %xmm8 @@ -1359,10 +1359,10 @@ addq $ 2 * SIZE, AO subq $-8 * SIZE, BO decq %rax - jg .L67 + jg L(67) ALIGN_4 -.L68: +L(68): mulps %xmm7, %xmm8 mulps %xmm7, %xmm9 @@ -1400,9 +1400,9 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1439,10 +1439,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L76 + jle L(76) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0)(AO) shufps $0, %xmm0, %xmm0 @@ -1481,9 +1481,9 @@ subq $-32 * SIZE, BO decq %rax BRANCH - jg .L72 + jg L(72) -.L76: +L(76): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1492,10 +1492,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L78 + je L(78) ALIGN_4 -.L77: +L(77): shufps $0, %xmm0, %xmm0 mulps %xmm0, %xmm1 mulps -28 * SIZE(BO), %xmm0 @@ -1507,10 +1507,10 @@ addq $ 1 * SIZE, AO subq $-8 * SIZE, BO decq %rax - jg .L77 + jg L(77) ALIGN_4 -.L78: +L(78): mulps %xmm7, %xmm8 mulps %xmm7, %xmm9 @@ -1544,16 +1544,16 @@ #endif ALIGN_4 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C ALIGN_4 -.L80: +L(80): testq $2, N - jle .L120 + jle L(120) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1567,10 +1567,10 @@ movq K, %rax sarq $2, %rax - jle .L83 + jle L(83) ALIGN_4 -.L82: +L(82): pshufd $0x50, %xmm1, %xmm0 movaps %xmm0, -32 * SIZE(BO) pshufd $0xfa, %xmm1, %xmm1 @@ -1589,17 +1589,17 @@ subq $-16 * SIZE, BO decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L83: +L(83): movq K, %rax andq $3, %rax BRANCH - jle .L90 + jle L(90) ALIGN_4 -.L85: +L(85): pshufd $0x50, %xmm1, %xmm0 movaps %xmm0, -32 * SIZE(BO) movsd 2 * SIZE(B), %xmm1 @@ -1607,20 +1607,20 @@ addq $ 2 * SIZE, B subq $-4 * SIZE, BO decq %rax - jne .L85 + jne L(85) ALIGN_4 -.L90: +L(90): movq C, CO1 leaq (C, LDC), CO2 movq A, AO movq M, I sarq $2, I - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1659,10 +1659,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L96 + jle L(96) ALIGN_3 -.L92: +L(92): PREFETCH (PREFETCHSIZE + 0)(AO) pshufd $0x4e, %xmm1, %xmm3 @@ -1701,9 +1701,9 @@ subq $-16 * SIZE, BO decq %rax BRANCH - jg .L92 + jg L(92) -.L96: +L(96): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1712,10 +1712,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L98 + je L(98) ALIGN_4 -.L97: +L(97): pshufd $0x4e, %xmm1, %xmm3 mulps %xmm0, %xmm1 addps %xmm1, %xmm8 @@ -1727,10 +1727,10 @@ addq $ 4 * SIZE, AO subq $-4 * SIZE, BO decq %rax - jg .L97 + jg L(97) ALIGN_4 -.L98: +L(98): mulps %xmm7, %xmm8 mulps %xmm7, %xmm9 @@ -1765,12 +1765,12 @@ addq $4 * SIZE, CO1 addq $4 * SIZE, CO2 decq I - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1806,10 +1806,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L106 + jle L(106) ALIGN_3 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0)(AO) mulps %xmm0, %xmm1 @@ -1836,9 +1836,9 @@ subq $-16 * SIZE, BO decq %rax BRANCH - jg .L102 + jg L(102) -.L106: +L(106): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1847,10 +1847,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L108 + je L(108) ALIGN_4 -.L107: +L(107): mulps %xmm0, %xmm1 movddup -30 * SIZE(AO), %xmm0 addps %xmm1, %xmm8 @@ -1859,10 +1859,10 @@ addq $ 2 * SIZE, AO subq $-4 * SIZE, BO decq %rax - jg .L107 + jg L(107) ALIGN_4 -.L108: +L(108): mulps %xmm7, %xmm8 #ifndef TRMMKERNEL @@ -1892,9 +1892,9 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L110: +L(110): testq $1, M - je .L119 + je L(119) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1930,10 +1930,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L116 + jle L(116) ALIGN_3 -.L112: +L(112): PREFETCH (PREFETCHSIZE + 0)(AO) shufps $0, %xmm0, %xmm0 @@ -1964,9 +1964,9 @@ subq $-16 * SIZE, BO decq %rax BRANCH - jg .L112 + jg L(112) -.L116: +L(116): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1975,10 +1975,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L118 + je L(118) ALIGN_4 -.L117: +L(117): shufps $0, %xmm0, %xmm0 mulps %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 @@ -1988,10 +1988,10 @@ addq $ 1 * SIZE, AO subq $-4 * SIZE, BO decq %rax - jg .L117 + jg L(117) ALIGN_4 -.L118: +L(118): mulps %xmm7, %xmm8 movhlps %xmm8, %xmm9 @@ -2017,16 +2017,16 @@ #endif ALIGN_4 -.L119: +L(119): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif leaq (C, LDC, 2), C ALIGN_4 -.L120: +L(120): testq $1, N - jle .L999 + jle L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -2040,10 +2040,10 @@ movq K, %rax sarq $2, %rax - jle .L123 + jle L(123) ALIGN_4 -.L122: +L(122): pshufd $0x50, %xmm1, %xmm0 movaps %xmm0, -32 * SIZE(BO) pshufd $0xfa, %xmm1, %xmm1 @@ -2056,17 +2056,17 @@ subq $-8 * SIZE, BO decq %rax - jne .L122 + jne L(122) ALIGN_4 -.L123: +L(123): movq K, %rax andq $3, %rax BRANCH - jle .L130 + jle L(130) ALIGN_4 -.L125: +L(125): pshufd $0x50, %xmm1, %xmm0 movlps %xmm0, -32 * SIZE(BO) movss 1 * SIZE(B), %xmm1 @@ -2074,19 +2074,19 @@ addq $ 1 * SIZE, B subq $-2 * SIZE, BO decq %rax - jne .L125 + jne L(125) ALIGN_4 -.L130: +L(130): movq C, CO1 movq A, AO movq M, I sarq $2, I - jle .L140 + jle L(140) ALIGN_4 -.L131: +L(131): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2123,10 +2123,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L136 + jle L(136) ALIGN_3 -.L132: +L(132): PREFETCH (PREFETCHSIZE + 0)(AO) mulps %xmm0, %xmm1 @@ -2153,9 +2153,9 @@ subq $ -8 * SIZE, BO decq %rax BRANCH - jg .L132 + jg L(132) -.L136: +L(136): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2164,10 +2164,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L138 + je L(138) ALIGN_4 -.L137: +L(137): mulps %xmm0, %xmm1 movaps -28 * SIZE(AO), %xmm0 addps %xmm1, %xmm8 @@ -2176,10 +2176,10 @@ addq $ 4 * SIZE, AO subq $-2 * SIZE, BO decq %rax - jg .L137 + jg L(137) ALIGN_4 -.L138: +L(138): mulps %xmm7, %xmm8 #ifndef TRMMKERNEL @@ -2207,12 +2207,12 @@ addq $4 * SIZE, CO1 decq I - jg .L131 + jg L(131) ALIGN_4 -.L140: +L(140): testq $2, M - je .L150 + je L(150) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2248,10 +2248,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L146 + jle L(146) ALIGN_3 -.L142: +L(142): PREFETCH (PREFETCHSIZE + 0)(AO) mulps %xmm0, %xmm1 @@ -2278,9 +2278,9 @@ subq $-8 * SIZE, BO decq %rax BRANCH - jg .L142 + jg L(142) -.L146: +L(146): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2289,10 +2289,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L148 + je L(148) ALIGN_4 -.L147: +L(147): mulps %xmm0, %xmm1 movddup -30 * SIZE(AO), %xmm0 addps %xmm1, %xmm8 @@ -2301,10 +2301,10 @@ addq $ 2 * SIZE, AO subq $-2 * SIZE, BO decq %rax - jg .L147 + jg L(147) ALIGN_4 -.L148: +L(148): mulps %xmm7, %xmm8 #ifndef TRMMKERNEL @@ -2331,9 +2331,9 @@ addq $2 * SIZE, CO1 ALIGN_4 -.L150: +L(150): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2369,10 +2369,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L156 + jle L(156) ALIGN_3 -.L152: +L(152): PREFETCH (PREFETCHSIZE + 0)(AO) mulss %xmm0, %xmm1 @@ -2399,9 +2399,9 @@ subq $-8 * SIZE, BO decq %rax BRANCH - jg .L152 + jg L(152) -.L156: +L(156): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2410,10 +2410,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L158 + je L(158) ALIGN_4 -.L157: +L(157): mulss %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addss %xmm1, %xmm8 @@ -2422,10 +2422,10 @@ addq $1 * SIZE, AO addq $2 * SIZE, BO decq %rax - jg .L157 + jg L(157) ALIGN_4 -.L158: +L(158): mulss %xmm7, %xmm8 #ifndef TRMMKERNEL @@ -2448,7 +2448,7 @@ #endif ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/gemm_kernel_4x8_nehalem.S b/kernel/x86_64/gemm_kernel_4x8_nehalem.S index 549ea13b3f..75cf5c3f97 100644 --- a/kernel/x86_64/gemm_kernel_4x8_nehalem.S +++ b/kernel/x86_64/gemm_kernel_4x8_nehalem.S @@ -164,10 +164,10 @@ movq N, J sarq $3, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -184,10 +184,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): prefetcht2 -32 * SIZE(BB) subq $-16 * SIZE, BB @@ -254,10 +254,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm12 @@ -380,10 +380,10 @@ subq $-16 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -393,10 +393,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addps %xmm1, %xmm12 movaps -32 * SIZE(BO), %xmm1 addps %xmm2, %xmm13 @@ -432,10 +432,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): addps %xmm1, %xmm12 addps %xmm2, %xmm13 addps %xmm3, %xmm14 @@ -527,13 +527,13 @@ addq $4 * SIZE, CO2 decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -578,10 +578,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -659,10 +659,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -672,10 +672,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -699,10 +699,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): addps %xmm1, %xmm8 addps %xmm2, %xmm9 addps %xmm3, %xmm10 @@ -759,10 +759,10 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L30: +L(30): testq $1, M BRANCH - jle .L39 + jle L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -801,10 +801,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movaps -32 * SIZE(BO), %xmm2 @@ -844,10 +844,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -857,10 +857,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -875,10 +875,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): addps %xmm2, %xmm8 addps %xmm3, %xmm12 @@ -934,7 +934,7 @@ ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addq $8, KK #endif @@ -945,12 +945,12 @@ subq $1, J BRANCH - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $4, N - jle .L70 + jle L(70) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -964,10 +964,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L50 + jle L(50) ALIGN_4 -.L41: +L(41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1014,10 +1014,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -1084,10 +1084,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1097,10 +1097,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): addps %xmm1, %xmm8 movaps -32 * SIZE(BO), %xmm1 addps %xmm2, %xmm9 @@ -1121,10 +1121,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): addps %xmm1, %xmm8 addps %xmm2, %xmm9 addps %xmm3, %xmm10 @@ -1192,13 +1192,13 @@ addq $4 * SIZE, CO2 decq I BRANCH - jg .L41 + jg L(41) ALIGN_4 -.L50: +L(50): testq $2, M BRANCH - jle .L60 + jle L(60) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1238,10 +1238,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1283,10 +1283,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1296,10 +1296,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1314,10 +1314,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): addps %xmm1, %xmm8 addps %xmm2, %xmm9 @@ -1356,10 +1356,10 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L69 + jle L(69) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1397,10 +1397,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movaps -32 * SIZE(BO), %xmm2 @@ -1428,11 +1428,11 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) addps %xmm9, %xmm8 ALIGN_3 -.L65: +L(65): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1442,10 +1442,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -1457,10 +1457,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): addps %xmm2, %xmm8 mulps %xmm7, %xmm8 @@ -1496,7 +1496,7 @@ ALIGN_4 -.L69: +L(69): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif @@ -1506,9 +1506,9 @@ leaq (C, LDC, 4), C ALIGN_4 -.L70: +L(70): testq $2, N - jle .L100 + jle L(100) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1522,10 +1522,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L80 + jle L(80) ALIGN_4 -.L71: +L(71): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1567,10 +1567,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -1613,10 +1613,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_3 -.L75: +L(75): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1626,10 +1626,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addps %xmm1, %xmm8 pshufd $0x00, %xmm3, %xmm1 mulps %xmm0, %xmm1 @@ -1644,10 +1644,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_3 -.L78: +L(78): addps %xmm1, %xmm8 addps %xmm2, %xmm9 @@ -1686,13 +1686,13 @@ addq $4 * SIZE, CO2 decq I BRANCH - jg .L71 + jg L(71) ALIGN_4 -.L80: +L(80): testq $2, M BRANCH - jle .L90 + jle L(90) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1732,10 +1732,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): addps %xmm1, %xmm8 movsd -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -1765,10 +1765,10 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) ALIGN_3 -.L85: +L(85): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1778,10 +1778,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): addps %xmm1, %xmm8 movsd -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -1793,10 +1793,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_3 -.L88: +L(88): addps %xmm1, %xmm8 mulps %xmm7, %xmm8 @@ -1827,10 +1827,10 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L90: +L(90): testq $1, M BRANCH - jle .L99 + jle L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1868,10 +1868,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_3 -.L92: +L(92): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movsd -32 * SIZE(BO), %xmm2 @@ -1899,11 +1899,11 @@ subq $1, %rax BRANCH - jg .L92 + jg L(92) addps %xmm9, %xmm8 ALIGN_3 -.L95: +L(95): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1913,10 +1913,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_3 -.L96: +L(96): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -1928,10 +1928,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_3 -.L98: +L(98): addps %xmm2, %xmm8 mulps %xmm7, %xmm8 @@ -1961,7 +1961,7 @@ ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -1971,9 +1971,9 @@ leaq (C, LDC, 2), C ALIGN_4 -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1986,10 +1986,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L110 + jle L(110) ALIGN_4 -.L101: +L(101): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2027,10 +2027,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_3 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -2061,10 +2061,10 @@ subq $ -4 * SIZE, BO subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_3 -.L105: +L(105): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2074,10 +2074,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L108 + je L(108) ALIGN_3 -.L106: +L(106): addps %xmm1, %xmm8 pshufd $0x00, %xmm3, %xmm1 movss -31 * SIZE(BO), %xmm3 @@ -2089,10 +2089,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_3 -.L108: +L(108): addps %xmm1, %xmm8 mulps %xmm7, %xmm8 @@ -2123,13 +2123,13 @@ addq $4 * SIZE, CO1 decq I BRANCH - jg .L101 + jg L(101) ALIGN_4 -.L110: +L(110): testq $2, M BRANCH - jle .L120 + jle L(120) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2166,10 +2166,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_3 -.L112: +L(112): addps %xmm1, %xmm8 movss -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -2199,10 +2199,10 @@ subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_3 -.L115: +L(115): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2212,10 +2212,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_3 -.L116: +L(116): addps %xmm1, %xmm8 movss -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -2227,10 +2227,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_3 -.L118: +L(118): addps %xmm1, %xmm8 mulps %xmm7, %xmm8 @@ -2257,10 +2257,10 @@ addq $2 * SIZE, CO1 ALIGN_4 -.L120: +L(120): testq $1, M BRANCH - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2297,10 +2297,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L125 + jle L(125) ALIGN_3 -.L122: +L(122): addss %xmm2, %xmm8 movss -32 * SIZE(BO), %xmm2 mulss %xmm0, %xmm2 @@ -2326,10 +2326,10 @@ subq $1, %rax BRANCH - jg .L122 + jg L(122) ALIGN_3 -.L125: +L(125): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2339,10 +2339,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_3 -.L126: +L(126): addss %xmm2, %xmm8 movss -32 * SIZE(BO), %xmm2 mulss %xmm0, %xmm2 @@ -2353,10 +2353,10 @@ subq $1, %rax BRANCH - jg .L126 + jg L(126) ALIGN_3 -.L128: +L(128): addps %xmm2, %xmm8 mulps %xmm7, %xmm8 @@ -2368,7 +2368,7 @@ movss %xmm8, (CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/gemm_kernel_8x4_barcelona.S b/kernel/x86_64/gemm_kernel_8x4_barcelona.S index 1849565914..e49aa189c1 100644 --- a/kernel/x86_64/gemm_kernel_8x4_barcelona.S +++ b/kernel/x86_64/gemm_kernel_8x4_barcelona.S @@ -473,9 +473,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L50 + jle L(50) -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -486,10 +486,10 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): prefetch (RPREFETCHSIZE + 0) * SIZE(B) movaps 0 * SIZE(B), %xmm3 @@ -547,17 +547,17 @@ addq $64 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movaps 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -573,10 +573,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): movq C, CO1 leaq (C, LDC, 1), CO2 movq A, AO @@ -585,10 +585,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -646,10 +646,10 @@ leaq (BO, %rax, 8), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_3 -.L12: +L(12): KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -659,7 +659,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -669,7 +669,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -679,7 +679,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -689,7 +689,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -699,7 +699,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -709,7 +709,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -719,7 +719,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -729,10 +729,10 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) BRANCH - jl .L12 + jl L(12) ALIGN_4 -.L15: +L(15): prefetch 16 * SIZE(BB) subq $-32 * SIZE, BB @@ -744,7 +744,7 @@ movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_3 @@ -757,14 +757,14 @@ addq $64 * SIZE, BO ALIGN_3 -.L16: +L(16): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L18 + je L(18) leaq (, %rax, 8), %rax leaq (AO, %rax, 4), AO @@ -772,7 +772,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulps %xmm1, %xmm0 mulps -28 * SIZE(AO, %rax, 4), %xmm1 addps %xmm0, %xmm8 @@ -799,10 +799,10 @@ movaps %xmm0, %xmm2 addq $SIZE * 2, %rax - jl .L17 + jl L(17) ALIGN_4 -.L18: +L(18): #ifndef TRMMKERNEL movups 0 * SIZE(CO1), %xmm0 movups 4 * SIZE(CO1), %xmm1 @@ -875,12 +875,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -924,10 +924,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) @@ -1041,10 +1041,10 @@ addq $ 32 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1053,10 +1053,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1073,10 +1073,10 @@ addq $ 4 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): mulps %xmm15, %xmm0 mulps %xmm15, %xmm1 mulps %xmm15, %xmm2 @@ -1126,9 +1126,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $2, M - je .L40 + je L(40) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1172,10 +1172,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) @@ -1294,10 +1294,10 @@ addq $ 16 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1306,10 +1306,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movsd 4 * SIZE(BO), %xmm9 @@ -1327,10 +1327,10 @@ addq $ 2 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): mulps %xmm15, %xmm0 mulps %xmm15, %xmm1 mulps %xmm15, %xmm2 @@ -1370,9 +1370,9 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L40: +L(40): testq $1, M - je .L49 + je L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1417,10 +1417,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): mulss %xmm8, %xmm9 addss %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) @@ -1539,10 +1539,10 @@ addq $ 8 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1551,10 +1551,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -1572,10 +1572,10 @@ addq $ 1 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): mulss %xmm15, %xmm0 mulss %xmm15, %xmm1 mulss %xmm15, %xmm2 @@ -1613,19 +1613,19 @@ #endif ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C # c += 4 * ldc decq J # j -- - jg .L01 + jg L(01) -.L50: +L(50): testq $2, N - je .L100 + je L(100) -.L51: +L(51): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1636,10 +1636,10 @@ movq K, %rax sarq $2, %rax - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): prefetch (RPREFETCHSIZE + 0) * SIZE(B) movaps 0 * SIZE(B), %xmm3 @@ -1672,17 +1672,17 @@ addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): movq K, %rax andq $3, %rax BRANCH - jle .L60 + jle L(60) ALIGN_4 -.L54: +L(54): movsd 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -1699,20 +1699,20 @@ addq $ 2 * SIZE, B addq $ 8 * SIZE, BO decq %rax - jne .L54 + jne L(54) ALIGN_4 -.L60: +L(60): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L70 + jle L(70) ALIGN_4 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1760,10 +1760,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1883,10 +1883,10 @@ addq $64 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1895,10 +1895,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -1915,10 +1915,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -1969,12 +1969,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L61 + jg L(61) ALIGN_4 -.L70: +L(70): testq $4, M - je .L80 + je L(80) #if !defined(TRMMKERNEL) || \ @@ -2019,10 +2019,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2086,10 +2086,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2098,10 +2098,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -2112,10 +2112,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2156,9 +2156,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L80: +L(80): testq $2, M - je .L90 + je L(90) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2202,10 +2202,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L85 + je L(85) ALIGN_4 -.L82: +L(82): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) @@ -2276,10 +2276,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L85: +L(85): #ifndef TRMMKERNEL movq K, %rax #else @@ -2288,10 +2288,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movsd 4 * SIZE(BO), %xmm9 @@ -2303,10 +2303,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movsd 0 * SIZE(CO2), %xmm10 @@ -2343,9 +2343,9 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L90: +L(90): testq $1, M - je .L99 + je L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2389,10 +2389,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) @@ -2463,10 +2463,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2475,10 +2475,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -2490,10 +2490,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): #ifndef TRMMKERNEL movss 0 * SIZE(CO1), %xmm8 movss 0 * SIZE(CO2), %xmm10 @@ -2526,7 +2526,7 @@ #endif ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -2534,11 +2534,11 @@ ALIGN_4 -.L100: +L(100): testq $1, N - je .L999 + je L(999) -.L101: +L(101): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -2549,11 +2549,11 @@ movq K, %rax sarq $3, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): prefetch (RPREFETCHSIZE + 0) * SIZE(B) movups 0 * SIZE(B), %xmm3 @@ -2586,17 +2586,17 @@ addq $32 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): movq K, %rax andq $7, %rax BRANCH - jle .L110 + jle L(110) ALIGN_4 -.L104: +L(104): movss 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -2606,19 +2606,19 @@ addq $ 1 * SIZE, B addq $ 4 * SIZE, BO decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L110: +L(110): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L120 + jle L(120) ALIGN_4 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2665,10 +2665,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulps %xmm9, %xmm8 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2741,10 +2741,10 @@ addq $64 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2753,10 +2753,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulps %xmm9, %xmm8 mulps -28 * SIZE(AO), %xmm9 addps %xmm8, %xmm0 @@ -2767,10 +2767,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2805,12 +2805,12 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L111 + jg L(111) ALIGN_4 -.L120: +L(120): testq $4, M - je .L130 + je L(130) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2852,10 +2852,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L125 + je L(125) ALIGN_4 -.L122: +L(122): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2893,10 +2893,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L122 + jne L(122) ALIGN_4 -.L125: +L(125): #ifndef TRMMKERNEL movq K, %rax #else @@ -2905,10 +2905,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): mulps %xmm8, %xmm9 movaps -28 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -2917,10 +2917,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2954,9 +2954,9 @@ addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L130: +L(130): testq $2, M - je .L140 + je L(140) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2998,10 +2998,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L135 + je L(135) ALIGN_4 -.L132: +L(132): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3047,10 +3047,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L132 + jne L(132) ALIGN_4 -.L135: +L(135): #ifndef TRMMKERNEL movq K, %rax #else @@ -3059,10 +3059,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): mulps %xmm8, %xmm9 movsd -30 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -3071,10 +3071,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): addps %xmm1, %xmm0 mulps %xmm15, %xmm0 @@ -3101,9 +3101,9 @@ addq $2 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L140: +L(140): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3145,10 +3145,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L145 + je L(145) ALIGN_4 -.L142: +L(142): mulss %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3182,10 +3182,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L142 + jne L(142) ALIGN_4 -.L145: +L(145): #ifndef TRMMKERNEL movq K, %rax #else @@ -3194,10 +3194,10 @@ movss ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): mulss %xmm8, %xmm9 movss -31 * SIZE(AO), %xmm8 addss %xmm9, %xmm0 @@ -3206,10 +3206,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): addss %xmm1, %xmm0 addss %xmm3, %xmm2 addss %xmm2, %xmm0 @@ -3223,7 +3223,7 @@ movss %xmm0, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/gemm_kernel_8x4_core2.S b/kernel/x86_64/gemm_kernel_8x4_core2.S index c31dc90dc1..cc2f89b2ac 100644 --- a/kernel/x86_64/gemm_kernel_8x4_core2.S +++ b/kernel/x86_64/gemm_kernel_8x4_core2.S @@ -161,10 +161,10 @@ movq N, J sarq $2, J - jle .L50 + jle L(50) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -177,10 +177,10 @@ movq K, %rax sarq $2, %rax - jle .L05 + jle L(05) ALIGN_4 -.L02: +L(02): prefetcht0 (PREFETCH_R + 0) * SIZE(B) movaps -28 * SIZE(B), %xmm7 movaps -24 * SIZE(B), %xmm11 @@ -233,17 +233,17 @@ subq $-16 * SIZE, B subq $-64 * SIZE, BO subq $1, %rax - jne .L02 + jne L(02) ALIGN_4 -.L05: +L(05): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L06: +L(06): movaps -32 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -259,10 +259,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO subq $1, %rax - jne .L06 + jne L(06) ALIGN_4 -.L10: +L(10): leaq (PREFETCH_R + 0) * SIZE(B), BB movq C, CO1 # coffset1 = c @@ -271,10 +271,10 @@ movq M, I sarq $3, I - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -330,10 +330,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): addps %xmm2, %xmm10 movaps -32 * SIZE(BO), %xmm2 addps %xmm3, %xmm14 @@ -463,10 +463,10 @@ subq $-64 * SIZE, BO subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): prefetcht2 -16 * SIZE(BB) #ifndef TRMMKERNEL @@ -476,10 +476,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): addps %xmm2, %xmm10 movaps -32 * SIZE(BO), %xmm2 addps %xmm3, %xmm14 @@ -514,10 +514,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): movaps ALPHA, %xmm7 addps %xmm2, %xmm10 @@ -602,12 +602,12 @@ addq $8 * SIZE, CO1 addq $8 * SIZE, CO2 subq $1, I - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M - jle .L30 + jle L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -643,10 +643,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L25 + jle L(25) ALIGN_4 -.L21: +L(21): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -716,10 +716,10 @@ subq $-16 * SIZE, AO subq $-64 * SIZE, BO subq $1, %rax - jg .L21 + jg L(21) ALIGN_4 -.L25: +L(25): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -728,10 +728,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): movaps -32 * SIZE(AO), %xmm0 movaps -32 * SIZE(BO), %xmm2 movaps -28 * SIZE(BO), %xmm3 @@ -751,10 +751,10 @@ addq $ 4 * SIZE, AO addq $16 * SIZE, BO subq $1, %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -807,9 +807,9 @@ subq $1, I ALIGN_4 -.L30: +L(30): testq $2, M - jle .L40 + jle L(40) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -850,10 +850,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L35 + jle L(35) ALIGN_4 -.L31: +L(31): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -32 * SIZE(AO), %xmm0 @@ -923,10 +923,10 @@ subq $ -8 * SIZE, AO subq $-64 * SIZE, BO subq $1, %rax - jg .L31 + jg L(31) ALIGN_4 -.L35: +L(35): movsd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -935,10 +935,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): movsd -32 * SIZE(AO), %xmm0 movsd -32 * SIZE(BO), %xmm2 movsd -28 * SIZE(BO), %xmm3 @@ -958,10 +958,10 @@ addq $ 2 * SIZE, AO addq $16 * SIZE, BO subq $1, %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movsd 0 * SIZE(CO2), %xmm2 @@ -1003,9 +1003,9 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L40: +L(40): testq $1, M - jle .L49 + jle L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1047,10 +1047,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L45 + jle L(45) ALIGN_4 -.L41: +L(41): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movss -32 * SIZE(AO), %xmm0 @@ -1120,10 +1120,10 @@ subq $ -4 * SIZE, AO subq $-64 * SIZE, BO subq $1, %rax - jg .L41 + jg L(41) ALIGN_4 -.L45: +L(45): movss ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1132,10 +1132,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): movss -32 * SIZE(AO), %xmm0 movss -32 * SIZE(BO), %xmm2 movss -28 * SIZE(BO), %xmm3 @@ -1155,10 +1155,10 @@ addq $ 1 * SIZE, AO addq $16 * SIZE, BO subq $1, %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): #ifndef TRMMKERNEL movss 0 * SIZE(CO1), %xmm0 movss 0 * SIZE(CO2), %xmm2 @@ -1198,22 +1198,22 @@ #endif ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C subq $1, J - jg .L01 + jg L(01) ALIGN_4 -.L50: +L(50): testq $2, N - jle .L100 + jle L(100) ALIGN_4 -.L51: +L(51): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -1224,12 +1224,12 @@ movq K, %rax sarq $3, %rax - jle .L53 + jle L(53) addq %rax, %rax ALIGN_4 -.L52: +L(52): movaps -32 * SIZE(B), %xmm3 movaps -28 * SIZE(B), %xmm7 @@ -1259,17 +1259,17 @@ addq $32 * SIZE, BO subq $1, %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): movq K, %rax andq $7, %rax BRANCH - jle .L55 + jle L(55) ALIGN_4 -.L54: +L(54): movss -32 * SIZE(B), %xmm8 movss -31 * SIZE(B), %xmm9 @@ -1282,20 +1282,20 @@ addq $2 * SIZE, B addq $8 * SIZE, BO subq $1, %rax - jne .L54 + jne L(54) ALIGN_4 -.L55: +L(55): movq C, CO1 leaq (C, LDC, 1), CO2 movq A, AO # aoffset = a movq M, I sarq $3, I - jle .L70 + jle L(70) ALIGN_4 -.L60: +L(60): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1333,10 +1333,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L65 + jle L(65) ALIGN_4 -.L61: +L(61): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -1414,10 +1414,10 @@ subq $-32 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jg .L61 + jg L(61) ALIGN_4 -.L65: +L(65): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1426,10 +1426,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): movaps -32 * SIZE(AO), %xmm0 movaps -28 * SIZE(AO), %xmm1 @@ -1451,10 +1451,10 @@ addq $8 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -1503,12 +1503,12 @@ addq $8 * SIZE, CO1 addq $8 * SIZE, CO2 subq $1, I - jg .L60 + jg L(60) ALIGN_4 -.L70: +L(70): testq $4, M - jle .L80 + jle L(80) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1544,10 +1544,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L75 + jle L(75) ALIGN_4 -.L71: +L(71): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -1587,10 +1587,10 @@ subq $-16 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jg .L71 + jg L(71) ALIGN_4 -.L75: +L(75): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1599,10 +1599,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): movaps -32 * SIZE(AO), %xmm0 movaps -32 * SIZE(BO), %xmm2 movaps -28 * SIZE(BO), %xmm3 @@ -1616,10 +1616,10 @@ addq $4 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm10, %xmm8 addps %xmm11, %xmm9 @@ -1659,9 +1659,9 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L80: +L(80): testq $2, M - jle .L90 + jle L(90) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1697,10 +1697,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L85 + jle L(85) ALIGN_4 -.L81: +L(81): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -32 * SIZE(AO), %xmm0 @@ -1740,10 +1740,10 @@ subq $ -8 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jg .L81 + jg L(81) ALIGN_4 -.L85: +L(85): movsd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1752,10 +1752,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): movsd -32 * SIZE(AO), %xmm0 movsd -32 * SIZE(BO), %xmm2 movsd -28 * SIZE(BO), %xmm3 @@ -1769,10 +1769,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): addps %xmm10, %xmm8 addps %xmm11, %xmm9 @@ -1809,9 +1809,9 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L90: +L(90): testq $1, M - jle .L99 + jle L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1847,10 +1847,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L95 + jle L(95) ALIGN_4 -.L91: +L(91): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1891,10 +1891,10 @@ subq $ -4 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jg .L91 + jg L(91) ALIGN_4 -.L95: +L(95): movss ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1903,10 +1903,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): movss -32 * SIZE(AO), %xmm0 movss -32 * SIZE(BO), %xmm2 movss -28 * SIZE(BO), %xmm3 @@ -1920,10 +1920,10 @@ addq $1 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): addss %xmm10, %xmm8 addss %xmm11, %xmm9 @@ -1957,7 +1957,7 @@ #endif ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -1967,12 +1967,12 @@ -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) ALIGN_4 -.L101: +L(101): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -1983,12 +1983,12 @@ movq K, %rax sarq $4, %rax - jle .L103 + jle L(103) addq %rax, %rax ALIGN_4 -.L102: +L(102): movss -32 * SIZE(B), %xmm0 movss -31 * SIZE(B), %xmm1 movss -30 * SIZE(B), %xmm2 @@ -2019,17 +2019,17 @@ addq $ 8 * SIZE, B subq $-32 * SIZE, BO subq $1, %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): movq K, %rax andq $15, %rax BRANCH - jle .L105 + jle L(105) ALIGN_4 -.L104: +L(104): movss -32 * SIZE(B), %xmm8 shufps $0, %xmm8, %xmm8 @@ -2039,19 +2039,19 @@ addq $1 * SIZE, B addq $4 * SIZE, BO subq $1, %rax - jne .L104 + jne L(104) ALIGN_4 -.L105: +L(105): movq C, CO1 movq A, AO movq M, I sarq $3, I - jle .L120 + jle L(120) ALIGN_4 -.L110: +L(110): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2088,10 +2088,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L115 + jle L(115) ALIGN_4 -.L111: +L(111): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -2137,10 +2137,10 @@ subq $-32 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L111 + jg L(111) ALIGN_4 -.L115: +L(115): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2149,10 +2149,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): movaps -32 * SIZE(AO), %xmm0 movaps -28 * SIZE(AO), %xmm1 @@ -2167,10 +2167,10 @@ addq $8 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addps %xmm9, %xmm8 addps %xmm13, %xmm12 @@ -2209,12 +2209,12 @@ addq $8 * SIZE, CO1 subq $1, I - jg .L110 + jg L(110) ALIGN_4 -.L120: +L(120): testq $4, M - jle .L130 + jle L(130) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2250,10 +2250,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L125 + jle L(125) ALIGN_4 -.L121: +L(121): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -2281,10 +2281,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L121 + jg L(121) ALIGN_4 -.L125: +L(125): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2293,10 +2293,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): movaps -32 * SIZE(AO), %xmm0 movaps -32 * SIZE(BO), %xmm2 @@ -2306,10 +2306,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): addps %xmm10, %xmm8 addps %xmm11, %xmm9 @@ -2345,9 +2345,9 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L130: +L(130): testq $2, M - jle .L140 + jle L(140) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2383,10 +2383,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L135 + jle L(135) ALIGN_4 -.L131: +L(131): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -32 * SIZE(AO), %xmm0 @@ -2412,10 +2412,10 @@ subq $ -8 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L131 + jg L(131) ALIGN_4 -.L135: +L(135): movsd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2424,10 +2424,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): movsd -32 * SIZE(AO), %xmm0 movsd -32 * SIZE(BO), %xmm2 @@ -2437,10 +2437,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 #endif @@ -2472,9 +2472,9 @@ addq $2 * SIZE, CO1 ALIGN_4 -.L140: +L(140): testq $1, M - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2510,10 +2510,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L145 + jle L(145) ALIGN_4 -.L141: +L(141): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movss -32 * SIZE(AO), %xmm0 @@ -2539,10 +2539,10 @@ subq $ -4 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L141 + jg L(141) ALIGN_4 -.L145: +L(145): movss ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2551,10 +2551,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): movss -32 * SIZE(AO), %xmm0 movss -32 * SIZE(BO), %xmm2 @@ -2564,10 +2564,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): #ifndef TRMMKERNEL movss 0 * SIZE(CO1), %xmm0 #endif @@ -2584,7 +2584,7 @@ movss %xmm8, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/gemm_kernel_8x4_penryn.S b/kernel/x86_64/gemm_kernel_8x4_penryn.S index b381de979a..9a2cbeaab5 100644 --- a/kernel/x86_64/gemm_kernel_8x4_penryn.S +++ b/kernel/x86_64/gemm_kernel_8x4_penryn.S @@ -164,10 +164,10 @@ movq N, J sarq $2, J NOBRANCH - jle .L50 + jle L(50) ALIGN_4 -.L10: +L(10): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -184,10 +184,10 @@ movq M, I sarq $3, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -247,10 +247,10 @@ #endif sarq $3, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH -32 * SIZE(PREA) addps %xmm6, %xmm10 addps %xmm3, %xmm14 @@ -501,10 +501,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): prefetcht0 -16 * SIZE(BB) #ifndef TRMMKERNEL @@ -514,10 +514,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addps %xmm6, %xmm10 addps %xmm3, %xmm14 movaps %xmm2, %xmm3 @@ -553,10 +553,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): addps %xmm6, %xmm10 addps %xmm3, %xmm14 addps %xmm4, %xmm11 @@ -664,13 +664,13 @@ addq $8 * SIZE, CO2 decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M BRANCH - jle .L30 + jle L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -716,10 +716,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): addps %xmm6, %xmm10 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x39, %xmm2, %xmm7 @@ -786,10 +786,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -797,10 +797,10 @@ #endif andq $3, %rax BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): addps %xmm6, %xmm10 pshufd $0x39, %xmm2, %xmm7 mulps %xmm0, %xmm2 @@ -821,10 +821,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): addps %xmm6, %xmm10 addps %xmm4, %xmm11 @@ -892,10 +892,10 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L30: +L(30): testq $2, M BRANCH - jle .L40 + jle L(40) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -940,10 +940,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm0, %xmm1 @@ -989,10 +989,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1000,10 +1000,10 @@ #endif andq $3, %rax BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): pshufd $0x44, %xmm0, %xmm1 movsd -30 * SIZE(AO), %xmm0 addps %xmm3, %xmm8 @@ -1019,10 +1019,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): movddup ALPHA, %xmm2 addps %xmm10, %xmm8 @@ -1066,10 +1066,10 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L40: +L(40): testq $1, M BRANCH - jle .L49 + jle L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1107,10 +1107,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x00, %xmm0, %xmm1 @@ -1142,10 +1142,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1153,10 +1153,10 @@ #endif andq $3, %rax BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 mulps %xmm1, %xmm2 @@ -1168,10 +1168,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): movddup ALPHA, %xmm2 addps %xmm9, %xmm8 mulps %xmm2, %xmm8 @@ -1207,7 +1207,7 @@ #endif ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif @@ -1218,12 +1218,12 @@ subq $1, J BRANCH - jg .L10 + jg L(10) ALIGN_4 -.L50: +L(50): testq $2, N - jle .L90 + jle L(90) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1241,10 +1241,10 @@ movq M, I sarq $3, I NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1296,10 +1296,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm3, %xmm8 @@ -1375,10 +1375,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1386,10 +1386,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addps %xmm3, %xmm8 pshufd $0x00, %xmm2, %xmm3 mulps %xmm0, %xmm3 @@ -1412,10 +1412,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): movddup ALPHA, %xmm7 addps %xmm3, %xmm8 @@ -1470,13 +1470,13 @@ addq $8 * SIZE, CO2 decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $4, M BRANCH - jle .L70 + jle L(70) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1519,10 +1519,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm3, %xmm8 @@ -1564,10 +1564,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1575,10 +1575,10 @@ #endif andq $3, %rax BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): addps %xmm3, %xmm8 pshufd $0x00, %xmm2, %xmm3 mulps %xmm0, %xmm3 @@ -1593,10 +1593,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): movddup ALPHA, %xmm7 addps %xmm10, %xmm8 @@ -1640,10 +1640,10 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L70: +L(70): testq $2, M BRANCH - jle .L80 + jle L(80) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1682,10 +1682,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm3, %xmm8 @@ -1717,10 +1717,10 @@ subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_3 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -1728,10 +1728,10 @@ #endif andq $3, %rax BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addps %xmm3, %xmm8 pshufd $0x44, %xmm0, %xmm1 movsd -30 * SIZE(AO), %xmm0 @@ -1744,10 +1744,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_3 -.L78: +L(78): movddup ALPHA, %xmm2 addps %xmm9, %xmm8 @@ -1782,10 +1782,10 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L80: +L(80): testq $1, M BRANCH - jle .L89 + jle L(89) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1823,10 +1823,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x00, %xmm0, %xmm1 @@ -1858,10 +1858,10 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) ALIGN_3 -.L85: +L(85): #ifndef TRMMKERNEL movq K, %rax #else @@ -1869,10 +1869,10 @@ #endif andq $3, %rax BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 mulps %xmm1, %xmm2 @@ -1884,10 +1884,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_3 -.L88: +L(88): movddup ALPHA, %xmm2 addps %xmm9, %xmm8 mulps %xmm2, %xmm8 @@ -1917,7 +1917,7 @@ #endif ALIGN_4 -.L89: +L(89): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -1927,9 +1927,9 @@ leaq (C, LDC, 2), C ALIGN_4 -.L90: +L(90): testq $1, N - jle .L999 + jle L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1942,10 +1942,10 @@ movq M, I sarq $3, I NOBRANCH - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1986,10 +1986,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_3 -.L92: +L(92): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x00, %xmm2, %xmm3 @@ -2033,10 +2033,10 @@ subq $1, %rax BRANCH - jg .L92 + jg L(92) ALIGN_3 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2044,10 +2044,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_3 -.L96: +L(96): pshufd $0x00, %xmm2, %xmm3 movss -31 * SIZE(BO), %xmm2 mulps %xmm3, %xmm0 @@ -2062,10 +2062,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_3 -.L98: +L(98): movddup ALPHA, %xmm7 addps %xmm10, %xmm8 @@ -2105,13 +2105,13 @@ addq $8 * SIZE, CO1 decq I BRANCH - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $4, M BRANCH - jle .L110 + jle L(110) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2149,10 +2149,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_3 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x00, %xmm2, %xmm3 @@ -2182,10 +2182,10 @@ subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_3 -.L105: +L(105): #ifndef TRMMKERNEL movq K, %rax #else @@ -2193,10 +2193,10 @@ #endif andq $3, %rax BRANCH - je .L108 + je L(108) ALIGN_3 -.L106: +L(106): pshufd $0x00, %xmm2, %xmm3 movss -31 * SIZE(BO), %xmm2 mulps %xmm0, %xmm3 @@ -2208,10 +2208,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_3 -.L108: +L(108): movddup ALPHA, %xmm7 addps %xmm9, %xmm8 @@ -2243,10 +2243,10 @@ addq $4 * SIZE, CO1 ALIGN_4 -.L110: +L(110): testq $2, M BRANCH - jle .L120 + jle L(120) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2285,10 +2285,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_3 -.L112: +L(112): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x00, %xmm2, %xmm3 @@ -2318,10 +2318,10 @@ subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_3 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2329,10 +2329,10 @@ #endif andq $3, %rax BRANCH - je .L118 + je L(118) ALIGN_3 -.L116: +L(116): pshufd $0x00, %xmm2, %xmm3 movss -31 * SIZE(BO), %xmm2 mulps %xmm0, %xmm3 @@ -2344,10 +2344,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_3 -.L118: +L(118): movddup ALPHA, %xmm2 mulps %xmm2, %xmm8 @@ -2376,10 +2376,10 @@ addq $2 * SIZE, CO1 ALIGN_4 -.L120: +L(120): testq $1, M BRANCH - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2417,10 +2417,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L125 + jle L(125) ALIGN_3 -.L122: +L(122): mulss %xmm0, %xmm2 movss -31 * SIZE(AO), %xmm0 addss %xmm2, %xmm8 @@ -2446,10 +2446,10 @@ subq $1, %rax BRANCH - jg .L122 + jg L(122) ALIGN_3 -.L125: +L(125): #ifndef TRMMKERNEL movq K, %rax #else @@ -2457,10 +2457,10 @@ #endif andq $3, %rax BRANCH - je .L128 + je L(128) ALIGN_3 -.L126: +L(126): mulss %xmm0, %xmm2 movss -31 * SIZE(AO), %xmm0 addss %xmm2, %xmm8 @@ -2471,10 +2471,10 @@ subq $1, %rax BRANCH - jg .L126 + jg L(126) ALIGN_3 -.L128: +L(128): movss ALPHA, %xmm2 addss %xmm9, %xmm8 mulss %xmm2, %xmm8 @@ -2486,7 +2486,7 @@ movss %xmm8, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/gemm_kernel_8x4_sse.S b/kernel/x86_64/gemm_kernel_8x4_sse.S index 1602c13c50..a93f597b63 100644 --- a/kernel/x86_64/gemm_kernel_8x4_sse.S +++ b/kernel/x86_64/gemm_kernel_8x4_sse.S @@ -408,9 +408,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L50 + jle L(50) -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -423,12 +423,12 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): PREFETCH (RPREFETCHSIZE + 0) * SIZE(B) movd 1 * SIZE(B), %mm1 @@ -475,17 +475,17 @@ addq $32 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movd 0 * SIZE(B), %mm0 movd 1 * SIZE(B), %mm1 movd 2 * SIZE(B), %mm2 @@ -508,10 +508,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a @@ -520,10 +520,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -583,10 +583,10 @@ leaq (BO, %rax, 8), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -607,7 +607,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -628,7 +628,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -649,7 +649,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -670,7 +670,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -691,7 +691,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -712,7 +712,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -733,7 +733,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -754,10 +754,10 @@ addq $16 * SIZE, %rax BRANCH - jl .L12 + jl L(12) ALIGN_3 -.L15: +L(15): PREFETCH -16 * SIZE(BB) subq $-16 * SIZE, BB @@ -767,7 +767,7 @@ movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_3 @@ -786,10 +786,10 @@ #else sarq $2, %rax NOBRANCH - jle .L16 + jle L(16) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -803,10 +803,10 @@ subq $-32 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) #endif -.L16: +L(16): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -815,7 +815,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L18 + je L(18) leaq (, %rax, 8), %rax leaq (AO, %rax, 4), AO @@ -823,7 +823,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulps %xmm0, %xmm1 addps %xmm1, %xmm8 movaps -28 * SIZE(BO, %rax, 8), %xmm1 @@ -850,10 +850,10 @@ movaps -20 * SIZE(AO, %rax, 4), %xmm2 addq $SIZE * 2, %rax - jl .L17 + jl L(17) ALIGN_4 -.L18: +L(18): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -936,12 +936,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -985,10 +985,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) @@ -1102,10 +1102,10 @@ addq $ 32 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1114,10 +1114,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1134,10 +1134,10 @@ addq $ 4 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): mulps %xmm15, %xmm0 mulps %xmm15, %xmm1 mulps %xmm15, %xmm2 @@ -1187,9 +1187,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $2, M - je .L40 + je L(40) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1233,10 +1233,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) @@ -1355,10 +1355,10 @@ addq $ 16 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1367,10 +1367,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1388,10 +1388,10 @@ addq $ 2 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): mulps %xmm15, %xmm0 mulps %xmm15, %xmm1 mulps %xmm15, %xmm2 @@ -1443,9 +1443,9 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L40: +L(40): testq $1, M - je .L49 + je L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1490,10 +1490,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): mulss %xmm8, %xmm9 addss %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) @@ -1612,10 +1612,10 @@ addq $ 8 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1624,10 +1624,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -1645,10 +1645,10 @@ addq $ 1 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): mulss %xmm15, %xmm0 mulss %xmm15, %xmm1 mulss %xmm15, %xmm2 @@ -1686,19 +1686,19 @@ #endif ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C # c += 4 * ldc decq J # j -- - jg .L01 + jg L(01) -.L50: +L(50): testq $2, N - je .L100 + je L(100) -.L51: +L(51): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1709,10 +1709,10 @@ movq K, %rax sarq $2, %rax - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): #if defined(PENTIUM4) || defined(GENERIC) movss 0 * SIZE(B), %xmm0 movss 1 * SIZE(B), %xmm1 @@ -1790,17 +1790,17 @@ #endif decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): movq K, %rax andq $3, %rax BRANCH - jle .L60 + jle L(60) ALIGN_4 -.L54: +L(54): #if defined(PENTIUM4) || defined(GENERIC) movss 0 * SIZE(B), %xmm0 movss 1 * SIZE(B), %xmm1 @@ -1828,20 +1828,20 @@ addq $ 2 * SIZE, B addq $ 8 * SIZE, BO decq %rax - jne .L54 + jne L(54) ALIGN_4 -.L60: +L(60): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L70 + jle L(70) ALIGN_4 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1889,10 +1889,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2012,10 +2012,10 @@ addq $64 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -2024,10 +2024,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -2044,10 +2044,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2098,12 +2098,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L61 + jg L(61) ALIGN_4 -.L70: +L(70): testq $4, M - je .L80 + je L(80) #if !defined(TRMMKERNEL) || \ @@ -2148,10 +2148,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2215,10 +2215,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2227,10 +2227,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -2241,10 +2241,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2285,9 +2285,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L80: +L(80): testq $2, M - je .L90 + je L(90) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2331,10 +2331,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L85 + je L(85) ALIGN_4 -.L82: +L(82): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) @@ -2405,10 +2405,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L85: +L(85): #ifndef TRMMKERNEL movq K, %rax #else @@ -2417,10 +2417,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -2432,10 +2432,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): #ifndef TRMMKERNEL #ifdef movsd xorps %xmm8, %xmm8 @@ -2478,9 +2478,9 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L90: +L(90): testq $1, M - je .L99 + je L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2524,10 +2524,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) @@ -2598,10 +2598,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2610,10 +2610,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -2625,10 +2625,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): #ifndef TRMMKERNEL movss 0 * SIZE(CO1), %xmm8 movss 0 * SIZE(CO2), %xmm10 @@ -2661,7 +2661,7 @@ #endif ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -2669,11 +2669,11 @@ ALIGN_4 -.L100: +L(100): testq $1, N - je .L999 + je L(999) -.L101: +L(101): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -2684,11 +2684,11 @@ movq K, %rax sarq $3, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): #if defined(PENTIUM4) || defined(GENERIC) movss 0 * SIZE(B), %xmm0 movss 1 * SIZE(B), %xmm1 @@ -2766,17 +2766,17 @@ #endif decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): movq K, %rax andq $7, %rax BRANCH - jle .L110 + jle L(110) ALIGN_4 -.L104: +L(104): #if defined(PENTIUM4) || defined(GENERIC) movss 0 * SIZE(B), %xmm0 shufps $0, %xmm0, %xmm0 @@ -2793,19 +2793,19 @@ addq $ 1 * SIZE, B addq $ 4 * SIZE, BO decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L110: +L(110): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L120 + jle L(120) ALIGN_4 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2852,10 +2852,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulps %xmm9, %xmm8 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2928,10 +2928,10 @@ addq $64 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2940,10 +2940,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulps %xmm9, %xmm8 mulps -28 * SIZE(AO), %xmm9 addps %xmm8, %xmm0 @@ -2954,10 +2954,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2992,12 +2992,12 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L111 + jg L(111) ALIGN_4 -.L120: +L(120): testq $4, M - je .L130 + je L(130) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3039,10 +3039,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L125 + je L(125) ALIGN_4 -.L122: +L(122): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3080,10 +3080,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L122 + jne L(122) ALIGN_4 -.L125: +L(125): #ifndef TRMMKERNEL movq K, %rax #else @@ -3092,10 +3092,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): mulps %xmm8, %xmm9 movaps -28 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -3104,10 +3104,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -3141,9 +3141,9 @@ addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L130: +L(130): testq $2, M - je .L140 + je L(140) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3185,10 +3185,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L135 + je L(135) ALIGN_4 -.L132: +L(132): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3234,10 +3234,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L132 + jne L(132) ALIGN_4 -.L135: +L(135): #ifndef TRMMKERNEL movq K, %rax #else @@ -3246,10 +3246,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): mulps %xmm8, %xmm9 movsd -30 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -3258,10 +3258,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): addps %xmm1, %xmm0 mulps %xmm15, %xmm0 @@ -3291,9 +3291,9 @@ addq $2 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L140: +L(140): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3335,10 +3335,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L145 + je L(145) ALIGN_4 -.L142: +L(142): mulss %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3372,10 +3372,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L142 + jne L(142) ALIGN_4 -.L145: +L(145): #ifndef TRMMKERNEL movq K, %rax #else @@ -3384,10 +3384,10 @@ movss ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): mulss %xmm8, %xmm9 movss -31 * SIZE(AO), %xmm8 addss %xmm9, %xmm0 @@ -3396,10 +3396,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): addss %xmm1, %xmm0 addss %xmm3, %xmm2 addss %xmm2, %xmm0 @@ -3413,7 +3413,7 @@ movss %xmm0, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp EMMS diff --git a/kernel/x86_64/gemm_kernel_8x4_sse3.S b/kernel/x86_64/gemm_kernel_8x4_sse3.S index c853e46d18..324836e075 100644 --- a/kernel/x86_64/gemm_kernel_8x4_sse3.S +++ b/kernel/x86_64/gemm_kernel_8x4_sse3.S @@ -394,9 +394,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L50 + jle L(50) -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -407,10 +407,10 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): movddup 0 * SIZE(B), %xmm0 movddup 2 * SIZE(B), %xmm1 movddup 4 * SIZE(B), %xmm2 @@ -436,17 +436,17 @@ addq $32 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movddup 0 * SIZE(B), %xmm0 movddup 2 * SIZE(B), %xmm1 @@ -456,10 +456,10 @@ addq $4 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a @@ -468,10 +468,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): prefetcht0 0 * SIZE(BB) prefetcht0 8 * SIZE(BB) subq $-16 * SIZE, BB @@ -532,9 +532,9 @@ #if 1 andq $-8, %rax salq $4, %rax - je .L15 + je L(15) -.L1X: +L(1X): KERNEL1 (64 * 0) KERNEL2 (64 * 0) KERNEL3 (64 * 0) @@ -553,7 +553,7 @@ KERNEL16(64 * 0) cmpq $128 * 1, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 1) KERNEL2 (64 * 1) KERNEL3 (64 * 1) @@ -572,7 +572,7 @@ KERNEL16(64 * 1) cmpq $128 * 2, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 2) KERNEL2 (64 * 2) KERNEL3 (64 * 2) @@ -591,7 +591,7 @@ KERNEL16(64 * 2) cmpq $128 * 3, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 3) KERNEL2 (64 * 3) KERNEL3 (64 * 3) @@ -610,7 +610,7 @@ KERNEL16(64 * 3) cmpq $128 * 4, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 4) KERNEL2 (64 * 4) KERNEL3 (64 * 4) @@ -629,7 +629,7 @@ KERNEL16(64 * 4) cmpq $128 * 5, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 5) KERNEL2 (64 * 5) KERNEL3 (64 * 5) @@ -648,7 +648,7 @@ KERNEL16(64 * 5) cmpq $128 * 6, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 6) KERNEL2 (64 * 6) KERNEL3 (64 * 6) @@ -667,7 +667,7 @@ KERNEL16(64 * 6) cmpq $128 * 7, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 7) KERNEL2 (64 * 7) KERNEL3 (64 * 7) @@ -688,17 +688,17 @@ addq $64 * 8 * SIZE, AO addq $64 * 8 * SIZE, BO subq $128 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO leaq (BO, %rax, 2), BO #else sarq $3, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): KERNEL1 (64 * 0) KERNEL2 (64 * 0) KERNEL3 (64 * 0) @@ -719,11 +719,11 @@ addq $64 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L12 + jne L(12) #endif ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else @@ -732,10 +732,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -766,10 +766,10 @@ addq $8 * SIZE, AO addq $8 * SIZE, BO decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 mulps %xmm15, %xmm0 @@ -859,12 +859,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -908,10 +908,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm9, %xmm0 @@ -1021,10 +1021,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1033,10 +1033,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -1054,10 +1054,10 @@ addq $4 * SIZE, AO addq $8 * SIZE, BO decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -1112,9 +1112,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $2, M - je .L40 + je L(40) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1155,10 +1155,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): shufps $0x50, %xmm9, %xmm9 mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1236,10 +1236,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1248,10 +1248,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): shufps $0x50, %xmm9, %xmm9 mulps %xmm8, %xmm9 addps %xmm9, %xmm0 @@ -1265,10 +1265,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhps 0 * SIZE(CO2), %xmm8 @@ -1309,9 +1309,9 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L40: +L(40): testq $1, M - je .L49 + je L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1350,10 +1350,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): shufps $0, %xmm8, %xmm8 movhps 4 * SIZE(BO), %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1407,10 +1407,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1419,10 +1419,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): shufps $0, %xmm8, %xmm8 movhps 4 * SIZE(BO), %xmm9 mulps %xmm8, %xmm9 @@ -1433,10 +1433,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movss 0 * SIZE(CO1), %xmm8 movss 0 * SIZE(CO2), %xmm9 @@ -1485,19 +1485,19 @@ #endif ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C # c += 4 * ldc decq J # j -- - jg .L01 + jg L(01) -.L50: +L(50): testq $2, N - je .L100 + je L(100) -.L51: +L(51): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1508,10 +1508,10 @@ movq K, %rax sarq $3, %rax - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): movddup 0 * SIZE(B), %xmm0 movddup 2 * SIZE(B), %xmm1 movddup 4 * SIZE(B), %xmm2 @@ -1537,37 +1537,37 @@ addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): movq K, %rax andq $7, %rax BRANCH - jle .L60 + jle L(60) ALIGN_4 -.L54: +L(54): movddup 0 * SIZE(B), %xmm0 movaps %xmm0, 0 * SIZE(BO) addq $ 2 * SIZE, B addq $ 4 * SIZE, BO decq %rax - jne .L54 + jne L(54) ALIGN_4 -.L60: +L(60): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L70 + jle L(70) ALIGN_4 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1612,10 +1612,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm9, %xmm0 @@ -1734,10 +1734,10 @@ addq $64 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1746,10 +1746,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -1768,10 +1768,10 @@ addq $8 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -1822,12 +1822,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L61 + jg L(61) ALIGN_4 -.L70: +L(70): testq $4, M - je .L80 + je L(80) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1868,10 +1868,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm9, %xmm0 @@ -1933,10 +1933,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -1945,10 +1945,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -1960,10 +1960,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2004,9 +2004,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L80: +L(80): testq $2, M - je .L90 + je L(90) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2045,10 +2045,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L85 + je L(85) ALIGN_4 -.L82: +L(82): shufps $0x50, %xmm9, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulps %xmm8, %xmm9 @@ -2094,10 +2094,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L85: +L(85): #ifndef TRMMKERNEL movq K, %rax #else @@ -2106,10 +2106,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): shufps $0x50, %xmm9, %xmm9 mulps %xmm8, %xmm9 movddup 2 * SIZE(AO), %xmm8 @@ -2119,10 +2119,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhps 0 * SIZE(CO2), %xmm8 @@ -2154,9 +2154,9 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L90: +L(90): testq $1, M - je .L99 + je L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2195,10 +2195,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): shufps $0, %xmm8, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulps %xmm8, %xmm9 @@ -2244,10 +2244,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2256,10 +2256,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): shufps $0, %xmm8, %xmm8 mulps %xmm8, %xmm9 movss 1 * SIZE(AO), %xmm8 @@ -2269,10 +2269,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movss 0 * SIZE(CO1), %xmm8 movss 0 * SIZE(CO2), %xmm9 @@ -2308,18 +2308,18 @@ #endif ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif leaq (C, LDC, 2), C # c += 4 * ldc ALIGN_4 -.L100: +L(100): testq $1, N - je .L999 + je L(999) -.L101: +L(101): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -2330,11 +2330,11 @@ movq K, %rax sarq $3, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movss 0 * SIZE(B), %xmm0 movss 1 * SIZE(B), %xmm1 movss 2 * SIZE(B), %xmm2 @@ -2365,17 +2365,17 @@ addq $16 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): movq K, %rax andq $7, %rax BRANCH - jle .L110 + jle L(110) ALIGN_4 -.L104: +L(104): movss 0 * SIZE(B), %xmm0 movss %xmm0, 0 * SIZE(BO) movss %xmm0, 1 * SIZE(BO) @@ -2383,19 +2383,19 @@ addq $ 1 * SIZE, B addq $ 2 * SIZE, BO decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L110: +L(110): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L120 + jle L(120) ALIGN_4 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2439,10 +2439,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulps %xmm8, %xmm9 movaps 4 * SIZE(AO), %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2513,10 +2513,10 @@ addq $64 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2525,10 +2525,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulps %xmm8, %xmm9 movaps 4 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -2541,10 +2541,10 @@ addq $8 * SIZE, AO addq $2 * SIZE, BO decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2582,12 +2582,12 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L111 + jg L(111) ALIGN_4 -.L120: +L(120): testq $4, M - je .L130 + je L(130) #if !defined(TRMMKERNEL) || \ @@ -2633,10 +2633,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L125 + je L(125) ALIGN_4 -.L122: +L(122): mulps %xmm8, %xmm9 movaps 4 * SIZE(AO), %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2674,10 +2674,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L122 + jne L(122) ALIGN_4 -.L125: +L(125): #ifndef TRMMKERNEL movq K, %rax #else @@ -2686,10 +2686,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): mulps %xmm8, %xmm9 movaps 4 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -2698,10 +2698,10 @@ addq $4 * SIZE, AO addq $2 * SIZE, BO decq %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2733,9 +2733,9 @@ addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L130: +L(130): testq $2, M - je .L140 + je L(140) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2776,10 +2776,10 @@ movq %rax, KKK #endif sarq $4, %rax - je .L135 + je L(135) ALIGN_4 -.L132: +L(132): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps 4 * SIZE(AO), %xmm8 @@ -2811,10 +2811,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L132 + jne L(132) ALIGN_4 -.L135: +L(135): #ifndef TRMMKERNEL movq K, %rax #else @@ -2823,10 +2823,10 @@ movaps ALPHA, %xmm15 andq $15, %rax # if (k & 1) BRANCH - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): movsd 0 * SIZE(AO), %xmm8 movsd 0 * SIZE(BO), %xmm9 @@ -2836,10 +2836,10 @@ addq $2 * SIZE, AO addq $2 * SIZE, BO decq %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 #endif @@ -2872,9 +2872,9 @@ addq $2 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L140: +L(140): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2915,10 +2915,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L145 + je L(145) ALIGN_4 -.L142: +L(142): mulss %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movss 1 * SIZE(AO), %xmm8 @@ -2950,10 +2950,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L142 + jne L(142) ALIGN_4 -.L145: +L(145): #ifndef TRMMKERNEL movq K, %rax #else @@ -2962,10 +2962,10 @@ movss ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): movss 0 * SIZE(AO), %xmm8 movss 0 * SIZE(BO), %xmm9 mulps %xmm8, %xmm9 @@ -2974,10 +2974,10 @@ addq $1 * SIZE, AO addq $2 * SIZE, BO decq %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movss 0 * SIZE(CO1), %xmm8 #endif @@ -2992,7 +2992,7 @@ movss %xmm0, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/gemm_ncopy_2.S b/kernel/x86_64/gemm_ncopy_2.S index b069f9cf7f..d132a9b65d 100644 --- a/kernel/x86_64/gemm_ncopy_2.S +++ b/kernel/x86_64/gemm_ncopy_2.S @@ -115,20 +115,20 @@ movq N, J sarq $1, J - jle .L20 + jle L(20) ALIGN_4 -.L12: +L(12): movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), A movq M, I sarq $2, I - jle .L14 + jle L(14) ALIGN_4 -.L13: +L(13): #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 movss 0 * SIZE(AO2), %xmm1 @@ -174,16 +174,16 @@ addq $4 * SIZE, AO2 subq $-8 * SIZE, B decq I - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): movq M, I andq $3, I - jle .L16 + jle L(16) ALIGN_4 -.L15: +L(15): #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 movss 0 * SIZE(AO2), %xmm1 @@ -200,26 +200,26 @@ addq $SIZE, AO2 addq $2 * SIZE, B decq I - jg .L15 + jg L(15) ALIGN_4 -.L16: +L(16): decq J - jg .L12 + jg L(12) ALIGN_4 -.L20: +L(20): testq $1, N - jle .L999 + jle L(999) movq A, AO1 movq M, I sarq $2, I - jle .L34 + jle L(34) ALIGN_4 -.L33: +L(33): #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 movss 1 * SIZE(AO1), %xmm1 @@ -244,16 +244,16 @@ addq $4 * SIZE, AO1 subq $-4 * SIZE, B decq I - jg .L33 + jg L(33) ALIGN_4 -.L34: +L(34): movq M, I andq $3, I - jle .L999 + jle L(999) ALIGN_4 -.L35: +L(35): #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 movss %xmm0, 0 * SIZE(B) @@ -265,11 +265,11 @@ addq $SIZE, AO1 addq $1 * SIZE, B decq I - jg .L35 + jg L(35) ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 movups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/gemm_ncopy_2_bulldozer.S b/kernel/x86_64/gemm_ncopy_2_bulldozer.S index 1911d3c748..477e76517c 100644 --- a/kernel/x86_64/gemm_ncopy_2_bulldozer.S +++ b/kernel/x86_64/gemm_ncopy_2_bulldozer.S @@ -102,20 +102,20 @@ movq N, J sarq $1, J - jle .L20 + jle L(20) ALIGN_4 -.L01: +L(01): movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), A movq M, I sarq $3, I - jle .L08 + jle L(08) ALIGN_4 -.L03: +L(03): #ifndef DOUBLE vmovss 0 * SIZE(AO1), %xmm0 @@ -194,18 +194,18 @@ addq $8 * SIZE, AO2 subq $-16 * SIZE, B decq I - jg .L03 + jg L(03) ALIGN_4 -.L08: +L(08): testq $4 , M - je .L14 + je L(14) ALIGN_4 -.L13: +L(13): #ifndef DOUBLE vmovss 0 * SIZE(AO1), %xmm0 vmovss 0 * SIZE(AO2), %xmm1 @@ -248,13 +248,13 @@ subq $-8 * SIZE, B ALIGN_4 -.L14: +L(14): movq M, I andq $3, I - jle .L16 + jle L(16) ALIGN_4 -.L15: +L(15): #ifndef DOUBLE vmovss 0 * SIZE(AO1), %xmm0 vmovss 0 * SIZE(AO2), %xmm1 @@ -272,26 +272,26 @@ addq $SIZE, AO2 addq $2 * SIZE, B decq I - jg .L15 + jg L(15) ALIGN_4 -.L16: +L(16): decq J - jg .L01 + jg L(01) ALIGN_4 -.L20: +L(20): testq $1, N - jle .L999 + jle L(999) movq A, AO1 movq M, I sarq $2, I - jle .L34 + jle L(34) ALIGN_4 -.L33: +L(33): #ifndef DOUBLE vmovups 0 * SIZE(AO1), %xmm0 @@ -307,16 +307,16 @@ addq $4 * SIZE, AO1 subq $-4 * SIZE, B decq I - jg .L33 + jg L(33) ALIGN_4 -.L34: +L(34): movq M, I andq $3, I - jle .L999 + jle L(999) ALIGN_4 -.L35: +L(35): #ifndef DOUBLE vmovss 0 * SIZE(AO1), %xmm0 vmovss %xmm0, 0 * SIZE(B) @@ -328,11 +328,11 @@ addq $SIZE, AO1 addq $1 * SIZE, B decq I - jg .L35 + jg L(35) ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI vmovups 0(%rsp), %xmm6 vmovups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/gemm_ncopy_4.S b/kernel/x86_64/gemm_ncopy_4.S index d30e9d362f..f9b2ef9074 100644 --- a/kernel/x86_64/gemm_ncopy_4.S +++ b/kernel/x86_64/gemm_ncopy_4.S @@ -143,10 +143,10 @@ movq N, J sarq $2, J - jle .L20 + jle L(20) ALIGN_4 -.L12: +L(12): movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), AO3 @@ -155,10 +155,10 @@ movq M, I sarq $2, I - jle .L14 + jle L(14) ALIGN_4 -.L13: +L(13): #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 movss 0 * SIZE(AO2), %xmm1 @@ -262,16 +262,16 @@ subq $-16 * SIZE, B decq I - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): movq M, I andq $3, I - jle .L16 + jle L(16) ALIGN_4 -.L15: +L(15): #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 movss 0 * SIZE(AO2), %xmm1 @@ -298,17 +298,17 @@ addq $SIZE, AO4 addq $4 * SIZE, B decq I - jg .L15 + jg L(15) ALIGN_4 -.L16: +L(16): decq J - jg .L12 + jg L(12) ALIGN_4 -.L20: +L(20): testq $2, N - jle .L30 + jle L(30) movq A, AO1 leaq (A, LDA), AO2 @@ -316,10 +316,10 @@ movq M, I sarq $2, I - jle .L24 + jle L(24) ALIGN_4 -.L23: +L(23): #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 movss 0 * SIZE(AO2), %xmm1 @@ -369,16 +369,16 @@ addq $4 * SIZE, AO2 subq $-8 * SIZE, B decq I - jg .L23 + jg L(23) ALIGN_4 -.L24: +L(24): movq M, I andq $3, I - jle .L30 + jle L(30) ALIGN_4 -.L25: +L(25): #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 movss 0 * SIZE(AO2), %xmm1 @@ -396,21 +396,21 @@ addq $SIZE, AO2 addq $2 * SIZE, B decq I - jg .L25 + jg L(25) ALIGN_4 -.L30: +L(30): testq $1, N - jle .L999 + jle L(999) movq A, AO1 movq M, I sarq $2, I - jle .L34 + jle L(34) ALIGN_4 -.L33: +L(33): #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 movss 1 * SIZE(AO1), %xmm1 @@ -435,16 +435,16 @@ addq $4 * SIZE, AO1 subq $-4 * SIZE, B decq I - jg .L33 + jg L(33) ALIGN_4 -.L34: +L(34): movq M, I andq $3, I - jle .L999 + jle L(999) ALIGN_4 -.L35: +L(35): #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 movss %xmm0, 0 * SIZE(B) @@ -456,11 +456,11 @@ addq $SIZE, AO1 addq $1 * SIZE, B decq I - jg .L35 + jg L(35) ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 movups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/gemm_ncopy_4_opteron.S b/kernel/x86_64/gemm_ncopy_4_opteron.S index ea39f89367..21b8b233b9 100644 --- a/kernel/x86_64/gemm_ncopy_4_opteron.S +++ b/kernel/x86_64/gemm_ncopy_4_opteron.S @@ -118,10 +118,10 @@ movq N, J sarq $2, J - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if 0 movq A, AO1 leaq (A, LDA, 1), AO2 @@ -130,10 +130,10 @@ movq M, I sarq $4, I - jle .L13 + jle L(13) ALIGN_4 -.L12: +L(12): MOVQ 0 * SIZE(AO1), %mm0 addq $8 * SIZE, AO1 MOVQ 0 * SIZE(AO2), %mm1 @@ -144,10 +144,10 @@ addq $8 * SIZE, AO4 decq I - jg .L12 + jg L(12) ALIGN_4 -.L13: +L(13): #endif movq A, AO1 @@ -158,10 +158,10 @@ movq M, I sarq $2, I - jle .L15 + jle L(15) ALIGN_4 -.L14: +L(14): RPREFETCH (RPREFETCHSIZE) * SIZE(AO1) MOVQ 0 * SIZE(AO1), %mm0 @@ -217,16 +217,16 @@ subq $-16 * SIZE, B decq I - jg .L14 + jg L(14) ALIGN_4 -.L15: +L(15): movq M, I andq $3, I - jle .L17 + jle L(17) ALIGN_4 -.L16: +L(16): MOVQ 0 * SIZE(AO1), %mm0 MOVQ 0 * SIZE(AO2), %mm1 MOVQ 0 * SIZE(AO3), %mm2 @@ -243,17 +243,17 @@ addq $SIZE, AO4 addq $4 * SIZE, B decq I - jg .L16 + jg L(16) ALIGN_4 -.L17: +L(17): decq J - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, N - jle .L30 + jle L(30) movq A, AO1 leaq (A, LDA), AO2 @@ -261,10 +261,10 @@ movq M, I sarq $2, I - jle .L24 + jle L(24) ALIGN_4 -.L23: +L(23): prefetch (RPREFETCHSIZE) * SIZE(AO1) MOVQ 0 * SIZE(AO1), %mm0 prefetch (RPREFETCHSIZE) * SIZE(AO2) @@ -291,16 +291,16 @@ addq $4 * SIZE, AO2 subq $-8 * SIZE, B decq I - jg .L23 + jg L(23) ALIGN_4 -.L24: +L(24): movq M, I andq $3, I - jle .L30 + jle L(30) ALIGN_4 -.L25: +L(25): MOVQ 0 * SIZE(AO1), %mm0 MOVQ 0 * SIZE(AO2), %mm1 @@ -311,21 +311,21 @@ addq $SIZE, AO2 addq $2 * SIZE, B decq I - jg .L25 + jg L(25) ALIGN_4 -.L30: +L(30): testq $1, N - jle .L999 + jle L(999) movq A, AO1 movq M, I sarq $2, I - jle .L34 + jle L(34) ALIGN_4 -.L33: +L(33): MOVQ 0 * SIZE(AO1), %mm0 MOVQ 1 * SIZE(AO1), %mm1 MOVQ 2 * SIZE(AO1), %mm2 @@ -339,27 +339,27 @@ addq $4 * SIZE, AO1 subq $-4 * SIZE, B decq I - jg .L33 + jg L(33) ALIGN_4 -.L34: +L(34): movq M, I andq $3, I - jle .L999 + jle L(999) ALIGN_4 -.L35: +L(35): MOVQ 0 * SIZE(AO1), %mm0 addq $SIZE, AO1 MOVNTQ %mm0, 0 * SIZE(B) addq $1 * SIZE, B decq I - jg .L35 + jg L(35) ALIGN_4 -.L999: +L(999): EMMS #ifdef WINDOWS_ABI diff --git a/kernel/x86_64/gemm_tcopy_2.S b/kernel/x86_64/gemm_tcopy_2.S index f35427b798..2e970ce424 100644 --- a/kernel/x86_64/gemm_tcopy_2.S +++ b/kernel/x86_64/gemm_tcopy_2.S @@ -140,10 +140,10 @@ movq M, J sarq $1, J - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): movq A, AO1 leaq (A, LDA ), AO2 leaq (A, LDA, 2), A @@ -153,10 +153,10 @@ movq N, I sarq $1, I - jle .L14 + jle L(14) ALIGN_4 -.L12: +L(12): #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 movhps 0 * SIZE(AO2), %xmm0 @@ -179,12 +179,12 @@ addq $2 * SIZE, AO1 addq $2 * SIZE, AO2 decq I - jg .L12 + jg L(12) ALIGN_4 -.L14: +L(14): testq $1, N - jle .L19 + jle L(19) #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 @@ -202,26 +202,26 @@ addq $2 * SIZE, BO1 ALIGN_4 -.L19: +L(19): decq J - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M - jle .L999 + jle L(999) ALIGN_4 -.L31: +L(31): movq A, AO1 movq B, BO movq N, I sarq $1, I - jle .L33 + jle L(33) ALIGN_4 -.L32: +L(32): #ifndef DOUBLE movsd 0 * SIZE(AO1), %xmm0 movsd %xmm0, 0 * SIZE(BO) @@ -234,12 +234,12 @@ addq $2 * SIZE, AO1 leaq (BO, M8, 2), BO decq I - jg .L32 + jg L(32) ALIGN_4 -.L33: +L(33): testq $1, N - jle .L999 + jle L(999) #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 @@ -251,7 +251,7 @@ addq $1 * SIZE, BO1 ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 movups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/gemm_tcopy_2_bulldozer.S b/kernel/x86_64/gemm_tcopy_2_bulldozer.S index d7552042eb..89af090edd 100644 --- a/kernel/x86_64/gemm_tcopy_2_bulldozer.S +++ b/kernel/x86_64/gemm_tcopy_2_bulldozer.S @@ -126,10 +126,10 @@ movq M, J sarq $1, J - jle .L20 + jle L(20) ALIGN_4 -.L01: +L(01): movq A, AO1 leaq (A, LDA ), AO2 leaq (A, LDA, 2), A @@ -139,11 +139,11 @@ movq N, I sarq $3, I - jle .L10 + jle L(10) ALIGN_4 -.L08: +L(08): #ifndef DOUBLE vmovsd 0 * SIZE(AO1), %xmm0 @@ -206,14 +206,14 @@ addq $8 * SIZE, AO1 addq $8 * SIZE, AO2 decq I - jg .L08 + jg L(08) ALIGN_4 -.L10: +L(10): testq $4, N - jle .L12 + jle L(12) #ifndef DOUBLE vmovsd 0 * SIZE(AO1), %xmm0 @@ -252,9 +252,9 @@ ALIGN_4 -.L12: +L(12): testq $2, N - jle .L14 + jle L(14) #ifndef DOUBLE vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO2), %xmm1 @@ -274,9 +274,9 @@ addq $2 * SIZE, AO2 ALIGN_4 -.L14: +L(14): testq $1, N - jle .L19 + jle L(19) #ifndef DOUBLE vmovss 0 * SIZE(AO1), %xmm0 @@ -294,26 +294,26 @@ addq $2 * SIZE, BO1 ALIGN_4 -.L19: +L(19): decq J - jg .L01 + jg L(01) ALIGN_4 -.L20: +L(20): testq $1, M - jle .L999 + jle L(999) ALIGN_4 -.L31: +L(31): movq A, AO1 movq B, BO movq N, I sarq $1, I - jle .L33 + jle L(33) ALIGN_4 -.L32: +L(32): #ifndef DOUBLE vmovsd 0 * SIZE(AO1), %xmm0 vmovsd %xmm0, 0 * SIZE(BO) @@ -325,12 +325,12 @@ addq $2 * SIZE, AO1 leaq (BO, M8, 2), BO decq I - jg .L32 + jg L(32) ALIGN_4 -.L33: +L(33): testq $1, N - jle .L999 + jle L(999) #ifndef DOUBLE vmovss 0 * SIZE(AO1), %xmm0 @@ -342,7 +342,7 @@ addq $1 * SIZE, BO1 ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI vmovups 0(%rsp), %xmm6 vmovups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/gemm_tcopy_4.S b/kernel/x86_64/gemm_tcopy_4.S index 177587c4b7..551c6e1089 100644 --- a/kernel/x86_64/gemm_tcopy_4.S +++ b/kernel/x86_64/gemm_tcopy_4.S @@ -174,10 +174,10 @@ movq M, J sarq $2, J - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): movq A, AO1 leaq (A, LDA ), AO2 leaq (A, LDA, 2), AO3 @@ -189,10 +189,10 @@ movq N, I sarq $2, I - jle .L13 + jle L(13) ALIGN_4 -.L12: +L(12): #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 movhps 2 * SIZE(AO1), %xmm0 @@ -274,12 +274,12 @@ addq $4 * SIZE, AO3 addq $4 * SIZE, AO4 decq I - jg .L12 + jg L(12) ALIGN_4 -.L13: +L(13): testq $2, N - jle .L14 + jle L(14) #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 @@ -314,9 +314,9 @@ addq $8 * SIZE, BO1 ALIGN_4 -.L14: +L(14): testq $1, N - jle .L19 + jle L(19) #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 @@ -341,17 +341,17 @@ addq $4 * SIZE, BO2 ALIGN_4 -.L19: +L(19): decq J - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M - jle .L30 + jle L(30) ALIGN_4 -.L21: +L(21): movq A, AO1 leaq (A, LDA ), AO2 leaq (A, LDA, 2), A @@ -361,10 +361,10 @@ movq N, I sarq $2, I - jle .L23 + jle L(23) ALIGN_4 -.L22: +L(22): #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 movhps 2 * SIZE(AO1), %xmm0 @@ -407,12 +407,12 @@ addq $4 * SIZE, AO2 leaq (BO, M8, 4), BO decq I - jg .L22 + jg L(22) ALIGN_4 -.L23: +L(23): testq $2, N - jle .L24 + jle L(24) #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 @@ -434,9 +434,9 @@ addq $4 * SIZE, BO1 ALIGN_4 -.L24: +L(24): testq $1, N - jle .L30 + jle L(30) #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 @@ -453,21 +453,21 @@ addq $2 * SIZE, BO2 ALIGN_4 -.L30: +L(30): testq $1, M - jle .L999 + jle L(999) ALIGN_4 -.L31: +L(31): movq A, AO1 movq B, BO movq N, I sarq $2, I - jle .L33 + jle L(33) ALIGN_4 -.L32: +L(32): #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 movhps 2 * SIZE(AO1), %xmm0 @@ -486,12 +486,12 @@ addq $4 * SIZE, AO1 leaq (BO, M8, 4), BO decq I - jg .L32 + jg L(32) ALIGN_4 -.L33: +L(33): testq $2, N - jle .L34 + jle L(34) #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 @@ -508,9 +508,9 @@ addq $2 * SIZE, BO1 ALIGN_4 -.L34: +L(34): testq $1, N - jle .L999 + jle L(999) #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 @@ -522,7 +522,7 @@ addq $1 * SIZE, BO2 ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 movups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/gemm_tcopy_4_opteron.S b/kernel/x86_64/gemm_tcopy_4_opteron.S index e8207ace37..05f27f482f 100644 --- a/kernel/x86_64/gemm_tcopy_4_opteron.S +++ b/kernel/x86_64/gemm_tcopy_4_opteron.S @@ -149,10 +149,10 @@ leaq (, M, SIZE), M8 movq M, J sarq $2, J - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if 0 movq A, AO1 leaq (A, LDA, 1), AO2 @@ -161,10 +161,10 @@ movq N, I sarq $3, I - jle .L13 + jle L(13) ALIGN_4 -.L12: +L(12): MOVQ 0 * SIZE(AO1), %mm0 addq $8 * SIZE, AO1 MOVQ 0 * SIZE(AO2), %mm1 @@ -175,10 +175,10 @@ addq $8 * SIZE, AO4 decq I - jg .L12 + jg L(12) ALIGN_4 -.L13: +L(13): #endif movq A, AO1 @@ -192,10 +192,10 @@ movq N, I sarq $2, I - jle .L15 + jle L(15) ALIGN_4 -.L14: +L(14): RPREFETCH (RPREFETCHSIZE) * SIZE(AO1) @@ -253,12 +253,12 @@ leaq (BO, M8, 4), BO decq I - jg .L14 + jg L(14) ALIGN_4 -.L15: +L(15): testq $2, N - jle .L16 + jle L(16) MOVQ 0 * SIZE(AO1), %mm0 MOVQ 1 * SIZE(AO1), %mm1 @@ -286,9 +286,9 @@ addq $8 * SIZE, BO1 ALIGN_4 -.L16: +L(16): testq $1, N - jle .L19 + jle L(19) MOVQ 0 * SIZE(AO1), %mm0 MOVQ 0 * SIZE(AO2), %mm1 @@ -303,17 +303,17 @@ addq $4 * SIZE, BO2 ALIGN_4 -.L19: +L(19): decq J - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M - jle .L30 + jle L(30) ALIGN_4 -.L21: +L(21): movq A, AO1 leaq (A, LDA ), AO2 leaq (A, LDA, 2), A @@ -323,10 +323,10 @@ movq N, I sarq $2, I - jle .L23 + jle L(23) ALIGN_4 -.L22: +L(22): RPREFETCH (RPREFETCHSIZE) * SIZE(AO1) MOVQ 0 * SIZE(AO1), %mm0 MOVQ 1 * SIZE(AO1), %mm1 @@ -353,12 +353,12 @@ addq $4 * SIZE, AO2 leaq (BO, M8, 4), BO decq I - jg .L22 + jg L(22) ALIGN_4 -.L23: +L(23): testq $2, N - jle .L24 + jle L(24) MOVQ 0 * SIZE(AO1), %mm0 MOVQ 1 * SIZE(AO1), %mm1 @@ -375,9 +375,9 @@ addq $4 * SIZE, BO1 ALIGN_4 -.L24: +L(24): testq $1, N - jle .L30 + jle L(30) MOVQ 0 * SIZE(AO1), %mm0 MOVQ 0 * SIZE(AO2), %mm1 @@ -388,21 +388,21 @@ addq $2 * SIZE, BO2 ALIGN_4 -.L30: +L(30): testq $1, M - jle .L999 + jle L(999) ALIGN_4 -.L31: +L(31): movq A, AO1 movq B, BO movq N, I sarq $2, I - jle .L33 + jle L(33) ALIGN_4 -.L32: +L(32): MOVQ 0 * SIZE(AO1), %mm0 MOVQ 1 * SIZE(AO1), %mm1 MOVQ 2 * SIZE(AO1), %mm2 @@ -416,12 +416,12 @@ addq $4 * SIZE, AO1 leaq (BO, M8, 4), BO decq I - jg .L32 + jg L(32) ALIGN_4 -.L33: +L(33): testq $2, N - jle .L34 + jle L(34) MOVQ 0 * SIZE(AO1), %mm0 MOVQ 1 * SIZE(AO1), %mm1 @@ -433,9 +433,9 @@ addq $2 * SIZE, BO1 ALIGN_4 -.L34: +L(34): testq $1, N - jle .L999 + jle L(999) MOVQ 0 * SIZE(AO1), %mm0 MOVNTQ %mm0, 0 * SIZE(BO2) @@ -443,7 +443,7 @@ addq $1 * SIZE, BO2 ALIGN_4 -.L999: +L(999): EMMS #ifdef WINDOWS_ABI diff --git a/kernel/x86_64/iamax.S b/kernel/x86_64/iamax.S index 00999e25f2..a8a0669dad 100644 --- a/kernel/x86_64/iamax.S +++ b/kernel/x86_64/iamax.S @@ -70,9 +70,9 @@ xorq RET, RET testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) ffreep %st movq $2, NUM @@ -84,17 +84,17 @@ #endif addq INCX, X decq M - jle .L999 + jle L(999) cmpq $SIZE, INCX - jne .L40 + jne L(40) movq M, I sarq $3, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -190,17 +190,17 @@ addq $8 * SIZE, X decq I - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): movq M, I andq $7, I - jle .L999 + jle L(999) ALIGN_4 -.L21: +L(21): FLD 0 * SIZE(X) #ifdef USE_ABS fabs @@ -214,17 +214,17 @@ addq $1 * SIZE, X incq NUM decq I - jg .L21 - jmp .L999 + jg L(21) + jmp L(999) ALIGN_4 -.L40: +L(40): movq M, I sarq $3, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): FLD 0 * SIZE(X) addq INCX, X #ifdef USE_ABS @@ -322,17 +322,17 @@ incq NUM decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): movq M, I andq $7, I - jle .L999 + jle L(999) ALIGN_4 -.L61: +L(61): FLD 0 * SIZE(X) #ifdef USE_ABS fabs @@ -346,10 +346,10 @@ addq INCX, X decq I - jg .L61 + jg L(61) ALIGN_4 -.L999: +L(999): ffreep %st ret diff --git a/kernel/x86_64/iamax_sse.S b/kernel/x86_64/iamax_sse.S index 6a5da06d52..11251612a9 100644 --- a/kernel/x86_64/iamax_sse.S +++ b/kernel/x86_64/iamax_sse.S @@ -76,10 +76,10 @@ pxor %xmm0, %xmm0 /* Return Value(Float) */ xor RET, RET /* Return Value(Int) */ testq M, M - jle .L999 + jle L(999) leaq (, INCX, SIZE), INCX testq INCX, INCX - jle .L999 + jle L(999) movq M, MM movq X, XX @@ -100,17 +100,17 @@ movups %xmm0, %xmm2 movups %xmm0, %xmm3 /* Generating "seed value" */ cmpq $SIZE, INCX - jne .L80 /* Incx != 1 goto L80 */ + jne L(80) /* Incx != 1 goto L80 */ /* Analigned Check */ testq $3, X /* 00000011 */ - jne .L30 /* Purely Unaligned Mode */ + jne L(30) /* Purely Unaligned Mode */ cmpq $8, M - jle .L30 /* if M <= 8 goto Unaligned mode */ + jle L(30) /* if M <= 8 goto Unaligned mode */ testq $4, X /* bit test 000100 */ - je .L05 + je L(05) movss 0 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -121,9 +121,9 @@ addq $SIZE, X ALIGN_3 -.L05: +L(05): testq $8, X - je .L06 + je L(06) movsd 0 * SIZE(X), %xmm4 unpcklps %xmm4, %xmm4 @@ -135,13 +135,13 @@ addq $2 * SIZE, X ALIGN_3 -.L06: +L(06): movq M, I sarq $4, I - jle .L15 + jle L(15) ALIGN_4 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -172,15 +172,15 @@ addq $16 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_4 -.L15: +L(15): andq $15, M - jle .L20 + jle L(20) testq $8, M - je .L16 + je L(16) movups 0 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -196,9 +196,9 @@ addq $8 * SIZE, X ALIGN_3 -.L16: +L(16): testq $4, M - je .L17 + je L(17) movups 0 * SIZE(X), %xmm6 #ifdef USE_ABS @@ -208,9 +208,9 @@ addq $4 * SIZE, X ALIGN_3 -.L17: +L(17): testq $2, M - je .L18 + je L(18) movsd 0 * SIZE(X), %xmm7 unpcklps %xmm7, %xmm7 @@ -220,9 +220,9 @@ MAXPS %xmm7, %xmm3 addq $2 * SIZE, X -.L18: +L(18): testq $1, M - je .L20 + je L(20) movss 0 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -231,7 +231,7 @@ MAXSS %xmm4, %xmm0 ALIGN_3 -.L20: +L(20): movq XX, X movq MM, M @@ -247,7 +247,7 @@ shufps $0, %xmm0, %xmm0 testq $4, X - je .L21 + je L(21) movss 0 * SIZE(X), %xmm1 @@ -259,12 +259,12 @@ #endif incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) ALIGN_3 -.L21: +L(21): testq $8, X - je .L22 + je L(22) movss 0 * SIZE(X), %xmm1 movss 1 * SIZE(X), %xmm2 @@ -278,19 +278,19 @@ #endif incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm2 - je .L999 + je L(999) ALIGN_3 -.L22: +L(22): movq M, I sarq $3, I - jle .L25 + jle L(25) ALIGN_4 -.L23: +L(23): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -314,16 +314,16 @@ .long 0xd9500f4c #endif testq $15, %r11 - jne .L24 + jne L(24) addq $8 * SIZE, X addq $8, RET decq I - jg .L23 - jmp .L25 + jg L(23) + jmp L(25) ALIGN_3 -.L24: +L(24): movss 0 * SIZE(X), %xmm1 movss 1 * SIZE(X), %xmm2 movss 2 * SIZE(X), %xmm3 @@ -347,32 +347,32 @@ incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm2 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm3 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm4 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm5 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm6 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm7 - je .L999 + je L(999) incq RET - jmp .L999 + jmp L(999) ALIGN_4 -.L25: +L(25): testq $4, M - je .L26 + je L(26) movss 0 * SIZE(X), %xmm1 movss 1 * SIZE(X), %xmm2 @@ -387,21 +387,21 @@ addq $4 * SIZE, X incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm2 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm3 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm4 - je .L999 + je L(999) ALIGN_3 -.L26: +L(26): testq $2, M - je .L27 + je L(27) movss 0 * SIZE(X), %xmm1 movss 1 * SIZE(X), %xmm2 @@ -412,25 +412,25 @@ addq $2 * SIZE, X incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm2 - je .L999 + je L(999) ALIGN_3 -.L27: +L(27): incq RET - jmp .L999 + jmp L(999) ALIGN_3 /* Unaligned Mode */ -.L30: +L(30): movq M, I sarq $4, I - jle .L35 + jle L(35) ALIGN_4 -.L31: +L(31): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -465,15 +465,15 @@ addq $16 * SIZE, X decq I - jg .L31 + jg L(31) ALIGN_4 -.L35: +L(35): andq $15, M - jle .L40 + jle L(40) testq $8, M - je .L36 + je L(36) movsd 0 * SIZE(X), %xmm4 movhps 2 * SIZE(X), %xmm4 @@ -492,9 +492,9 @@ addq $8 * SIZE, X ALIGN_3 -.L36: +L(36): testq $4, M - je .L37 + je L(37) movsd 0 * SIZE(X), %xmm6 movhps 2 * SIZE(X), %xmm6 @@ -505,9 +505,9 @@ addq $4 * SIZE, X ALIGN_3 -.L37: +L(37): testq $2, M - je .L38 + je L(38) movsd 0 * SIZE(X), %xmm7 unpcklps %xmm7, %xmm7 @@ -517,19 +517,19 @@ MAXPS %xmm7, %xmm3 addq $2 * SIZE, X -.L38: +L(38): testq $1, M - je .L40 + je L(40) movss 0 * SIZE(X), %xmm4 #ifdef USE_ABS andps %xmm15, %xmm4 #endif MAXSS %xmm4, %xmm0 - jmp .L40 + jmp L(40) ALIGN_4 -.L40: +L(40): movq XX, X movq MM, M @@ -546,10 +546,10 @@ movq M, I sarq $3, I - jle .L45 + jle L(45) ALIGN_4 -.L43: +L(43): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -575,16 +575,16 @@ .long 0xd9500f4c #endif testq $15, %r11 - jne .L44 + jne L(44) addq $8 * SIZE, X addq $8, RET decq I - jg .L43 - jmp .L45 + jg L(43) + jmp L(45) ALIGN_3 -.L44: +L(44): movss 0 * SIZE(X), %xmm1 movss 1 * SIZE(X), %xmm2 movss 2 * SIZE(X), %xmm3 @@ -608,32 +608,32 @@ incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm2 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm3 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm4 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm5 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm6 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm7 - je .L999 + je L(999) incq RET - jmp .L999 + jmp L(999) ALIGN_4 -.L45: +L(45): testq $4, M - je .L46 + je L(46) movss 0 * SIZE(X), %xmm1 movss 1 * SIZE(X), %xmm2 @@ -648,21 +648,21 @@ addq $4 * SIZE, X incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm2 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm3 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm4 - je .L999 + je L(999) ALIGN_3 -.L46: +L(46): testq $2, M - je .L47 + je L(47) movss 0 * SIZE(X), %xmm1 movss 1 * SIZE(X), %xmm2 @@ -673,24 +673,24 @@ addq $2 * SIZE, X incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm2 - je .L999 + je L(999) ALIGN_3 -.L47: +L(47): incq RET - jmp .L999 + jmp L(999) ALIGN_3 -.L80: +L(80): movq M, I sarq $3, I - jle .L85 + jle L(85) ALIGN_4 -.L81: +L(81): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -752,15 +752,15 @@ MAXSS %xmm7, %xmm3 decq I - jg .L81 + jg L(81) ALIGN_4 -.L85: +L(85): andq $7, M - jle .L90 + jle L(90) testq $4, M - je .L86 + je L(86) movss 0 * SIZE(X), %xmm4 addq INCX, X @@ -791,9 +791,9 @@ MAXSS %xmm7, %xmm3 ALIGN_3 -.L86: +L(86): testq $2, M - je .L87 + je L(87) movss 0 * SIZE(X), %xmm4 addq INCX, X @@ -810,9 +810,9 @@ MAXSS %xmm5, %xmm1 ALIGN_3 -.L87: +L(87): testq $1, M - je .L90 + je L(90) movss 0 * SIZE(X), %xmm6 addq INCX, X @@ -822,7 +822,7 @@ MAXSS %xmm6, %xmm2 ALIGN_4 -.L90: +L(90): movq XX, X movq MM, M @@ -833,10 +833,10 @@ movq M, I sarq $3, I - jle .L95 + jle L(95) ALIGN_4 -.L93: +L(93): LOAD_AND_COMPARE_TO_MXX %xmm1 LOAD_AND_COMPARE_TO_MXX %xmm2 LOAD_AND_COMPARE_TO_MXX %xmm3 @@ -860,15 +860,15 @@ .long 0xd9500f4c #endif testq $15, %r11 - jne .L94 + jne L(94) addq $8, RET decq I - jg .L93 - jmp .L95 + jg L(93) + jmp L(95) ALIGN_3 -.L94: +L(94): subq INCX, X movss 0 * SIZE(X), %xmm8 subq INCX, X @@ -897,32 +897,32 @@ #endif incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm2 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm3 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm4 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm5 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm6 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm7 - je .L999 + je L(999) incq RET - jmp .L999 + jmp L(999) ALIGN_4 -.L95: +L(95): testq $4, M - je .L96 + je L(96) movss 0 * SIZE(X), %xmm1 addq INCX, X @@ -941,21 +941,21 @@ #endif incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm2 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm3 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm4 - je .L999 + je L(999) ALIGN_3 -.L96: +L(96): testq $2, M - je .L97 + je L(97) movss 0 * SIZE(X), %xmm1 addq INCX, X @@ -967,17 +967,17 @@ #endif incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm2 - je .L999 + je L(999) ALIGN_3 -.L97: +L(97): incq RET ALIGN_3 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/iamax_sse2.S b/kernel/x86_64/iamax_sse2.S index 6808f191ba..79ff582b67 100644 --- a/kernel/x86_64/iamax_sse2.S +++ b/kernel/x86_64/iamax_sse2.S @@ -63,10 +63,10 @@ pxor %xmm0, %xmm0 xor RET, RET testq M, M - jle .L999 + jle L(999) leaq (, INCX, SIZE), INCX testq INCX, INCX - jle .L999 + jle L(999) movq M, MM movq X, XX @@ -87,17 +87,17 @@ movapd %xmm0, %xmm2 movapd %xmm0, %xmm3 cmpq $SIZE, INCX - jne .L80 + jne L(80) /* Analigned Check */ cmpq $7, M - jle .L50 + jle L(50) testq $7, X - jne .L50 # Purely Unaligned Mode + jne L(50) # Purely Unaligned Mode testq $15, X # Checking for 128bit align - je .L05 + je L(05) movsd 0 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -109,13 +109,13 @@ addq $SIZE, X ALIGN_3 -.L05: +L(05): movq M, I sarq $4, I - jle .L15 + jle L(15) ALIGN_4 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -174,15 +174,15 @@ addq $16 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_4 -.L15: +L(15): andq $15, M - jle .L20 + jle L(20) testq $8, M - je .L16 + je L(16) movapd 0 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -210,9 +210,9 @@ addq $8 * SIZE, X ALIGN_3 -.L16: +L(16): testq $4, M - je .L17 + je L(17) movapd 0 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -228,9 +228,9 @@ addq $4 * SIZE, X ALIGN_3 -.L17: +L(17): testq $2, M - je .L18 + je L(18) movapd 0 * SIZE(X), %xmm6 #ifdef USE_ABS @@ -239,9 +239,9 @@ maxpd %xmm6, %xmm2 addq $2 * SIZE, X -.L18: +L(18): testq $1, M - je .L20 + je L(20) movsd 0 * SIZE(X), %xmm7 #ifdef USE_ABS @@ -252,7 +252,7 @@ ALIGN_3 /* Finding Index */ -.L20: +L(20): movq XX, X movq MM, M @@ -266,7 +266,7 @@ ALIGN_3 testq $15, X # Checking for 128bit align - je .L21 + je L(21) movsd 0 * SIZE(X), %xmm1 #ifdef USE_ABS @@ -274,18 +274,18 @@ #endif incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) addq $SIZE, X decq M ALIGN_3 -.L21: +L(21): movq M, I sarq $3, I - jle .L25 + jle L(25) ALIGN_4 -.L22: +L(22): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -324,16 +324,16 @@ .long 0xd9500f4c #endif testq $3, %r11 - jne .L23 + jne L(23) addq $8 * SIZE, X addq $8, RET decq I - jg .L22 - jmp .L25 + jg L(22) + jmp L(25) ALIGN_4 -.L23: +L(23): movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 movsd 2 * SIZE(X), %xmm3 @@ -357,32 +357,32 @@ incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm2 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm3 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm4 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm5 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm6 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm7 - je .L999 + je L(999) incq RET - jmp .L999 + jmp L(999) ALIGN_3 -.L25: +L(25): testq $4, M - je .L27 + je L(27) movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 @@ -397,21 +397,21 @@ addq $4 * SIZE, X incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm2 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm3 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm4 - je .L999 + je L(999) ALIGN_3 -.L27: +L(27): testq $2, M - je .L28 + je L(28) movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 @@ -422,25 +422,25 @@ addq $2 * SIZE, X incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm2 - je .L999 + je L(999) ALIGN_3 -.L28: +L(28): incq RET - jmp .L999 + jmp L(999) ALIGN_3 /* Unaligned Mode */ -.L50: +L(50): movq M, I sarq $4, I - jle .L55 + jle L(55) ALIGN_4 -.L51: +L(51): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -507,15 +507,15 @@ addq $16 * SIZE, X decq I - jg .L51 + jg L(51) ALIGN_4 -.L55: +L(55): andq $15, M - jle .L60 + jle L(60) testq $8, M - je .L56 + je L(56) movsd 0 * SIZE(X), %xmm4 movhpd 1 * SIZE(X), %xmm4 @@ -548,9 +548,9 @@ addq $8 * SIZE, X ALIGN_3 -.L56: +L(56): testq $4, M - je .L57 + je L(57) movsd 0 * SIZE(X), %xmm4 movhpd 1 * SIZE(X), %xmm4 @@ -568,9 +568,9 @@ addq $4 * SIZE, X ALIGN_3 -.L57: +L(57): testq $2, M - je .L58 + je L(58) movsd 0 * SIZE(X), %xmm6 movhpd 1 * SIZE(X), %xmm6 @@ -580,9 +580,9 @@ maxpd %xmm6, %xmm2 addq $2 * SIZE, X -.L58: +L(58): testq $1, M - je .L60 + je L(60) movsd 0 * SIZE(X), %xmm7 unpcklpd %xmm7, %xmm7 @@ -592,7 +592,7 @@ maxpd %xmm7, %xmm3 ALIGN_3 -.L60: +L(60): movq XX, X movq MM, M @@ -606,10 +606,10 @@ movq M, I sarq $3, I - jle .L65 + jle L(65) ALIGN_4 -.L62: +L(62): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -652,16 +652,16 @@ .long 0xd9500f4c #endif testq $3, %r11 - jne .L63 + jne L(63) addq $8 * SIZE, X addq $8, RET decq I - jg .L62 - jmp .L65 + jg L(62) + jmp L(65) ALIGN_4 -.L63: +L(63): movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 movsd 2 * SIZE(X), %xmm3 @@ -685,32 +685,32 @@ incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm2 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm3 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm4 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm5 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm6 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm7 - je .L999 + je L(999) incq RET - jmp .L999 + jmp L(999) ALIGN_3 -.L65: +L(65): testq $4, M - je .L67 + je L(67) movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 @@ -725,21 +725,21 @@ addq $4 * SIZE, X incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm2 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm3 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm4 - je .L999 + je L(999) ALIGN_3 -.L67: +L(67): testq $2, M - je .L68 + je L(68) movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 @@ -750,24 +750,24 @@ addq $2 * SIZE, X incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm2 - je .L999 + je L(999) ALIGN_3 -.L68: +L(68): incq RET - jmp .L999 + jmp L(999) ALIGN_4 -.L80: +L(80): movq M, I sarq $4, I - jle .L85 + jle L(85) ALIGN_4 -.L81: +L(81): movsd 0 * SIZE(X), %xmm4 addq INCX, X movhpd 0 * SIZE(X), %xmm4 @@ -841,15 +841,15 @@ maxpd %xmm7, %xmm3 decq I - jg .L81 + jg L(81) ALIGN_4 -.L85: +L(85): andq $15, M - jle .L90 + jle L(90) testq $8, M - je .L86 + je L(86) movsd 0 * SIZE(X), %xmm4 addq INCX, X @@ -888,9 +888,9 @@ maxpd %xmm7, %xmm3 ALIGN_3 -.L86: +L(86): testq $4, M - je .L87 + je L(87) movsd 0 * SIZE(X), %xmm4 addq INCX, X @@ -911,9 +911,9 @@ maxpd %xmm5, %xmm1 ALIGN_3 -.L87: +L(87): testq $2, M - je .L88 + je L(88) movsd 0 * SIZE(X), %xmm6 addq INCX, X @@ -925,9 +925,9 @@ maxpd %xmm6, %xmm2 ALIGN_3 -.L88: +L(88): testq $1, M - je .L90 + je L(90) movsd 0 * SIZE(X), %xmm7 unpcklpd %xmm7, %xmm7 @@ -944,7 +944,7 @@ maxsd %xmm1, %xmm0 ALIGN_4 -.L90: +L(90): movq XX, X movq MM, M @@ -958,10 +958,10 @@ movq M, I sarq $3, I - jle .L95 + jle L(95) ALIGN_4 -.L92: +L(92): movsd 0 * SIZE(X), %xmm1 addq INCX, X movhpd 0 * SIZE(X), %xmm1 @@ -1008,15 +1008,15 @@ .long 0xd9500f4c #endif testq $3, %r11 - jne .L93 + jne L(93) addq $8, RET decq I - jg .L92 - jmp .L95 + jg L(92) + jmp L(95) ALIGN_4 -.L93: +L(93): subq INCX, X movsd 0 * SIZE(X), %xmm8 subq INCX, X @@ -1048,32 +1048,32 @@ incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm2 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm3 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm4 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm5 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm6 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm7 - je .L999 + je L(999) incq RET - jmp .L999 + jmp L(999) ALIGN_3 -.L95: +L(95): testq $4, M - je .L97 + je L(97) movsd 0 * SIZE(X), %xmm1 addq INCX, X @@ -1091,21 +1091,21 @@ #endif incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm2 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm3 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm4 - je .L999 + je L(999) ALIGN_3 -.L97: +L(97): testq $2, M - je .L98 + je L(98) movsd 0 * SIZE(X), %xmm1 addq INCX, X @@ -1117,17 +1117,17 @@ #endif incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm2 - je .L999 + je L(999) ALIGN_3 -.L98: +L(98): incq RET ALIGN_3 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/izamax.S b/kernel/x86_64/izamax.S index b24b2e6925..c76a7e8bb0 100644 --- a/kernel/x86_64/izamax.S +++ b/kernel/x86_64/izamax.S @@ -70,9 +70,9 @@ xorq RET, RET testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) ffreep %st movq $2, NUM @@ -85,17 +85,17 @@ faddp %st, %st(1) addq INCX, X decq M - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX - jne .L40 + jne L(40) movq M, I sarq $2, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -151,16 +151,16 @@ addq $8 * SIZE, X decq I - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): movq M, I andq $3, I - jle .L999 + jle L(999) ALIGN_4 -.L21: +L(21): FLD 0 * SIZE(X) fabs FLD 1 * SIZE(X) @@ -175,17 +175,17 @@ addq $2 * SIZE, X decq I - jg .L21 - jmp .L999 + jg L(21) + jmp L(999) ALIGN_4 -.L40: +L(40): movq M, I sarq $2, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): FLD 0 * SIZE(X) fabs FLD 1 * SIZE(X) @@ -239,17 +239,17 @@ incq NUM decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): movq M, I andq $3, I - jle .L999 + jle L(999) ALIGN_4 -.L61: +L(61): FLD 0 * SIZE(X) fabs FLD 1 * SIZE(X) @@ -264,10 +264,10 @@ addq INCX, X decq I - jg .L61 + jg L(61) ALIGN_4 -.L999: +L(999): ffreep %st ret diff --git a/kernel/x86_64/izamax_sse.S b/kernel/x86_64/izamax_sse.S index e273b8cc6e..84ffee6a55 100644 --- a/kernel/x86_64/izamax_sse.S +++ b/kernel/x86_64/izamax_sse.S @@ -63,9 +63,9 @@ pxor %xmm0, %xmm0 xor RET, RET testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) salq $ZBASE_SHIFT, INCX movq M, MM @@ -84,15 +84,15 @@ shufps $0, %xmm0, %xmm0 movaps %xmm0, %xmm1 cmpq $2 * SIZE, INCX - jne .L70 + jne L(70) -.L30: +L(30): movq M, I sarq $3, I - jle .L35 + jle L(35) ALIGN_4 -.L31: +L(31): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -128,15 +128,15 @@ addq $16 * SIZE, X decq I - jg .L31 + jg L(31) ALIGN_4 -.L35: +L(35): andq $7, M - jle .L40 + jle L(40) testq $4, M - je .L36 + je L(36) movsd 0 * SIZE(X), %xmm4 movhps 2 * SIZE(X), %xmm4 @@ -155,9 +155,9 @@ addq $8 * SIZE, X ALIGN_3 -.L36: +L(36): testq $2, M - je .L37 + je L(37) movss 0 * SIZE(X), %xmm4 movss 1 * SIZE(X), %xmm5 @@ -174,9 +174,9 @@ addq $4 * SIZE, X ALIGN_3 -.L37: +L(37): testq $1, M - je .L40 + je L(40) movss 0 * SIZE(X), %xmm4 movss 1 * SIZE(X), %xmm5 @@ -186,7 +186,7 @@ maxss %xmm4, %xmm0 ALIGN_4 -.L40: +L(40): movq XX, X movq MM, M @@ -201,10 +201,10 @@ movq M, I sarq $2, I - jle .L45 + jle L(45) ALIGN_4 -.L41: +L(41): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -230,16 +230,16 @@ .long 0xd9500f4c #endif testq $15, %r11 - jne .L43 + jne L(43) addq $8 * SIZE, X addq $4, RET decq I - jg .L41 - jmp .L45 + jg L(41) + jmp L(45) ALIGN_4 -.L43: +L(43): movss 0 * SIZE(X), %xmm1 movss 1 * SIZE(X), %xmm2 movss 2 * SIZE(X), %xmm3 @@ -266,21 +266,21 @@ incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm3 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm5 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm7 - je .L999 + je L(999) ALIGN_3 -.L45: +L(45): testq $2, M - je .L47 + je L(47) movss 0 * SIZE(X), %xmm1 movss 1 * SIZE(X), %xmm2 @@ -297,24 +297,24 @@ incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm3 - je .L999 + je L(999) ALIGN_3 -.L47: +L(47): incq RET - jmp .L999 + jmp L(999) ALIGN_3 -.L70: +L(70): movq M, I sarq $3, I - jle .L75 + jle L(75) ALIGN_4 -.L71: +L(71): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -357,15 +357,15 @@ maxps %xmm7, %xmm0 decq I - jg .L71 + jg L(71) ALIGN_4 -.L75: +L(75): andq $7, M - jle .L80 + jle L(80) testq $4, M - je .L76 + je L(76) movsd 0 * SIZE(X), %xmm4 addq INCX, X @@ -386,9 +386,9 @@ maxps %xmm4, %xmm0 ALIGN_3 -.L76: +L(76): testq $2, M - je .L77 + je L(77) movss 0 * SIZE(X), %xmm4 movss 1 * SIZE(X), %xmm5 @@ -406,9 +406,9 @@ maxss %xmm6, %xmm1 ALIGN_3 -.L77: +L(77): testq $1, M - je .L80 + je L(80) movss 0 * SIZE(X), %xmm4 movss 1 * SIZE(X), %xmm5 @@ -418,7 +418,7 @@ maxss %xmm4, %xmm0 ALIGN_4 -.L80: +L(80): movq XX, X movq MM, M @@ -433,10 +433,10 @@ movq M, I sarq $2, I - jle .L85 + jle L(85) ALIGN_4 -.L81: +L(81): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -466,15 +466,15 @@ .long 0xd9500f4c #endif testq $15, %r11 - jne .L83 + jne L(83) addq $4, RET decq I - jg .L81 - jmp .L85 + jg L(81) + jmp L(85) ALIGN_4 -.L83: +L(83): subq INCX, X movss 0 * SIZE(X), %xmm7 movss 1 * SIZE(X), %xmm8 @@ -504,21 +504,21 @@ incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm3 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm5 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm7 - je .L999 + je L(999) ALIGN_3 -.L85: +L(85): testq $2, M - je .L87 + je L(87) movss 0 * SIZE(X), %xmm1 movss 1 * SIZE(X), %xmm2 @@ -536,17 +536,17 @@ incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm3 - je .L999 + je L(999) ALIGN_3 -.L87: +L(87): incq RET ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/izamax_sse2.S b/kernel/x86_64/izamax_sse2.S index c656a652dc..9f8dc3af8f 100644 --- a/kernel/x86_64/izamax_sse2.S +++ b/kernel/x86_64/izamax_sse2.S @@ -63,9 +63,9 @@ pxor %xmm0, %xmm0 xor RET, RET testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) salq $ZBASE_SHIFT, INCX movq M, MM @@ -86,14 +86,14 @@ movapd %xmm0, %xmm2 movapd %xmm0, %xmm3 cmpq $2 * SIZE, INCX - jne .L60 + jne L(60) movq M, I sarq $3, I - jle .L25 + jle L(25) ALIGN_4 -.L21: +L(21): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -142,15 +142,15 @@ addq $16 * SIZE, X decq I - jg .L21 + jg L(21) ALIGN_4 -.L25: +L(25): andq $7, M - jle .L30 + jle L(30) testq $4, M - je .L26 + je L(26) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -174,9 +174,9 @@ addq $8 * SIZE, X ALIGN_3 -.L26: +L(26): testq $2, M - je .L27 + je L(27) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -190,9 +190,9 @@ maxpd %xmm4, %xmm0 ALIGN_3 -.L27: +L(27): testq $1, M - je .L30 + je L(30) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -202,7 +202,7 @@ maxsd %xmm4, %xmm2 ALIGN_4 -.L30: +L(30): movq XX, X movq MM, M @@ -216,10 +216,10 @@ movq M, I sarq $2, I - jle .L35 + jle L(35) ALIGN_4 -.L31: +L(31): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -252,16 +252,16 @@ .long 0xd9500f4c #endif testq $3, %r11 - jne .L33 + jne L(33) addq $8 * SIZE, X addq $4, RET decq I - jg .L31 - jmp .L35 + jg L(31) + jmp L(35) ALIGN_4 -.L33: +L(33): movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 movsd 2 * SIZE(X), %xmm3 @@ -288,21 +288,21 @@ incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm3 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm5 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm7 - je .L999 + je L(999) ALIGN_3 -.L35: +L(35): testq $2, M - je .L36 + je L(36) movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 @@ -319,24 +319,24 @@ addpd %xmm4, %xmm3 incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm3 - je .L999 + je L(999) ALIGN_3 -.L36: +L(36): incq RET - jmp .L999 + jmp L(999) ALIGN_3 -.L60: +L(60): movq M, I sarq $3, I - jle .L65 + jle L(65) ALIGN_4 -.L61: +L(61): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -390,15 +390,15 @@ maxpd %xmm6, %xmm3 decq I - jg .L61 + jg L(61) ALIGN_4 -.L65: +L(65): andq $7, M - jle .L70 + jle L(70) testq $4, M - je .L66 + je L(66) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -423,9 +423,9 @@ maxpd %xmm6, %xmm1 ALIGN_3 -.L66: +L(66): testq $2, M - je .L67 + je L(67) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -439,9 +439,9 @@ maxpd %xmm4, %xmm2 ALIGN_3 -.L67: +L(67): testq $1, M - je .L70 + je L(70) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -451,7 +451,7 @@ maxsd %xmm4, %xmm3 ALIGN_3 -.L70: +L(70): movq XX, X movq MM, M @@ -465,10 +465,10 @@ movq M, I sarq $2, I - jle .L75 + jle L(75) ALIGN_4 -.L71: +L(71): #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) prefetch PREFETCHSIZE * SIZE(X) #endif @@ -509,15 +509,15 @@ .long 0xd9500f4c #endif testq $3, %r11 - jne .L73 + jne L(73) addq $4, RET decq I - jg .L71 - jmp .L75 + jg L(71) + jmp L(75) ALIGN_4 -.L73: +L(73): subq INCX, X movsd 0 * SIZE(X), %xmm7 movsd 1 * SIZE(X), %xmm8 @@ -547,21 +547,21 @@ incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm3 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm5 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm7 - je .L999 + je L(999) ALIGN_3 -.L75: +L(75): testq $2, M - je .L76 + je L(76) movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 @@ -579,17 +579,17 @@ addpd %xmm4, %xmm3 incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm3 - je .L999 + je L(999) ALIGN_3 -.L76: +L(76): incq RET ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/nrm2.S b/kernel/x86_64/nrm2.S index 61cf8c4529..165aa36215 100644 --- a/kernel/x86_64/nrm2.S +++ b/kernel/x86_64/nrm2.S @@ -56,9 +56,9 @@ fldz testq M, M - jle .L999 + jle L(999) testq INCX, INCX - je .L999 + je L(999) salq $BASE_SHIFT, INCX @@ -66,14 +66,14 @@ fldz fldz cmpq $SIZE, INCX - jne .L40 + jne L(40) movq M, I sarq $3, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -109,32 +109,32 @@ faddp %st, %st(1) decq I - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): andq $7, M - jle .L998 + jle L(998) ALIGN_4 -.L21: +L(21): FLD (X) fmul %st(0), %st faddp %st,%st(1) addq $1 * SIZE, X decq M - jg .L21 - jmp .L998 + jg L(21) + jmp L(998) ALIGN_4 -.L40: +L(40): movq M, I sarq $3, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): FLD (X) addq INCX, X fmul %st(0), %st @@ -172,31 +172,31 @@ faddp %st, %st(1) decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): andq $7, M - jle .L998 + jle L(998) ALIGN_4 -.L61: +L(61): FLD (X) addq INCX, X fmul %st(0), %st faddp %st,%st(1) decq M - jg .L61 + jg L(61) ALIGN_4 -.L998: +L(998): faddp %st,%st(2) faddp %st,%st(1) faddp %st,%st(1) ALIGN_4 -.L999: +L(999): fsqrt #ifndef XDOUBLE sub $2 * SIZE, %rsp diff --git a/kernel/x86_64/nrm2_sse.S b/kernel/x86_64/nrm2_sse.S index c1f3a45fc5..42334b51c8 100644 --- a/kernel/x86_64/nrm2_sse.S +++ b/kernel/x86_64/nrm2_sse.S @@ -54,19 +54,19 @@ pxor %xmm0, %xmm0 testq M, M - jle .L999 + jle L(999) pxor %xmm1, %xmm1 testq INCX, INCX - je .L999 + je L(999) pxor %xmm2, %xmm2 leaq (, INCX, SIZE), INCX pxor %xmm3, %xmm3 cmpq $SIZE, INCX - jne .L40 + jne L(40) testq $SIZE, X - je .L05 + je L(05) movss 0 * SIZE(X), %xmm4 cvtss2sd %xmm4, %xmm6 @@ -74,13 +74,13 @@ addsd %xmm6, %xmm3 addq INCX, X decq M - jle .L998 + jle L(998) ALIGN_3 -.L05: +L(05): movq M, I sarq $3, I - jle .L14 + jle L(14) movsd 0 * SIZE(X), %xmm4 movsd 2 * SIZE(X), %xmm5 @@ -88,10 +88,10 @@ movsd 6 * SIZE(X), %xmm7 addq $8 * SIZE, X decq I - jle .L12 + jle L(12) ALIGN_3 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -118,10 +118,10 @@ addq $8 * SIZE, X decq I - jg .L10 + jg L(10) ALIGN_3 -.L12: +L(12): cvtps2pd %xmm4, %xmm8 cvtps2pd %xmm5, %xmm9 cvtps2pd %xmm6, %xmm10 @@ -139,9 +139,9 @@ ALIGN_3 -.L14: +L(14): testq $4, M - je .L15 + je L(15) movsd 0 * SIZE(X), %xmm4 movsd 2 * SIZE(X), %xmm5 @@ -154,9 +154,9 @@ addq $4 * SIZE, X ALIGN_3 -.L15: +L(15): testq $2, M - je .L16 + je L(16) movsd 0 * SIZE(X), %xmm4 cvtps2pd %xmm4, %xmm6 @@ -165,24 +165,24 @@ addq $2 * SIZE, X ALIGN_3 -.L16: +L(16): testq $1, M - je .L998 + je L(998) movss 0 * SIZE(X), %xmm4 cvtss2sd %xmm4, %xmm6 mulsd %xmm6, %xmm6 addsd %xmm6, %xmm3 - jmp .L998 + jmp L(998) ALIGN_4 -.L40: +L(40): movq M, I sarq $3, I - jle .L44 + jle L(44) ALIGN_4 -.L41: +L(41): movss (X), %xmm4 addq INCX, X movss (X), %xmm5 @@ -230,12 +230,12 @@ addsd %xmm11, %xmm3 decq I - jg .L41 + jg L(41) ALIGN_3 -.L44: +L(44): testq $4, M - je .L45 + je L(45) movss (X), %xmm4 addq INCX, X @@ -262,9 +262,9 @@ addsd %xmm11, %xmm3 ALIGN_3 -.L45: +L(45): testq $2, M - je .L46 + je L(46) movss (X), %xmm4 addq INCX, X @@ -279,9 +279,9 @@ addsd %xmm7, %xmm2 ALIGN_3 -.L46: +L(46): testq $1, M - je .L998 + je L(998) movss (X), %xmm4 cvtss2sd %xmm4, %xmm6 @@ -289,7 +289,7 @@ addsd %xmm6, %xmm3 ALIGN_4 -.L998: +L(998): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -303,7 +303,7 @@ #endif ALIGN_4 -.L999: +L(999): sqrtsd %xmm0, %xmm0 cvtsd2ss %xmm0, %xmm0 diff --git a/kernel/x86_64/qdot.S b/kernel/x86_64/qdot.S index 2243b6b6d8..59e46a17c0 100644 --- a/kernel/x86_64/qdot.S +++ b/kernel/x86_64/qdot.S @@ -83,16 +83,16 @@ fldz cmpl $SIZE, INCX - jne .L14 + jne L(14) cmpl $SIZE, INCY - jne .L14 + jne L(14) movl N, %eax sarl $2, %eax - jle .L15 + jle L(15) ALIGN_3 -.L16: +L(16): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -121,16 +121,16 @@ addl $4 * SIZE, X addl $4 * SIZE, Y decl %eax - jg .L16 + jg L(16) ALIGN_3 -.L15: +L(15): movl N, %eax andl $3, %eax - jle .L27 + jle L(27) ALIGN_3 -.L22: +L(22): FLD (X) addl $SIZE, X FLD (Y) @@ -138,18 +138,18 @@ addl $SIZE, Y faddp %st,%st(1) decl %eax - jg .L22 + jg L(22) - jmp .L27 + jmp L(27) ALIGN_3 -.L14: +L(14): movl N, %eax sarl $2, %eax - jle .L30 + jle L(30) ALIGN_3 -.L31: +L(31): FLD (X) addl INCX, X FLD (Y) @@ -179,16 +179,16 @@ faddp %st,%st(4) decl %eax - jg .L31 + jg L(31) ALIGN_3 -.L30: +L(30): movl N, %eax andl $3, %eax - jle .L27 + jle L(27) ALIGN_3 -.L37: +L(37): FLD (X) addl INCX, X FLD (Y) @@ -196,10 +196,10 @@ addl INCY, Y faddp %st, %st(1) decl %eax - jg .L37 + jg L(37) ALIGN_3 -.L27: +L(27): faddp %st,%st(2) faddp %st,%st(2) faddp %st,%st(1) diff --git a/kernel/x86_64/qgemm_kernel_2x2.S b/kernel/x86_64/qgemm_kernel_2x2.S index c11f3a91d6..340e4fe352 100644 --- a/kernel/x86_64/qgemm_kernel_2x2.S +++ b/kernel/x86_64/qgemm_kernel_2x2.S @@ -102,10 +102,10 @@ movq N, %rax sarq $1, %rax movq %rax, J - je .L30 + je L(30) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -119,10 +119,10 @@ movq M, I sarq $1, I - je .L20 + je L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -163,10 +163,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -247,20 +247,20 @@ addq $8 * SIZE,BO decq %rax - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif and $3, %rax - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -283,10 +283,10 @@ addq $2 * SIZE,BO decq %rax - jne .L16 + jne L(16) ALIGN_4 -.L18: +L(18): #ifndef TRMMKERNEL FLD ALPHA @@ -332,16 +332,16 @@ addq $2 * SIZE, CO decq I - jne .L11 + jne L(11) ALIGN_4 -.L20: +L(20): movq M, %rax andq $1, %rax - je .L29 + je L(29) ALIGN_4 -.L21: +L(21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -372,10 +372,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -422,20 +422,20 @@ addq $8 * SIZE,BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif and $3, %rax - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -450,10 +450,10 @@ addq $2 * SIZE,BO decq %rax - jne .L26 + jne L(26) ALIGN_4 -.L28: +L(28): #ifndef TRMMKERNEL FLD ALPHA @@ -488,20 +488,20 @@ addq $1 * SIZE, CO ALIGN_4 -.L29: +L(29): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif movq BO, B decq J - jne .L01 + jne L(01) ALIGN_4 -.L30: +L(30): movq N, %rax testq $1, %rax - je .L999 + je L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -515,10 +515,10 @@ movq M, I sarq $1, I - je .L40 + je L(40) ALIGN_4 -.L31: +L(31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -555,10 +555,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(BO) @@ -601,20 +601,20 @@ addq $4 * SIZE,BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif and $3, %rax - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): FLD -8 * SIZE(BO) FLD -8 * SIZE(AO) @@ -629,10 +629,10 @@ addq $1 * SIZE,BO decq %rax - jne .L36 + jne L(36) ALIGN_4 -.L38: +L(38): #ifndef TRMMKERNEL FLD ALPHA @@ -666,16 +666,16 @@ addq $2 * SIZE, CO decq I - jne .L31 + jne L(31) ALIGN_4 -.L40: +L(40): movq M, %rax andq $1, %rax - je .L49 + je L(49) ALIGN_4 -.L41: +L(41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -705,10 +705,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -735,20 +735,20 @@ addq $4 * SIZE,BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif and $3, %rax - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -759,10 +759,10 @@ addq $1 * SIZE,BO decq %rax - jne .L46 + jne L(46) ALIGN_4 -.L48: +L(48): #ifndef TRMMKERNEL FLD ALPHA @@ -791,7 +791,7 @@ addq $1 * SIZE, CO ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addq $1, KK #endif @@ -799,7 +799,7 @@ movq BO, B ALIGN_4 -.L999: +L(999): EMMS movq 0(%rsp), %rbx diff --git a/kernel/x86_64/qgemv_n.S b/kernel/x86_64/qgemv_n.S index c9d345cb18..7e24b931c3 100644 --- a/kernel/x86_64/qgemv_n.S +++ b/kernel/x86_64/qgemv_n.S @@ -100,9 +100,9 @@ movq $0, IS test M, M - jle .L79 # goto END + jle L(79) # goto END test N, N - jle .L79 # goto END + jle L(79) # goto END movq LDA, %rax imulq $P, %rax # P * lda @@ -113,7 +113,7 @@ salq $BASE_SHIFT, LDA ALIGN_2 -.L32: +L(32): movq $P, %rax movq N, MIN_N subq IS, MIN_N @@ -125,17 +125,17 @@ leaq (X,XP, 1), XP cmpq $SIZE, INCX - je .L34 # if incx == 1 goto L34 + je L(34) # if incx == 1 goto L34 movq BUFFER, XP movq XP, X1 movq MIN_N, I sarq $2,I - jle .L35 + jle L(35) ALIGN_2 -.L36: +L(36): FLD (X) addq INCX,X FLD (X) @@ -152,33 +152,33 @@ addq $4 * SIZE, X1 decq I - jg .L36 + jg L(36) ALIGN_3 -.L35: +L(35): movq MIN_N, I andq $3, I - jle .L34 + jle L(34) ALIGN_2 -.L42: +L(42): FLD (X) addq INCX, X FST (X1) addq $SIZE, X1 decq I - jg .L42 + jg L(42) ALIGN_3 /* Main Routine */ -.L34: +L(34): movq Y, Y1 movq M, J sarq $2, J - jle .L47 + jle L(47) ALIGN_2 -.L48: +L(48): movq A, A1 # a_offset = a fldz addq $4 * SIZE, A # a += 4 @@ -189,10 +189,10 @@ fldz FLD (X1) # bt1 = b_offset sarq $1, I - jle .L51 + jle L(51) ALIGN_2 -.L80: +L(80): FLD 0 * SIZE(A1) # at1 = *(a_offset + 0) fmul %st(1), %st # at1 *= bt1 faddp %st, %st(2) # ct1 += at1 @@ -233,12 +233,12 @@ addq $2 * SIZE, X1 # b_offset += 2 decq I - jg .L80 + jg L(80) -.L51: +L(51): movq MIN_N, I andq $1, I - je .L57 + je L(57) FLD 0 * SIZE(A1) # at1 = *(a_offset + 0) fmul %st(1), %st # at1 *= bt1 @@ -258,7 +258,7 @@ fldz ALIGN_2 -.L57: +L(57): ffreep %st(0) fxch %st(4) @@ -289,16 +289,16 @@ addq INCY, Y1 decq J # j -- - jg .L48 + jg L(48) ALIGN_3 -.L47: +L(47): movq M, J andq $3, J # j = (m & 3) - jle .L60 + jle L(60) ALIGN_2 -.L61: +L(61): movq A, A1 # a_offset = a fldz addq $SIZE, A # a++ @@ -308,10 +308,10 @@ fldz movq MIN_N, I sarq $3, I - jle .L64 + jle L(64) ALIGN_2 -.L65: +L(65): FLD 0 * SIZE(X1) FLD (A1) fmulp %st, %st(1) @@ -362,15 +362,15 @@ addq $8 * SIZE, X1 decq I - jg .L65 + jg L(65) -.L64: +L(64): movq MIN_N,I andq $7, I - jle .L70 + jle L(70) ALIGN_2 -.L71: +L(71): FLD (X1) addq $SIZE, X1 FLD (A1) @@ -378,10 +378,10 @@ addq LDA, A1 # a_offset += lda faddp %st, %st(1) decq I - jg .L71 + jg L(71) ALIGN_2 -.L70: +L(70): faddp %st, %st(1) faddp %st, %st(1) faddp %st, %st(1) @@ -392,15 +392,15 @@ FST (Y1) addq INCY, Y1 decq J - jg .L61 + jg L(61) -.L60: +L(60): addq PLDA_M, A addq $P, IS cmpq N, IS - jl .L32 + jl L(32) -.L79: +L(79): EMMS movq 0(%rsp), %rbx diff --git a/kernel/x86_64/qgemv_t.S b/kernel/x86_64/qgemv_t.S index 32372ff15d..969f8163bf 100644 --- a/kernel/x86_64/qgemv_t.S +++ b/kernel/x86_64/qgemv_t.S @@ -99,9 +99,9 @@ movq $0, IS test M, M - jle .L79 # goto END + jle L(79) # goto END test N, N - jle .L79 # goto END + jle L(79) # goto END movq N, %rax imulq LDA, %rax @@ -112,7 +112,7 @@ salq $BASE_SHIFT, LDA ALIGN_2 -.L32: +L(32): movq $P, %rax movq M, MIN_M subq IS , MIN_M @@ -126,17 +126,17 @@ movq X1, XP cmpq $SIZE, INCX - je .L34 + je L(34) movq BUFFER, X1 movq X1, XP movq MIN_M, I sarq $2, I - jle .L35 + jle L(35) ALIGN_3 -.L36: +L(36): FLD (X) addq INCX, X FST 0 * SIZE(X1) @@ -155,35 +155,35 @@ addq $4 * SIZE, X1 decq I - jg .L36 + jg L(36) ALIGN_3 -.L35: +L(35): movq MIN_M, I andq $3,I - jle .L34 + jle L(34) ALIGN_2 -.L42: +L(42): FLD (X) addq INCX, X FST (X1) addq $SIZE, X1 decq I - jg .L42 + jg L(42) ALIGN_3 /* Main Routine */ -.L34: +L(34): movq Y, Y1 # coffset = y movq N, J sarq $2, J - jle .L47 + jle L(47) ALIGN_3 -.L48: +L(48): movq A, A1 leaq (A, LDA), A2 leaq (A, LDA, 4), A @@ -198,10 +198,10 @@ movq MIN_M, I sarq $2,I - jle .L51 + jle L(51) ALIGN_3 -.L80: +L(80): FLD 0 * SIZE(A1) # at = *(a_offset + 0 * lda) fmul %st(1),%st # at1 *= bt1 @@ -278,16 +278,16 @@ addq $4 * SIZE, X1 decq I - jg .L80 + jg L(80) ALIGN_3 -.L51: +L(51): movq MIN_M, I andq $3, I - je .L81 + je L(81) ALIGN_3 -.L52: +L(52): FLD (A1) # at = *(a_offset + 0 * lda) fmul %st(1),%st # at1 *= bt1 faddp %st,%st(2) # ct1 += at1 @@ -309,10 +309,10 @@ addq $SIZE, A2 addq $SIZE, X1 decq I - jg .L52 + jg L(52) ALIGN_3 -.L81: +L(81): ffreep %st(0) fxch %st(4) @@ -343,16 +343,16 @@ addq INCY, Y1 decq J - jg .L48 + jg L(48) ALIGN_3 -.L47: +L(47): movq N, J andq $3, J - jle .L60 + jle L(60) ALIGN_2 -.L61: +L(61): movq A, A1 # a_offset = a fldz # ct1 = ZERO fldz # ct1 = ZERO @@ -365,10 +365,10 @@ movq MIN_M, I sarq $3,I - jle .L64 + jle L(64) ALIGN_3 -.L65: +L(65): FLD 0 * SIZE(X1) FLD 0 * SIZE(A1) fmulp %st, %st(1) @@ -413,16 +413,16 @@ addq $8 * SIZE, A1 decq I - jg .L65 + jg L(65) ALIGN_3 -.L64: +L(64): movq MIN_M, I andq $7, I - jle .L70 + jle L(70) ALIGN_3 -.L71: +L(71): FLD (X1) FLD (A1) fmulp %st, %st(1) @@ -431,10 +431,10 @@ addq $SIZE, X1 addq $SIZE, A1 decq I - jg .L71 + jg L(71) ALIGN_3 -.L70: +L(70): faddp %st, %st(1) faddp %st, %st(1) faddp %st, %st(1) @@ -445,18 +445,18 @@ FST (Y1) addq INCY, Y1 decq J - jg .L61 + jg L(61) ALIGN_3 -.L60: +L(60): addq NLDA, A addq $P, IS cmpq M, IS - jl .L32 + jl L(32) ALIGN_3 -.L79: +L(79): EMMS movq 0(%rsp), %rbx diff --git a/kernel/x86_64/qtrsm_kernel_LN_2x2.S b/kernel/x86_64/qtrsm_kernel_LN_2x2.S index 0a545faf87..591f3ad457 100644 --- a/kernel/x86_64/qtrsm_kernel_LN_2x2.S +++ b/kernel/x86_64/qtrsm_kernel_LN_2x2.S @@ -133,10 +133,10 @@ movq N, %rax sarq $1, %rax movq %rax, J - je .L30 + je L(30) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -173,10 +173,10 @@ movq M, %rax andq $1, %rax - je .L20 + je L(20) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -203,10 +203,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -253,10 +253,10 @@ addq $8 * SIZE,BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -264,10 +264,10 @@ subq KK, %rax #endif and $3, %rax - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -282,10 +282,10 @@ addq $2 * SIZE,BO decq %rax - jne .L26 + jne L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -393,13 +393,13 @@ #endif ALIGN_4 -.L20: +L(20): movq M, I sarq $1, I - je .L29 + je L(29) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -436,10 +436,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -520,10 +520,10 @@ addq $8 * SIZE,BO decq %rax - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -531,10 +531,10 @@ subq KK, %rax #endif and $3, %rax - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -557,10 +557,10 @@ addq $2 * SIZE,BO decq %rax - jne .L16 + jne L(16) ALIGN_4 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -735,10 +735,10 @@ #endif decq I - jne .L11 + jne L(11) ALIGN_4 -.L29: +L(29): #ifdef LN movq K, %rax salq $BASE_SHIFT, %rax @@ -758,13 +758,13 @@ #endif decq J - jne .L01 + jne L(01) ALIGN_4 -.L30: +L(30): movq N, %rax testq $1, %rax - je .L999 + je L(999) #if defined(LT) || defined(RN) movq A, AO @@ -800,10 +800,10 @@ movq M, %rax andq $1, %rax - je .L40 + je L(40) ALIGN_4 -.L41: +L(41): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -829,10 +829,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -859,10 +859,10 @@ addq $4 * SIZE,BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -870,10 +870,10 @@ subq KK, %rax #endif and $3, %rax - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -884,10 +884,10 @@ addq $1 * SIZE,BO decq %rax - jne .L46 + jne L(46) ALIGN_4 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -972,13 +972,13 @@ #endif ALIGN_4 -.L40: +L(40): movq M, I sarq $1, I - je .L49 + je L(49) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -1011,10 +1011,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(BO) @@ -1057,10 +1057,10 @@ addq $4 * SIZE,BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1068,10 +1068,10 @@ subq KK, %rax #endif and $3, %rax - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): FLD -8 * SIZE(BO) FLD -8 * SIZE(AO) @@ -1086,10 +1086,10 @@ addq $1 * SIZE,BO decq %rax - jne .L36 + jne L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1202,10 +1202,10 @@ #endif decq I - jne .L31 + jne L(31) ALIGN_4 -.L49: +L(49): #ifdef LN movq K, %rax salq $BASE_SHIFT, %rax @@ -1225,7 +1225,7 @@ #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/qtrsm_kernel_LT_2x2.S b/kernel/x86_64/qtrsm_kernel_LT_2x2.S index 16063fbcdd..ab62f709c0 100644 --- a/kernel/x86_64/qtrsm_kernel_LT_2x2.S +++ b/kernel/x86_64/qtrsm_kernel_LT_2x2.S @@ -133,10 +133,10 @@ movq N, %rax sarq $1, %rax movq %rax, J - je .L30 + je L(30) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -173,10 +173,10 @@ movq M, I sarq $1, I - je .L20 + je L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -213,10 +213,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -297,10 +297,10 @@ addq $8 * SIZE,BO decq %rax - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -308,10 +308,10 @@ subq KK, %rax #endif and $3, %rax - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -334,10 +334,10 @@ addq $2 * SIZE,BO decq %rax - jne .L16 + jne L(16) ALIGN_4 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -512,16 +512,16 @@ #endif decq I - jne .L11 + jne L(11) ALIGN_4 -.L20: +L(20): movq M, %rax andq $1, %rax - je .L29 + je L(29) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -548,10 +548,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -598,10 +598,10 @@ addq $8 * SIZE,BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -609,10 +609,10 @@ subq KK, %rax #endif and $3, %rax - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -627,10 +627,10 @@ addq $2 * SIZE,BO decq %rax - jne .L26 + jne L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -738,7 +738,7 @@ #endif ALIGN_4 -.L29: +L(29): #ifdef LN movq K, %rax salq $BASE_SHIFT, %rax @@ -758,13 +758,13 @@ #endif decq J - jne .L01 + jne L(01) ALIGN_4 -.L30: +L(30): movq N, %rax testq $1, %rax - je .L999 + je L(999) #if defined(LT) || defined(RN) movq A, AO @@ -800,10 +800,10 @@ movq M, I sarq $1, I - je .L40 + je L(40) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -836,10 +836,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(BO) @@ -882,10 +882,10 @@ addq $4 * SIZE,BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -893,10 +893,10 @@ subq KK, %rax #endif and $3, %rax - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): FLD -8 * SIZE(BO) FLD -8 * SIZE(AO) @@ -911,10 +911,10 @@ addq $1 * SIZE,BO decq %rax - jne .L36 + jne L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1027,16 +1027,16 @@ #endif decq I - jne .L31 + jne L(31) ALIGN_4 -.L40: +L(40): movq M, %rax andq $1, %rax - je .L49 + je L(49) ALIGN_4 -.L41: +L(41): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1062,10 +1062,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -1092,10 +1092,10 @@ addq $4 * SIZE,BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1103,10 +1103,10 @@ subq KK, %rax #endif and $3, %rax - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -1117,10 +1117,10 @@ addq $1 * SIZE,BO decq %rax - jne .L46 + jne L(46) ALIGN_4 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1205,7 +1205,7 @@ #endif ALIGN_4 -.L49: +L(49): #ifdef LN movq K, %rax salq $BASE_SHIFT, %rax @@ -1225,7 +1225,7 @@ #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/qtrsm_kernel_RT_2x2.S b/kernel/x86_64/qtrsm_kernel_RT_2x2.S index 4c94ac02cf..571425f25f 100644 --- a/kernel/x86_64/qtrsm_kernel_RT_2x2.S +++ b/kernel/x86_64/qtrsm_kernel_RT_2x2.S @@ -132,7 +132,7 @@ movq N, %rax testq $1, %rax - je .L30 + je L(30) #if defined(LT) || defined(RN) movq A, AO @@ -168,10 +168,10 @@ movq M, I sarq $1, I - je .L40 + je L(40) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -204,10 +204,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(BO) @@ -250,10 +250,10 @@ addq $4 * SIZE,BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -261,10 +261,10 @@ subq KK, %rax #endif and $3, %rax - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): FLD -8 * SIZE(BO) FLD -8 * SIZE(AO) @@ -279,10 +279,10 @@ addq $1 * SIZE,BO decq %rax - jne .L36 + jne L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -395,16 +395,16 @@ #endif decq I - jne .L31 + jne L(31) ALIGN_4 -.L40: +L(40): movq M, %rax andq $1, %rax - je .L49 + je L(49) ALIGN_4 -.L41: +L(41): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -430,10 +430,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -460,10 +460,10 @@ addq $4 * SIZE,BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -471,10 +471,10 @@ subq KK, %rax #endif and $3, %rax - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -485,10 +485,10 @@ addq $1 * SIZE,BO decq %rax - jne .L46 + jne L(46) ALIGN_4 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -573,7 +573,7 @@ #endif ALIGN_4 -.L49: +L(49): #ifdef LN movq K, %rax salq $BASE_SHIFT, %rax @@ -593,14 +593,14 @@ #endif ALIGN_4 -.L30: +L(30): movq N, %rax sarq $1, %rax movq %rax, J - je .L999 + je L(999) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -637,10 +637,10 @@ movq M, I sarq $1, I - je .L20 + je L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -677,10 +677,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -761,10 +761,10 @@ addq $8 * SIZE,BO decq %rax - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -772,10 +772,10 @@ subq KK, %rax #endif and $3, %rax - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -798,10 +798,10 @@ addq $2 * SIZE,BO decq %rax - jne .L16 + jne L(16) ALIGN_4 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -976,16 +976,16 @@ #endif decq I - jne .L11 + jne L(11) ALIGN_4 -.L20: +L(20): movq M, %rax andq $1, %rax - je .L29 + je L(29) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1012,10 +1012,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -1062,10 +1062,10 @@ addq $8 * SIZE,BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1073,10 +1073,10 @@ subq KK, %rax #endif and $3, %rax - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -1091,10 +1091,10 @@ addq $2 * SIZE,BO decq %rax - jne .L26 + jne L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1202,7 +1202,7 @@ #endif ALIGN_4 -.L29: +L(29): #ifdef LN movq K, %rax salq $BASE_SHIFT, %rax @@ -1222,10 +1222,10 @@ #endif decq J - jne .L01 + jne L(01) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/rot.S b/kernel/x86_64/rot.S index 6b2ad7fd87..61167fa998 100644 --- a/kernel/x86_64/rot.S +++ b/kernel/x86_64/rot.S @@ -69,19 +69,19 @@ salq $BASE_SHIFT, INCY testq N, N - jle .L999 + jle L(999) cmpq $SIZE, INCX - jne .L50 + jne L(50) cmpq $SIZE, INCY - jne .L50 + jne L(50) movq N, I sarq $2, I - jle .L15 + jle L(15) ALIGN_4 -.L10: +L(10): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -170,16 +170,16 @@ addq $4 * SIZE, Y decq I - jg .L10 + jg L(10) ALIGN_4 -.L15: +L(15): movq N, I andq $3, I - jle .L999 + jle L(999) ALIGN_4 -.L16: +L(16): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -203,17 +203,17 @@ addq $SIZE, Y decq I - jg .L16 - jmp .L999 + jg L(16) + jmp L(999) ALIGN_4 -.L50: +L(50): movq N, I sarq $2, I - jle .L55 + jle L(55) ALIGN_4 -.L51: +L(51): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -303,16 +303,16 @@ addq INCY, Y decq I - jg .L51 + jg L(51) ALIGN_4 -.L55: +L(55): movq N, I andq $3, I - jle .L999 + jle L(999) ALIGN_4 -.L56: +L(56): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -336,11 +336,11 @@ addq INCY, Y decq I - jg .L56 + jg L(56) ALIGN_4 -.L999: +L(999): ffreep %st ffreep %st ret diff --git a/kernel/x86_64/rot_sse.S b/kernel/x86_64/rot_sse.S index 6e37292ed9..4d82fde56a 100644 --- a/kernel/x86_64/rot_sse.S +++ b/kernel/x86_64/rot_sse.S @@ -72,15 +72,15 @@ pshufd $0x0, %xmm1, S cmpq $0, N - jle .L999 + jle L(999) cmpq $SIZE, INCX - jne .L50 + jne L(50) cmpq $SIZE, INCY - jne .L50 + jne L(50) testq $SIZE, X - je .L05 + je L(05) movss 0 * SIZE(Y), %xmm1 movss 0 * SIZE(X), %xmm0 @@ -103,14 +103,14 @@ addq $1 * SIZE, X addq $1 * SIZE, Y decq N - jle .L999 + jle L(999) -.L05: +L(05): testq $2 * SIZE, X - je .L10 + je L(10) cmpq $1, N - je .L17 + je L(17) movsd 0 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -133,16 +133,16 @@ addq $2 * SIZE, X addq $2 * SIZE, Y subq $2, N - jle .L999 + jle L(999) ALIGN_2 -.L10: +L(10): testq $3 * SIZE, Y - jne .L20 + jne L(20) movq N, %rax sarq $5, %rax - jle .L14 + jle L(14) movaps 0 * SIZE(Y), %xmm1 movaps 4 * SIZE(Y), %xmm3 @@ -155,10 +155,10 @@ movaps 12 * SIZE(X), %xmm10 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -291,10 +291,10 @@ addq $32 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movaps %xmm1, %xmm4 mulps S, %xmm1 movaps %xmm3, %xmm6 @@ -404,12 +404,12 @@ addq $32 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $31, N - jle .L999 + jle L(999) testq $16, N - jle .L15 + jle L(15) movaps 0 * SIZE(Y), %xmm1 movaps 0 * SIZE(X), %xmm0 @@ -475,9 +475,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $8, N - jle .L16 + jle L(16) movaps 0 * SIZE(Y), %xmm1 movaps 0 * SIZE(X), %xmm0 @@ -513,9 +513,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $4, N - jle .L17 + jle L(17) movaps 0 * SIZE(Y), %xmm1 movaps 0 * SIZE(X), %xmm0 @@ -539,9 +539,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $2, N - jle .L18 + jle L(18) movsd 0 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -565,9 +565,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L18: +L(18): testq $1, N - jle .L999 + jle L(999) movss 0 * SIZE(Y), %xmm1 movss 0 * SIZE(X), %xmm0 @@ -586,16 +586,16 @@ movss %xmm0, 0 * SIZE(X) movss %xmm2, 0 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L20: +L(20): movq N, %rax sarq $5, %rax - jle .L24 + jle L(24) ALIGN_3 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -751,15 +751,15 @@ addq $32 * SIZE, X addq $32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L24: +L(24): testq $31, N - jle .L999 + jle L(999) testq $16, N - jle .L25 + jle L(25) movsd 0 * SIZE(Y), %xmm1 movhps 2 * SIZE(Y), %xmm1 @@ -833,9 +833,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $8, N - jle .L26 + jle L(26) movsd 0 * SIZE(Y), %xmm1 movhps 2 * SIZE(Y), %xmm1 @@ -876,9 +876,9 @@ ALIGN_3 -.L26: +L(26): testq $4, N - jle .L27 + jle L(27) movsd 0 * SIZE(Y), %xmm1 movhps 2 * SIZE(Y), %xmm1 @@ -904,9 +904,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $2, N - jle .L28 + jle L(28) movsd 0 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -930,9 +930,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L28: +L(28): testq $1, N - jle .L999 + jle L(999) movss 0 * SIZE(Y), %xmm1 movss 0 * SIZE(X), %xmm0 @@ -951,17 +951,17 @@ movss %xmm0, 0 * SIZE(X) movss %xmm2, 0 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L50: +L(50): movq N, %rax sarq $2, %rax - jle .L55 + jle L(55) ALIGN_3 -.L53: +L(53): movss (Y), %xmm1 movss (X), %xmm0 @@ -1047,16 +1047,16 @@ addq INCY, Y decq %rax - jg .L53 + jg L(53) ALIGN_3 -.L55: +L(55): movq N, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_3 -.L56: +L(56): movss (Y), %xmm1 movss (X), %xmm0 @@ -1079,10 +1079,10 @@ addq INCY, Y decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/rot_sse2.S b/kernel/x86_64/rot_sse2.S index aa5852cac1..e41eebfc74 100644 --- a/kernel/x86_64/rot_sse2.S +++ b/kernel/x86_64/rot_sse2.S @@ -72,15 +72,15 @@ pshufd $0x44, %xmm1, S cmpq $0, N - jle .L999 + jle L(999) cmpq $SIZE, INCX - jne .L50 + jne L(50) cmpq $SIZE, INCY - jne .L50 + jne L(50) testq $SIZE, X - je .L10 + je L(10) movsd 0 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -103,16 +103,16 @@ addq $1 * SIZE, X addq $1 * SIZE, Y decq N - jle .L999 + jle L(999) ALIGN_2 -.L10: +L(10): testq $SIZE, Y - jne .L20 + jne L(20) movq N, %rax sarq $4, %rax - jle .L14 + jle L(14) movaps 0 * SIZE(Y), %xmm1 movaps 2 * SIZE(Y), %xmm3 @@ -125,10 +125,10 @@ movaps 6 * SIZE(X), %xmm10 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -261,10 +261,10 @@ addq $16 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movaps %xmm1, %xmm4 mulpd S, %xmm1 movaps %xmm3, %xmm6 @@ -375,12 +375,12 @@ ALIGN_3 -.L14: +L(14): testq $15, N - jle .L999 + jle L(999) testq $8, N - jle .L15 + jle L(15) movaps 0 * SIZE(Y), %xmm1 movaps 0 * SIZE(X), %xmm0 @@ -446,9 +446,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $4, N - jle .L16 + jle L(16) movaps 0 * SIZE(Y), %xmm1 movaps 0 * SIZE(X), %xmm0 @@ -484,9 +484,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $2, N - jle .L17 + jle L(17) movaps 0 * SIZE(Y), %xmm1 movaps 0 * SIZE(X), %xmm0 @@ -510,9 +510,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $1, N - jle .L999 + jle L(999) movsd 0 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -531,18 +531,18 @@ movsd %xmm0, 0 * SIZE(X) movsd %xmm2, 0 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L20: +L(20): movaps -1 * SIZE(Y), %xmm1 movq N, %rax sarq $4, %rax - jle .L24 + jle L(24) ALIGN_3 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -703,15 +703,15 @@ addq $16 * SIZE, X addq $16 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L24: +L(24): testq $15, N - jle .L999 + jle L(999) testq $8, N - jle .L25 + jle L(25) movaps 1 * SIZE(Y), %xmm3 movaps 3 * SIZE(Y), %xmm8 @@ -787,9 +787,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $4, N - jle .L26 + jle L(26) movaps 1 * SIZE(Y), %xmm3 movaps 3 * SIZE(Y), %xmm8 @@ -831,9 +831,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $2, N - jle .L27 + jle L(27) movaps 1 * SIZE(Y), %xmm4 movaps 0 * SIZE(X), %xmm0 @@ -860,9 +860,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $1, N - jle .L999 + jle L(999) unpckhpd %xmm1, %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -881,21 +881,21 @@ movsd %xmm0, 0 * SIZE(X) movsd %xmm2, 0 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L50: +L(50): movq N, %rax cmpq $0, INCX - je .L56 + je L(56) cmpq $0, INCY - je .L56 + je L(56) sarq $2, %rax - jle .L55 + jle L(55) ALIGN_3 -.L53: +L(53): movsd (Y), %xmm1 movhps (Y, INCY), %xmm1 movsd (X), %xmm0 @@ -947,16 +947,16 @@ leaq (Y, INCY, 2), Y decq %rax - jg .L53 + jg L(53) ALIGN_3 -.L55: +L(55): movq N, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_3 -.L56: +L(56): movsd (Y), %xmm1 movsd (X), %xmm0 @@ -979,10 +979,10 @@ addq INCY, Y decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/scal.S b/kernel/x86_64/scal.S index 1f8e4d4447..8956802152 100644 --- a/kernel/x86_64/scal.S +++ b/kernel/x86_64/scal.S @@ -55,18 +55,18 @@ ftst fnstsw %ax andb $68, %ah - je .L300 + je L(300) /* Alpha == ZERO */ cmpq $1, INCX - jne .L104 + jne L(104) movq M, I sarq $3, I - jle .L102 + jle L(102) ALIGN_4 -.L101: +L(101): fld %st FST 0 * SIZE(X) fld %st @@ -86,34 +86,34 @@ addq $8 * SIZE, X decq I - jg .L101 + jg L(101) ALIGN_4 -.L102: +L(102): movq M, I andq $7, I - jle .L999 + jle L(999) ALIGN_4 -.L103: +L(103): fld %st FST 0 * SIZE(X) addq $SIZE, X decq I - jg .L103 - jmp .L999 + jg L(103) + jmp L(999) ALIGN_4 -.L104: +L(104): salq $BASE_SHIFT, INCX movq M, I sarq $3, I - jle .L106 + jle L(106) ALIGN_4 -.L105: +L(105): fld %st FST 0 * SIZE(X) addq INCX, X @@ -140,36 +140,36 @@ addq INCX, X decq I - jg .L105 + jg L(105) ALIGN_4 -.L106: +L(106): movq M, I andq $7, I - jle .L999 + jle L(999) ALIGN_4 -.L107: +L(107): fld %st FST 0 * SIZE(X) addq INCX, X decq I - jg .L107 - jmp .L999 + jg L(107) + jmp L(999) ALIGN_4 /* Alpha != ZERO */ -.L300: +L(300): cmpq $1,INCX - jne .L304 + jne L(304) movq M, I sarq $3, I - jle .L302 + jle L(302) ALIGN_4 -.L301: +L(301): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -208,34 +208,34 @@ addq $8 * SIZE, X decq I - jg .L301 + jg L(301) ALIGN_4 -.L302: +L(302): movq M, I andq $7, I - jle .L999 + jle L(999) ALIGN_4 -.L303: +L(303): FLD 0 * SIZE(X) fmul %st(1), %st FST 0 * SIZE(X) addq $SIZE, X decq I - jg .L303 - jmp .L999 + jg L(303) + jmp L(999) ALIGN_4 -.L304: +L(304): salq $BASE_SHIFT, INCX movq M, I sarq $3, I - jle .L306 + jle L(306) ALIGN_4 -.L305: +L(305): FLD 0 * SIZE(X) fmul %st(1), %st FST 0 * SIZE(X) @@ -277,25 +277,25 @@ addq INCX, X decq I - jg .L305 + jg L(305) ALIGN_4 -.L306: +L(306): movq M, I andq $7, I - jle .L999 + jle L(999) ALIGN_4 -.L307: +L(307): FLD 0 * SIZE(X) fmul %st(1), %st FST 0 * SIZE(X) addq INCX, X decq I - jg .L307 + jg L(307) ALIGN_4 -.L999: +L(999): ffreep %st(0) ret diff --git a/kernel/x86_64/scal_atom.S b/kernel/x86_64/scal_atom.S index 284ea4515a..a1c26cafb7 100644 --- a/kernel/x86_64/scal_atom.S +++ b/kernel/x86_64/scal_atom.S @@ -69,27 +69,27 @@ SAVEREGISTERS testq M, M - jle .L999 + jle L(999) pxor %xmm1, %xmm1 lea (, INCX, SIZE), INCX comisd %xmm0, %xmm1 - jne .L100 - jp .L100 + jne L(100) + jp L(100) cmpq $1, %r9 - je .L100 + je L(100) /* Alpha == ZERO */ cmpq $SIZE, INCX - jne .L50 + jne L(50) movq M, I sarq $3, I - jle .L12 + jle L(12) ALIGN_4 -.L11: +L(11): movsd %xmm1, 0 * SIZE(X) movsd %xmm1, 1 * SIZE(X) movsd %xmm1, 2 * SIZE(X) @@ -102,12 +102,12 @@ addq $8 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_4 -.L12: +L(12): testq $4, M - je .L14 + je L(14) movsd %xmm1, 0 * SIZE(X) movsd %xmm1, 1 * SIZE(X) @@ -117,9 +117,9 @@ addq $4 * SIZE, X ALIGN_3 -.L14: +L(14): testq $2, M - je .L15 + je L(15) movsd %xmm1, 0 * SIZE(X) movsd %xmm1, 1 * SIZE(X) @@ -127,21 +127,21 @@ addq $2 * SIZE, X ALIGN_3 -.L15: +L(15): testq $1, M - je .L999 + je L(999) movsd %xmm1, 0 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_4 -.L50: +L(50): movq M, I sarq $3, I - jle .L52 + jle L(52) ALIGN_4 -.L51: +L(51): movsd %xmm1, 0 * SIZE(X) addq INCX, X movsd %xmm1, 0 * SIZE(X) @@ -160,15 +160,15 @@ addq INCX, X decq I - jg .L51 + jg L(51) ALIGN_4 -.L52: +L(52): testq $7, M - je .L999 + je L(999) testq $4, M - je .L53 + je L(53) movsd %xmm1, 0 * SIZE(X) addq INCX, X @@ -180,9 +180,9 @@ addq INCX, X ALIGN_3 -.L53: +L(53): testq $2, M - je .L54 + je L(54) movsd %xmm1, 0 * SIZE(X) addq INCX, X @@ -190,25 +190,25 @@ addq INCX, X ALIGN_3 -.L54: +L(54): testq $1, M - je .L999 + je L(999) movsd %xmm1, 0 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_4 /* Alpha != ZERO */ -.L100: +L(100): cmpq $SIZE, INCX - jne .L150 + jne L(150) unpcklpd %xmm0, %xmm0 movq M, I sarq $3, I - jle .L113 + jle L(113) movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 @@ -225,10 +225,10 @@ mulsd %xmm0, %xmm4 decq I - jle .L112 + jle L(112) ALIGN_4 -.L111: +L(111): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -263,10 +263,10 @@ addq $8 * SIZE, X decq I - jg .L111 + jg L(111) ALIGN_4 -.L112: +L(112): movsd %xmm1, 0 * SIZE(X) mulsd %xmm0, %xmm5 movsd %xmm2, 1 * SIZE(X) @@ -283,9 +283,9 @@ addq $8 * SIZE, X ALIGN_3 -.L113: +L(113): testq $4, M - je .L115 + je L(115) movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 @@ -305,9 +305,9 @@ addq $4 * SIZE, X ALIGN_3 -.L115: +L(115): testq $2, M - je .L116 + je L(116) movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 @@ -321,26 +321,26 @@ addq $2 * SIZE, X ALIGN_3 -.L116: +L(116): testq $1, M - je .L999 + je L(999) movsd 0 * SIZE(X), %xmm1 mulsd %xmm0, %xmm1 movsd %xmm1, 0 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_3 /* incx != 1 */ -.L150: +L(150): movq X, XX movq M, I # rcx = n sarq $3, I # (n >> 3) - jle .L152 + jle L(152) ALIGN_4 -.L151: +L(151): movsd 0 * SIZE(X), %xmm1 addq INCX, X movsd 0 * SIZE(X), %xmm2 @@ -383,15 +383,15 @@ movsd %xmm8, 0 * SIZE(XX) addq INCX, XX decq I - jg .L151 + jg L(151) ALIGN_4 -.L152: +L(152): testq $7, M - je .L999 + je L(999) testq $4, M - je .L153 + je L(153) movsd 0 * SIZE(X), %xmm1 addq INCX, X @@ -416,9 +416,9 @@ addq INCX, XX ALIGN_3 -.L153: +L(153): testq $2, M - je .L154 + je L(154) movsd 0 * SIZE(X), %xmm1 addq INCX, X @@ -433,16 +433,16 @@ addq INCX, XX ALIGN_3 -.L154: +L(154): testq $1, M - je .L999 + je L(999) movsd 0 * SIZE(X), %xmm1 mulsd %xmm0, %xmm1 movsd %xmm1, 0 * SIZE(X) ALIGN_4 -.L999: +L(999): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/scal_sse.S b/kernel/x86_64/scal_sse.S index 88ef4a3eda..2c3d910dbf 100644 --- a/kernel/x86_64/scal_sse.S +++ b/kernel/x86_64/scal_sse.S @@ -69,7 +69,7 @@ SAVEREGISTERS testq M, M - jle .L999 + jle L(999) lea (, INCX, SIZE), INCX @@ -77,44 +77,44 @@ comiss %xmm0, %xmm1 shufps $0, %xmm0, %xmm0 - jne .L100 # Alpha != ZERO + jne L(100) # Alpha != ZERO cmpq $1, %r9 - je .L100 + je L(100) /* Alpha == ZERO */ cmpq $SIZE, INCX - jne .L50 + jne L(50) /* INCX == 1 */ cmpq $3, M - jle .L14 + jle L(14) testq $4, X # aligned for double word? - je .L05 + je L(05) movss %xmm1, 0 * SIZE(X) addq $SIZE, X decq M - jle .L999 + jle L(999) ALIGN_3 -.L05: +L(05): testq $8, X # aligned for quad word? - je .L06 + je L(06) movsd %xmm1, 0 * SIZE(X) addq $2 * SIZE, X subq $2, M - jle .L999 + jle L(999) ALIGN_3 -.L06: +L(06): movq M, I sarq $4, I - jle .L12 + jle L(12) ALIGN_4 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -125,52 +125,52 @@ movaps %xmm1, 12 * SIZE(X) addq $16 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_4 -.L12: +L(12): testq $15, M - je .L999 + je L(999) testq $8, M - je .L13 + je L(13) movaps %xmm1, 0 * SIZE(X) movaps %xmm1, 4 * SIZE(X) addq $8 * SIZE, X ALIGN_3 -.L13: +L(13): testq $4, M - je .L14 + je L(14) movaps %xmm1, 0 * SIZE(X) addq $4 * SIZE, X ALIGN_3 -.L14: +L(14): testq $2, M - je .L15 + je L(15) movsd %xmm1, 0 * SIZE(X) addq $2 * SIZE, X ALIGN_3 -.L15: +L(15): testq $1, M - je .L999 + je L(999) movss %xmm1, 0 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_4 /* incx != 1 */ -.L50: +L(50): movq M, I # rcx = n sarq $3, I # (n >> 3) - jle .L52 + jle L(52) ALIGN_4 -.L51: +L(51): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -193,15 +193,15 @@ addq INCX, X decq I - jg .L51 + jg L(51) ALIGN_4 -.L52: +L(52): testq $7, M - je .L999 + je L(999) testq $4, M - je .L53 + je L(53) movss %xmm1, (X) addq INCX, X @@ -213,9 +213,9 @@ addq INCX, X ALIGN_3 -.L53: +L(53): testq $2, M - je .L54 + je L(54) movss %xmm1, (X) addq INCX, X @@ -223,52 +223,52 @@ addq INCX, X ALIGN_3 -.L54: +L(54): testq $1, M - je .L999 + je L(999) movss %xmm1, (X) - jmp .L999 + jmp L(999) ALIGN_4 /* Alpha != ZERO */ -.L100: +L(100): cmpq $SIZE, INCX - jne .L150 + jne L(150) subq $-32 * SIZE, X cmpq $3, M - jle .L116 + jle L(116) testq $SIZE, X - je .L105 + je L(105) movss -32 * SIZE(X), %xmm1 mulss %xmm0, %xmm1 movss %xmm1, -32 * SIZE(X) addq $SIZE, X decq M - jle .L999 + jle L(999) ALIGN_3 -.L105: +L(105): testq $2 * SIZE, X - je .L110 + je L(110) movsd -32 * SIZE(X), %xmm1 mulps %xmm0, %xmm1 movsd %xmm1, -32 * SIZE(X) addq $2 * SIZE, X subq $2, M - jle .L999 + jle L(999) ALIGN_3 -.L110: +L(110): movq M, I sarq $5, I - jle .L113 + jle L(113) #if defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) @@ -290,10 +290,10 @@ mulps -4 * SIZE(X), %xmm8 decq I - jle .L112 + jle L(112) ALIGN_4 -.L111: +L(111): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -332,10 +332,10 @@ subq $-32 * SIZE, X decq I - jg .L111 + jg L(111) ALIGN_4 -.L112: +L(112): movaps %xmm1, -32 * SIZE(X) movaps %xmm2, -28 * SIZE(X) movaps %xmm3, -24 * SIZE(X) @@ -357,10 +357,10 @@ movaps -8 * SIZE(X), %xmm7 movaps -4 * SIZE(X), %xmm8 decq I - jle .L112 + jle L(112) ALIGN_4 -.L111: +L(111): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -397,10 +397,10 @@ subq $-32 * SIZE, X decq I - jg .L111 + jg L(111) ALIGN_4 -.L112: +L(112): mulps %xmm0, %xmm1 movaps %xmm1, -32 * SIZE(X) mulps %xmm0, %xmm2 @@ -424,12 +424,12 @@ subq $-32 * SIZE, X ALIGN_3 -.L113: +L(113): testq $31, M - je .L999 + je L(999) testq $16, M - je .L114 + je L(114) movaps -32 * SIZE(X), %xmm1 movaps -28 * SIZE(X), %xmm3 @@ -448,9 +448,9 @@ addq $16 * SIZE, X ALIGN_3 -.L114: +L(114): testq $8, M - je .L115 + je L(115) movaps -32 * SIZE(X), %xmm1 movaps -28 * SIZE(X), %xmm3 @@ -462,9 +462,9 @@ addq $8 * SIZE, X ALIGN_3 -.L115: +L(115): testq $4, M - je .L116 + je L(116) movaps -32 * SIZE(X), %xmm1 mulps %xmm0, %xmm1 @@ -472,9 +472,9 @@ addq $4 * SIZE, X ALIGN_3 -.L116: +L(116): testq $2, M - je .L117 + je L(117) movsd -32 * SIZE(X), %xmm1 mulps %xmm0, %xmm1 @@ -482,26 +482,26 @@ addq $2 * SIZE, X ALIGN_3 -.L117: +L(117): testq $1, M - je .L999 + je L(999) movss -32 * SIZE(X), %xmm1 mulss %xmm0, %xmm1 movss %xmm1, -32 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_3 /* incx != 1 */ -.L150: +L(150): movq X, XX movq M, I # rcx = n sarq $3, I # (n >> 3) - jle .L152 + jle L(152) ALIGN_4 -.L151: +L(151): movss (X), %xmm1 addq INCX, X movss (X), %xmm2 @@ -545,15 +545,15 @@ movss %xmm8, (XX) addq INCX, XX decq I - jg .L151 + jg L(151) ALIGN_4 -.L152: +L(152): testq $7, M - je .L999 + je L(999) testq $4, M - je .L153 + je L(153) movss (X), %xmm1 addq INCX, X @@ -579,9 +579,9 @@ addq INCX, XX ALIGN_3 -.L153: +L(153): testq $2, M - je .L154 + je L(154) movss (X), %xmm1 addq INCX, X @@ -597,16 +597,16 @@ addq INCX, XX ALIGN_3 -.L154: +L(154): testq $1, M - je .L999 + je L(999) movss (X), %xmm1 mulss %xmm0, %xmm1 movss %xmm1, (X) ALIGN_4 -.L999: +L(999): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/scal_sse2.S b/kernel/x86_64/scal_sse2.S index 485e6ef462..6d5bb5b585 100644 --- a/kernel/x86_64/scal_sse2.S +++ b/kernel/x86_64/scal_sse2.S @@ -70,39 +70,39 @@ SAVEREGISTERS testq M, M - jle .L999 + jle L(999) leaq (, INCX, SIZE), INCX xorps %xmm1, %xmm1 comisd %xmm0, %xmm1 - jne .L100 # Alpha != ZERO - jp .L100 # For Alpha = NaN + jne L(100) # Alpha != ZERO + jp L(100) # For Alpha = NaN cmpq $1, FLAG - je .L100 # disable the Alpha=zero path as it does not handle x=inf or nan + je L(100) # disable the Alpha=zero path as it does not handle x=inf or nan /* Alpha == ZERO */ cmpq $SIZE, INCX - jne .L50 + jne L(50) /* INCX == 1 */ testq $15, X # aligned for quad word? - je .L05 + je L(05) movsd %xmm1, 0 * SIZE(X) addq $SIZE, X decq M - jle .L999 + jle L(999) ALIGN_3 -.L05: +L(05): /* Aligned Mode */ movq M, I # rcx = n sarq $4, I - jle .L12 + jle L(12) ALIGN_4 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -123,14 +123,14 @@ addq $16 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_4 -.L12: +L(12): testq $15, M - je .L999 + je L(999) testq $8, M - je .L13 + je L(13) movaps %xmm1, 0 * SIZE(X) movaps %xmm1, 2 * SIZE(X) @@ -139,38 +139,38 @@ addq $8 * SIZE, X ALIGN_3 -.L13: +L(13): testq $4, M - je .L14 + je L(14) movaps %xmm1, 0 * SIZE(X) movaps %xmm1, 2 * SIZE(X) addq $4 * SIZE, X ALIGN_3 -.L14: +L(14): testq $2, M - je .L15 + je L(15) movaps %xmm1, 0 * SIZE(X) addq $2 * SIZE, X ALIGN_3 -.L15: +L(15): testq $1, M - je .L999 + je L(999) movsd %xmm1, 0 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_4 -.L50: +L(50): movq M, I sarq $3, I - jle .L52 + jle L(52) ALIGN_4 -.L51: +L(51): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -193,15 +193,15 @@ addq INCX, X decq I - jg .L51 + jg L(51) ALIGN_4 -.L52: +L(52): testq $7, M - je .L999 + je L(999) testq $4, M - je .L53 + je L(53) movsd %xmm1, (X) addq INCX, X @@ -213,9 +213,9 @@ addq INCX, X ALIGN_3 -.L53: +L(53): testq $2, M - je .L54 + je L(54) movsd %xmm1, (X) addq INCX, X @@ -223,38 +223,38 @@ addq INCX, X ALIGN_3 -.L54: +L(54): testq $1, M - je .L999 + je L(999) movsd %xmm1, (X) - jmp .L999 + jmp L(999) ALIGN_4 /* Alpha != ZERO */ -.L100: +L(100): unpcklpd %xmm0, %xmm0 cmpq $SIZE, INCX - jne .L150 + jne L(150) testq $SIZE, X - je .L105 + je L(105) movsd 0 * SIZE(X), %xmm1 mulsd %xmm0, %xmm1 movsd %xmm1, 0 * SIZE(X) addq $SIZE, X decq M - jle .L999 + jle L(999) ALIGN_3 -.L105: +L(105): subq $-16 * SIZE, X movq M, I # rcx = n sarq $4, I - jle .L113 + jle L(113) #if defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) @@ -276,10 +276,10 @@ mulpd -2 * SIZE(X), %xmm8 decq I - jle .L112 + jle L(112) ALIGN_4 -.L111: +L(111): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -318,10 +318,10 @@ subq $-16 * SIZE, X decq I - jg .L111 + jg L(111) ALIGN_4 -.L112: +L(112): movaps %xmm1, -16 * SIZE(X) movaps %xmm2, -14 * SIZE(X) movaps %xmm3, -12 * SIZE(X) @@ -342,10 +342,10 @@ movaps -2 * SIZE(X), %xmm8 decq I - jle .L112 + jle L(112) ALIGN_4 -.L111: +L(111): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -384,10 +384,10 @@ subq $-16 * SIZE, X decq I - jg .L111 + jg L(111) ALIGN_4 -.L112: +L(112): mulpd %xmm0, %xmm1 movaps %xmm1, -16 * SIZE(X) mulpd %xmm0, %xmm2 @@ -410,12 +410,12 @@ subq $-16 * SIZE, X ALIGN_3 -.L113: +L(113): testq $15, M - je .L999 + je L(999) testq $8, M - je .L114 + je L(114) movaps -16 * SIZE(X), %xmm1 movaps -14 * SIZE(X), %xmm2 @@ -433,9 +433,9 @@ addq $8 * SIZE, X ALIGN_3 -.L114: +L(114): testq $4, M - je .L115 + je L(115) movaps -16 * SIZE(X), %xmm1 movaps -14 * SIZE(X), %xmm2 @@ -447,9 +447,9 @@ addq $4 * SIZE, X ALIGN_3 -.L115: +L(115): testq $2, M - je .L116 + je L(116) movaps -16 * SIZE(X), %xmm1 mulpd %xmm0, %xmm1 @@ -457,26 +457,26 @@ addq $2 * SIZE, X ALIGN_3 -.L116: +L(116): testq $1, M - je .L999 + je L(999) movsd -16 * SIZE(X), %xmm1 mulsd %xmm0, %xmm1 movsd %xmm1, -16 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_3 /* incx != 1 */ -.L150: +L(150): movq X, XX movq M, I # rcx = n sarq $3, I # (n >> 3) - jle .L152 + jle L(152) ALIGN_4 -.L151: +L(151): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(X) #endif @@ -523,15 +523,15 @@ movsd %xmm8, (XX) addq INCX, XX decq I - jg .L151 + jg L(151) ALIGN_4 -.L152: +L(152): testq $7, M - je .L999 + je L(999) testq $4, M - je .L153 + je L(153) movsd (X), %xmm1 addq INCX, X @@ -557,9 +557,9 @@ addq INCX, XX ALIGN_3 -.L153: +L(153): testq $2, M - je .L154 + je L(154) movsd (X), %xmm1 addq INCX, X @@ -575,16 +575,16 @@ addq INCX, XX ALIGN_3 -.L154: +L(154): testq $1, M - je .L999 + je L(999) movsd (X), %xmm1 mulsd %xmm0, %xmm1 movsd %xmm1, (X) ALIGN_4 -.L999: +L(999): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/sgemm_kernel_16x2_bulldozer.S b/kernel/x86_64/sgemm_kernel_16x2_bulldozer.S index b31a934f23..0c26923eb8 100644 --- a/kernel/x86_64/sgemm_kernel_16x2_bulldozer.S +++ b/kernel/x86_64/sgemm_kernel_16x2_bulldozer.S @@ -1033,13 +1033,13 @@ STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -1059,10 +1059,10 @@ movq Ndiv6, J cmpq $0, J - je .L2_0 + je L(2_0) ALIGN_4 -.L6_01: +L(6_01): // copy to sub buffer movq K, %rax salq $1,%rax // K * 2 ; read 2 values @@ -1071,10 +1071,10 @@ leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $3 , %rax // K / 8 - jz .L6_01a_2 + jz L(6_01a_2) ALIGN_4 -.L6_01a_1: +L(6_01a_1): prefetcht0 512(BO1) prefetcht0 512(BO2) @@ -1121,19 +1121,19 @@ addq $12*SIZE,BO decq %rax - jnz .L6_01a_1 + jnz L(6_01a_1) -.L6_01a_2: +L(6_01a_2): movq K, %rax andq $7, %rax // K % 8 - jz .L6_02c + jz L(6_02c) ALIGN_4 -.L6_02b: +L(6_02b): vmovsd 0 * SIZE(BO1), %xmm0 vmovss 0 * SIZE(BO2), %xmm2 @@ -1143,9 +1143,9 @@ addq $2*SIZE,BO2 addq $3*SIZE,BO decq %rax - jnz .L6_02b + jnz L(6_02b) -.L6_02c: +L(6_02c): movq K, %rax salq $1,%rax // K * 2 @@ -1154,10 +1154,10 @@ leaq BUFFER2, BO // second buffer to BO movq K, %rax sarq $3 , %rax // K / 8 - jz .L6_02c_2 + jz L(6_02c_2) ALIGN_4 -.L6_02c_1: +L(6_02c_1): prefetcht0 512(BO2) prefetchw 512(BO) @@ -1204,17 +1204,17 @@ addq $12*SIZE,BO decq %rax - jnz .L6_02c_1 + jnz L(6_02c_1) -.L6_02c_2: +L(6_02c_2): movq K, %rax andq $7, %rax // K % 8 - jz .L6_03c + jz L(6_03c) ALIGN_4 -.L6_03b: +L(6_03b): vmovss 1*SIZE(BO1), %xmm0 vmovsd 0*SIZE(BO2), %xmm1 @@ -1224,14 +1224,14 @@ addq $2*SIZE,BO2 addq $3*SIZE,BO decq %rax - jnz .L6_03b + jnz L(6_03b) -.L6_03c: +L(6_03c): movq BO2, B // next offset of B -.L6_10: +L(6_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -1242,11 +1242,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L6_20 + je L(6_20) ALIGN_4 -.L6_11: +L(6_11): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1255,7 +1255,7 @@ movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L6_16 + je L(6_16) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1266,7 +1266,7 @@ negq %rax ALIGN_4 -.L6_12: +L(6_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x3_1(xxx) @@ -1280,7 +1280,7 @@ KERNEL16x3_3(xxx) KERNEL16x3_4(xxx) - je .L6_16 + je L(6_16) KERNEL16x3_1(xxx) KERNEL16x3_2(xxx) @@ -1293,16 +1293,16 @@ KERNEL16x3_3(xxx) KERNEL16x3_4(xxx) - je .L6_16 + je L(6_16) - jmp .L6_12 + jmp L(6_12) ALIGN_4 -.L6_16: +L(6_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_19 + je L(6_19) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1314,16 +1314,16 @@ negq %rax ALIGN_4 -.L6_17: +L(6_17): KERNEL16x3_SUB(xxx) addq $3, BI addq $16, %rax - jl .L6_17 + jl L(6_17) ALIGN_4 -.L6_19: +L(6_19): vbroadcastss ALPHA, %xmm0 @@ -1361,25 +1361,25 @@ addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L6_11 + jg L(6_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L6_20: +L(6_20): // Test rest of M testq $15, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N testq $8, M - jz .L6_21pre + jz L(6_21pre) ALIGN_4 /**************************************************************************/ -.L6_20_1: +L(6_20_1): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1388,7 +1388,7 @@ movq K, %rax andq $-8, %rax - je .L6_20_6 + je L(6_20_6) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1399,7 +1399,7 @@ negq %rax ALIGN_4 -.L6_20_2: +L(6_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x3_1(xxx) @@ -1413,7 +1413,7 @@ KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L6_20_6 + je L(6_20_6) KERNEL8x3_1(xxx) KERNEL8x3_2(xxx) @@ -1426,16 +1426,16 @@ KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L6_20_6 + je L(6_20_6) - jmp .L6_20_2 + jmp L(6_20_2) ALIGN_4 -.L6_20_6: +L(6_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_20_9 + je L(6_20_9) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1447,16 +1447,16 @@ negq %rax ALIGN_4 -.L6_20_7: +L(6_20_7): KERNEL8x3_SUB(xxx) addq $3, BI addq $8, %rax - jl .L6_20_7 + jl L(6_20_7) ALIGN_4 -.L6_20_9: +L(6_20_9): vbroadcastss ALPHA, %xmm0 @@ -1487,13 +1487,13 @@ /**************************************************************************/ -.L6_21pre: +L(6_21pre): testq $4, M - jz .L6_30 + jz L(6_30) ALIGN_4 -.L6_21: +L(6_21): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1502,7 +1502,7 @@ movq K, %rax andq $-8, %rax - je .L6_26 + je L(6_26) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1513,7 +1513,7 @@ negq %rax ALIGN_4 -.L6_22: +L(6_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x3_1(xxx) @@ -1527,7 +1527,7 @@ KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L6_26 + je L(6_26) KERNEL4x3_1(xxx) KERNEL4x3_2(xxx) @@ -1540,16 +1540,16 @@ KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L6_26 + je L(6_26) - jmp .L6_22 + jmp L(6_22) ALIGN_4 -.L6_26: +L(6_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_29 + je L(6_29) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1561,16 +1561,16 @@ negq %rax ALIGN_4 -.L6_27: +L(6_27): KERNEL4x3_SUB(xxx) addq $3, BI addq $4, %rax - jl .L6_27 + jl L(6_27) ALIGN_4 -.L6_29: +L(6_29): vbroadcastss ALPHA, %xmm0 @@ -1587,13 +1587,13 @@ ALIGN_4 -.L6_30: +L(6_30): testq $2, M - jz .L6_40 + jz L(6_40) ALIGN_4 -.L6_31: +L(6_31): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1602,7 +1602,7 @@ movq K, %rax andq $-8, %rax - je .L6_36 + je L(6_36) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1613,7 +1613,7 @@ negq %rax ALIGN_4 -.L6_32: +L(6_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x3_1(xxx) @@ -1627,7 +1627,7 @@ KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L6_36 + je L(6_36) KERNEL2x3_1(xxx) KERNEL2x3_2(xxx) @@ -1640,16 +1640,16 @@ KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L6_36 + je L(6_36) - jmp .L6_32 + jmp L(6_32) ALIGN_4 -.L6_36: +L(6_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_39 + je L(6_39) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1661,16 +1661,16 @@ negq %rax ALIGN_4 -.L6_37: +L(6_37): KERNEL2x3_SUB(xxx) addq $3, BI addq $2, %rax - jl .L6_37 + jl L(6_37) ALIGN_4 -.L6_39: +L(6_39): vmovss ALPHA, %xmm0 @@ -1692,13 +1692,13 @@ addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L6_40: +L(6_40): testq $1, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N ALIGN_4 -.L6_41: +L(6_41): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1707,7 +1707,7 @@ movq K, %rax andq $-8, %rax - je .L6_46 + je L(6_46) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1717,7 +1717,7 @@ negq %rax ALIGN_4 -.L6_42: +L(6_42): KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -1729,7 +1729,7 @@ KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L6_46 + je L(6_46) KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -1741,16 +1741,16 @@ KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L6_46 + je L(6_46) - jmp .L6_42 + jmp L(6_42) ALIGN_4 -.L6_46: +L(6_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_49 + je L(6_49) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1761,16 +1761,16 @@ negq %rax ALIGN_4 -.L6_47: +L(6_47): KERNEL1x3_SUB(xxx) addq $3, BI addq $1, %rax - jl .L6_47 + jl L(6_47) ALIGN_4 -.L6_49: +L(6_49): vmovss ALPHA, %xmm0 @@ -1791,7 +1791,7 @@ /***************************************************************************************************************/ -.L7_10: +L(7_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -1802,11 +1802,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L7_20 + je L(7_20) ALIGN_4 -.L7_11: +L(7_11): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -1815,7 +1815,7 @@ movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L7_16 + je L(7_16) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1826,7 +1826,7 @@ negq %rax ALIGN_4 -.L7_12: +L(7_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x3_1(xxx) @@ -1840,7 +1840,7 @@ KERNEL16x3_3(xxx) KERNEL16x3_4(xxx) - je .L7_16 + je L(7_16) KERNEL16x3_1(xxx) KERNEL16x3_2(xxx) @@ -1853,16 +1853,16 @@ KERNEL16x3_3(xxx) KERNEL16x3_4(xxx) - je .L7_16 + je L(7_16) - jmp .L7_12 + jmp L(7_12) ALIGN_4 -.L7_16: +L(7_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_19 + je L(7_19) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1874,16 +1874,16 @@ negq %rax ALIGN_4 -.L7_17: +L(7_17): KERNEL16x3_SUB(xxx) addq $3, BI addq $16, %rax - jl .L7_17 + jl L(7_17) ALIGN_4 -.L7_19: +L(7_19): vbroadcastss ALPHA, %xmm0 @@ -1921,25 +1921,25 @@ addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L7_11 + jg L(7_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L7_20: +L(7_20): // Test rest of M testq $15, M - jz .L7_60 // to next 3 lines of N + jz L(7_60) // to next 3 lines of N testq $8, M - jz .L7_21pre + jz L(7_21pre) ALIGN_4 /**************************************************************************/ -.L7_20_1: +L(7_20_1): leaq BUFFER2, BO // first buffer to BO addq $6 * SIZE, BO @@ -1948,7 +1948,7 @@ movq K, %rax andq $-8, %rax - je .L7_20_6 + je L(7_20_6) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1959,7 +1959,7 @@ negq %rax ALIGN_4 -.L7_20_2: +L(7_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x3_1(xxx) @@ -1973,7 +1973,7 @@ KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L7_20_6 + je L(7_20_6) KERNEL8x3_1(xxx) KERNEL8x3_2(xxx) @@ -1986,16 +1986,16 @@ KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L7_20_6 + je L(7_20_6) - jmp .L7_20_2 + jmp L(7_20_2) ALIGN_4 -.L7_20_6: +L(7_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_20_9 + je L(7_20_9) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2007,15 +2007,15 @@ negq %rax ALIGN_4 -.L7_20_7: +L(7_20_7): KERNEL8x3_SUB(xxx) addq $3, BI addq $8, %rax - jl .L7_20_7 + jl L(7_20_7) ALIGN_4 -.L7_20_9: +L(7_20_9): vbroadcastss ALPHA, %xmm0 @@ -2046,13 +2046,13 @@ /**************************************************************************/ -.L7_21pre: +L(7_21pre): testq $4, M - jz .L7_30 + jz L(7_30) ALIGN_4 -.L7_21: +L(7_21): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -2061,7 +2061,7 @@ movq K, %rax andq $-8, %rax - je .L7_26 + je L(7_26) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2072,7 +2072,7 @@ negq %rax ALIGN_4 -.L7_22: +L(7_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x3_1(xxx) @@ -2086,7 +2086,7 @@ KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L7_26 + je L(7_26) KERNEL4x3_1(xxx) KERNEL4x3_2(xxx) @@ -2099,16 +2099,16 @@ KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L7_26 + je L(7_26) - jmp .L7_22 + jmp L(7_22) ALIGN_4 -.L7_26: +L(7_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_29 + je L(7_29) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2120,16 +2120,16 @@ negq %rax ALIGN_4 -.L7_27: +L(7_27): KERNEL4x3_SUB(xxx) addq $3, BI addq $4, %rax - jl .L7_27 + jl L(7_27) ALIGN_4 -.L7_29: +L(7_29): vbroadcastss ALPHA, %xmm0 @@ -2145,13 +2145,13 @@ ALIGN_4 -.L7_30: +L(7_30): testq $2, M - jz .L7_40 + jz L(7_40) ALIGN_4 -.L7_31: +L(7_31): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -2160,7 +2160,7 @@ movq K, %rax andq $-8, %rax - je .L7_36 + je L(7_36) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2171,7 +2171,7 @@ negq %rax ALIGN_4 -.L7_32: +L(7_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x3_1(xxx) @@ -2185,7 +2185,7 @@ KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L7_36 + je L(7_36) KERNEL2x3_1(xxx) KERNEL2x3_2(xxx) @@ -2198,16 +2198,16 @@ KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L7_36 + je L(7_36) - jmp .L7_32 + jmp L(7_32) ALIGN_4 -.L7_36: +L(7_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_39 + je L(7_39) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2219,16 +2219,16 @@ negq %rax ALIGN_4 -.L7_37: +L(7_37): KERNEL2x3_SUB(xxx) addq $3, BI addq $2, %rax - jl .L7_37 + jl L(7_37) ALIGN_4 -.L7_39: +L(7_39): vmovss ALPHA, %xmm0 @@ -2249,13 +2249,13 @@ addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L7_40: +L(7_40): testq $1, M - jz .L7_60 // to next 3 lines of N + jz L(7_60) // to next 3 lines of N ALIGN_4 -.L7_41: +L(7_41): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -2264,7 +2264,7 @@ movq K, %rax andq $-8, %rax - je .L7_46 + je L(7_46) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2274,7 +2274,7 @@ negq %rax ALIGN_4 -.L7_42: +L(7_42): KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -2286,7 +2286,7 @@ KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L7_46 + je L(7_46) KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -2298,16 +2298,16 @@ KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L7_46 + je L(7_46) - jmp .L7_42 + jmp L(7_42) ALIGN_4 -.L7_46: +L(7_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_49 + je L(7_49) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2318,16 +2318,16 @@ negq %rax ALIGN_4 -.L7_47: +L(7_47): KERNEL1x3_SUB(xxx) addq $3, BI addq $1, %rax - jl .L7_47 + jl L(7_47) ALIGN_4 -.L7_49: +L(7_49): vmovss ALPHA, %xmm0 @@ -2344,15 +2344,15 @@ -.L7_60: +L(7_60): decq J // j -- - jg .L6_01 + jg L(6_01) -.L2_0: +L(2_0): cmpq $0, Nmod6 // N % 6 == 0 - je .L999 + je L(999) /************************************************************************************************ * Loop for Nmod6 / 2 > 0 @@ -2360,30 +2360,30 @@ movq Nmod6, J sarq $1, J // j = j / 2 - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -2393,11 +2393,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2406,7 +2406,7 @@ movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2417,7 +2417,7 @@ negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x2_1(xxx) @@ -2430,7 +2430,7 @@ KERNEL16x2_3(xxx) KERNEL16x2_4(xxx) - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x2_1(xxx) @@ -2443,16 +2443,16 @@ KERNEL16x2_3(xxx) KERNEL16x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2464,16 +2464,16 @@ negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB(xxx) addq $2, BI addq $16, %rax - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vbroadcastss ALPHA, %xmm0 @@ -2499,25 +2499,25 @@ addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 3 lines of N + jz L(2_60) // to next 3 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2526,7 +2526,7 @@ movq K, %rax andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2537,7 +2537,7 @@ negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x2_1(xxx) @@ -2550,7 +2550,7 @@ KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_20_6 + je L(2_20_6) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x2_1(xxx) @@ -2563,16 +2563,16 @@ KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2584,16 +2584,16 @@ negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB(xxx) addq $2, BI addq $8, %rax - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): vbroadcastss ALPHA, %xmm0 @@ -2617,13 +2617,13 @@ /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2632,7 +2632,7 @@ movq K, %rax andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -2643,7 +2643,7 @@ negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x2_1(xxx) @@ -2656,7 +2656,7 @@ KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x2_1(xxx) @@ -2669,16 +2669,16 @@ KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2690,16 +2690,16 @@ negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB(xxx) addq $2, BI addq $4, %rax - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): vbroadcastss ALPHA, %xmm0 @@ -2714,13 +2714,13 @@ ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2729,7 +2729,7 @@ movq K, %rax andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2740,7 +2740,7 @@ negq %rax ALIGN_4 -.L2_32: +L(2_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -2753,7 +2753,7 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -2766,16 +2766,16 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2787,16 +2787,16 @@ negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB(xxx) addq $2, BI addq $2, %rax - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): vmovss ALPHA, %xmm0 @@ -2813,13 +2813,13 @@ addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2828,7 +2828,7 @@ movq K, %rax andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2838,7 +2838,7 @@ negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -2850,7 +2850,7 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -2862,16 +2862,16 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2882,16 +2882,16 @@ negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) addq $2, BI addq $1, %rax - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vmovss ALPHA, %xmm0 @@ -2904,14 +2904,14 @@ addq $1 * SIZE, CO1 # coffset += 1 ALIGN_4 -.L2_60: +L(2_60): decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -2919,30 +2919,30 @@ movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovss (BO1), %xmm0 vmovss %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -2952,11 +2952,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -2965,7 +2965,7 @@ movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -2975,7 +2975,7 @@ negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x1_1(xxx) @@ -2988,7 +2988,7 @@ KERNEL16x1_3(xxx) KERNEL16x1_4(xxx) - je .L1_16 + je L(1_16) KERNEL16x1_1(xxx) KERNEL16x1_2(xxx) @@ -3000,16 +3000,16 @@ KERNEL16x1_3(xxx) KERNEL16x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -3020,16 +3020,16 @@ negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB(xxx) addq $1, BI addq $16, %rax - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vbroadcastss ALPHA, %xmm0 @@ -3045,25 +3045,25 @@ addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3072,7 +3072,7 @@ movq K, %rax andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -3082,7 +3082,7 @@ negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x1_1(xxx) @@ -3095,7 +3095,7 @@ KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_20_6 + je L(1_20_6) KERNEL8x1_1(xxx) KERNEL8x1_2(xxx) @@ -3107,16 +3107,16 @@ KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -3127,16 +3127,16 @@ negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB(xxx) addq $1, BI addq $8, %rax - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): vbroadcastss ALPHA, %xmm0 @@ -3153,13 +3153,13 @@ /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3168,7 +3168,7 @@ movq K, %rax andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -3178,7 +3178,7 @@ negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x1_1(xxx) @@ -3191,7 +3191,7 @@ KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) KERNEL4x1_1(xxx) KERNEL4x1_2(xxx) @@ -3203,16 +3203,16 @@ KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -3223,16 +3223,16 @@ negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB(xxx) addq $1, BI addq $4, %rax - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): vbroadcastss ALPHA, %xmm0 @@ -3244,13 +3244,13 @@ ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3259,7 +3259,7 @@ movq K, %rax andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -3269,7 +3269,7 @@ negq %rax ALIGN_4 -.L1_32: +L(1_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -3282,7 +3282,7 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) KERNEL2x1_1(xxx) KERNEL2x1_2(xxx) @@ -3294,16 +3294,16 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -3314,16 +3314,16 @@ negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB(xxx) addq $1, BI addq $2, %rax - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): vmovss ALPHA, %xmm0 @@ -3336,13 +3336,13 @@ addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3351,7 +3351,7 @@ movq K, %rax andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -3360,7 +3360,7 @@ negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -3372,7 +3372,7 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -3384,16 +3384,16 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -3403,16 +3403,16 @@ negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) addq $1, BI addq $1, %rax - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vmovss ALPHA, %xmm0 @@ -3424,7 +3424,7 @@ ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp @@ -3514,13 +3514,13 @@ STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -3549,30 +3549,30 @@ movq Ndiv6, J cmpq $0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -3586,11 +3586,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3627,7 +3627,7 @@ #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3638,7 +3638,7 @@ negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x2_1(xxx) @@ -3651,7 +3651,7 @@ KERNEL16x2_3(xxx) KERNEL16x2_4(xxx) - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x2_1(xxx) @@ -3664,12 +3664,12 @@ KERNEL16x2_3(xxx) KERNEL16x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3677,7 +3677,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3689,16 +3689,16 @@ negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB(xxx) addq $2, BI addq $16, %rax - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vbroadcastss ALPHA, %xmm0 @@ -3756,25 +3756,25 @@ addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 3 lines of N + jz L(2_60) // to next 3 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3812,7 +3812,7 @@ andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3823,7 +3823,7 @@ negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x2_1(xxx) @@ -3836,7 +3836,7 @@ KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_20_6 + je L(2_20_6) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x2_1(xxx) @@ -3849,12 +3849,12 @@ KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -3862,7 +3862,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3874,16 +3874,16 @@ negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB(xxx) addq $2, BI addq $8, %rax - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): vbroadcastss ALPHA, %xmm0 @@ -3934,13 +3934,13 @@ /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3978,7 +3978,7 @@ andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -3989,7 +3989,7 @@ negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x2_1(xxx) @@ -4002,7 +4002,7 @@ KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x2_1(xxx) @@ -4015,12 +4015,12 @@ KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4028,7 +4028,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4040,16 +4040,16 @@ negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB(xxx) addq $2, BI addq $4, %rax - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): vbroadcastss ALPHA, %xmm0 @@ -4088,13 +4088,13 @@ ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4132,7 +4132,7 @@ andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4143,7 +4143,7 @@ negq %rax ALIGN_4 -.L2_32: +L(2_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -4156,7 +4156,7 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -4169,12 +4169,12 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -4182,7 +4182,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4194,16 +4194,16 @@ negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB(xxx) addq $2, BI addq $2, %rax - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): vmovss ALPHA, %xmm0 @@ -4246,13 +4246,13 @@ addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4288,7 +4288,7 @@ #endif andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4298,7 +4298,7 @@ negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -4310,7 +4310,7 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -4322,12 +4322,12 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -4335,7 +4335,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4346,16 +4346,16 @@ negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) addq $2, BI addq $1, %rax - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vmovss ALPHA, %xmm0 @@ -4395,17 +4395,17 @@ -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -4413,30 +4413,30 @@ movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovss (BO1), %xmm0 vmovss %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -4450,11 +4450,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4490,7 +4490,7 @@ #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -4500,7 +4500,7 @@ negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x1_1(xxx) @@ -4513,7 +4513,7 @@ KERNEL16x1_3(xxx) KERNEL16x1_4(xxx) - je .L1_16 + je L(1_16) KERNEL16x1_1(xxx) KERNEL16x1_2(xxx) @@ -4525,12 +4525,12 @@ KERNEL16x1_3(xxx) KERNEL16x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -4538,7 +4538,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -4549,16 +4549,16 @@ negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB(xxx) addq $1, BI addq $16, %rax - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vbroadcastss ALPHA, %xmm0 @@ -4600,25 +4600,25 @@ addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4655,7 +4655,7 @@ andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -4665,7 +4665,7 @@ negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x1_1(xxx) @@ -4678,7 +4678,7 @@ KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_20_6 + je L(1_20_6) KERNEL8x1_1(xxx) KERNEL8x1_2(xxx) @@ -4690,12 +4690,12 @@ KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -4703,7 +4703,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -4714,16 +4714,16 @@ negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB(xxx) addq $1, BI addq $8, %rax - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): vbroadcastss ALPHA, %xmm0 @@ -4763,13 +4763,13 @@ /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4806,7 +4806,7 @@ andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -4816,7 +4816,7 @@ negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x1_1(xxx) @@ -4829,7 +4829,7 @@ KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) KERNEL4x1_1(xxx) KERNEL4x1_2(xxx) @@ -4841,12 +4841,12 @@ KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4854,7 +4854,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -4865,16 +4865,16 @@ negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB(xxx) addq $1, BI addq $4, %rax - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): vbroadcastss ALPHA, %xmm0 @@ -4909,13 +4909,13 @@ ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4952,7 +4952,7 @@ andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -4962,7 +4962,7 @@ negq %rax ALIGN_4 -.L1_32: +L(1_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -4975,7 +4975,7 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) KERNEL2x1_1(xxx) KERNEL2x1_2(xxx) @@ -4987,12 +4987,12 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -5000,7 +5000,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -5011,16 +5011,16 @@ negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB(xxx) addq $1, BI addq $2, %rax - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): vmovss ALPHA, %xmm0 @@ -5056,13 +5056,13 @@ addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5097,7 +5097,7 @@ #endif andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -5106,7 +5106,7 @@ negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -5118,7 +5118,7 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -5130,12 +5130,12 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -5143,7 +5143,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -5153,16 +5153,16 @@ negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) addq $1, BI addq $1, %rax - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vmovss ALPHA, %xmm0 @@ -5195,7 +5195,7 @@ ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/sgemm_kernel_16x2_piledriver.S b/kernel/x86_64/sgemm_kernel_16x2_piledriver.S index 35b01de07b..32125266e8 100644 --- a/kernel/x86_64/sgemm_kernel_16x2_piledriver.S +++ b/kernel/x86_64/sgemm_kernel_16x2_piledriver.S @@ -1060,13 +1060,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -1086,10 +1086,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv6, J cmpq $0, J - je .L2_0 + je L(2_0) ALIGN_4 -.L6_01: +L(6_01): // copy to sub buffer movq K, %rax salq $1,%rax // K * 2 ; read 2 values @@ -1098,10 +1098,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $3 , %rax // K / 8 - jz .L6_01a_2 + jz L(6_01a_2) ALIGN_4 -.L6_01a_1: +L(6_01a_1): prefetcht0 512(BO1) prefetcht0 512(BO2) @@ -1148,19 +1148,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $12*SIZE,BO decq %rax - jnz .L6_01a_1 + jnz L(6_01a_1) -.L6_01a_2: +L(6_01a_2): movq K, %rax andq $7, %rax // K % 8 - jz .L6_02c + jz L(6_02c) ALIGN_4 -.L6_02b: +L(6_02b): vmovsd 0 * SIZE(BO1), %xmm0 vmovss 0 * SIZE(BO2), %xmm2 @@ -1170,9 +1170,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2*SIZE,BO2 addq $3*SIZE,BO decq %rax - jnz .L6_02b + jnz L(6_02b) -.L6_02c: +L(6_02c): movq K, %rax salq $1,%rax // K * 2 @@ -1181,10 +1181,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. leaq BUFFER2, BO // second buffer to BO movq K, %rax sarq $3 , %rax // K / 8 - jz .L6_02c_2 + jz L(6_02c_2) ALIGN_4 -.L6_02c_1: +L(6_02c_1): prefetcht0 512(BO2) prefetchw 512(BO) @@ -1231,17 +1231,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $12*SIZE,BO decq %rax - jnz .L6_02c_1 + jnz L(6_02c_1) -.L6_02c_2: +L(6_02c_2): movq K, %rax andq $7, %rax // K % 8 - jz .L6_03c + jz L(6_03c) ALIGN_4 -.L6_03b: +L(6_03b): vmovss 1*SIZE(BO1), %xmm0 vmovsd 0*SIZE(BO2), %xmm1 @@ -1251,14 +1251,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2*SIZE,BO2 addq $3*SIZE,BO decq %rax - jnz .L6_03b + jnz L(6_03b) -.L6_03c: +L(6_03c): movq BO2, B // next offset of B -.L6_10: +L(6_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -1269,11 +1269,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L6_20 + je L(6_20) ALIGN_4 -.L6_11: +L(6_11): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1282,7 +1282,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L6_16 + je L(6_16) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1293,7 +1293,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_12: +L(6_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x3_1(xxx) @@ -1307,7 +1307,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x3_3(xxx) KERNEL16x3_4(xxx) - je .L6_16 + je L(6_16) KERNEL16x3_1(xxx) KERNEL16x3_2(xxx) @@ -1320,16 +1320,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x3_3(xxx) KERNEL16x3_4(xxx) - je .L6_16 + je L(6_16) - jmp .L6_12 + jmp L(6_12) ALIGN_4 -.L6_16: +L(6_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_19 + je L(6_19) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1341,16 +1341,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_17: +L(6_17): KERNEL16x3_SUB(xxx) addq $3, BI addq $16, %rax - jl .L6_17 + jl L(6_17) ALIGN_4 -.L6_19: +L(6_19): vbroadcastss ALPHA, %xmm0 @@ -1388,25 +1388,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L6_11 + jg L(6_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L6_20: +L(6_20): // Test rest of M testq $15, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N testq $8, M - jz .L6_21pre + jz L(6_21pre) ALIGN_4 /**************************************************************************/ -.L6_20_1: +L(6_20_1): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1415,7 +1415,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_20_6 + je L(6_20_6) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1426,7 +1426,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_20_2: +L(6_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x3_1(xxx) @@ -1440,7 +1440,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L6_20_6 + je L(6_20_6) KERNEL8x3_1(xxx) KERNEL8x3_2(xxx) @@ -1453,16 +1453,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L6_20_6 + je L(6_20_6) - jmp .L6_20_2 + jmp L(6_20_2) ALIGN_4 -.L6_20_6: +L(6_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_20_9 + je L(6_20_9) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1474,16 +1474,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_20_7: +L(6_20_7): KERNEL8x3_SUB(xxx) addq $3, BI addq $8, %rax - jl .L6_20_7 + jl L(6_20_7) ALIGN_4 -.L6_20_9: +L(6_20_9): vbroadcastss ALPHA, %xmm0 @@ -1514,13 +1514,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L6_21pre: +L(6_21pre): testq $4, M - jz .L6_30 + jz L(6_30) ALIGN_4 -.L6_21: +L(6_21): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1529,7 +1529,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_26 + je L(6_26) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1540,7 +1540,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_22: +L(6_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x3_1(xxx) @@ -1554,7 +1554,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L6_26 + je L(6_26) KERNEL4x3_1(xxx) KERNEL4x3_2(xxx) @@ -1567,16 +1567,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L6_26 + je L(6_26) - jmp .L6_22 + jmp L(6_22) ALIGN_4 -.L6_26: +L(6_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_29 + je L(6_29) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1588,16 +1588,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_27: +L(6_27): KERNEL4x3_SUB(xxx) addq $3, BI addq $4, %rax - jl .L6_27 + jl L(6_27) ALIGN_4 -.L6_29: +L(6_29): vbroadcastss ALPHA, %xmm0 @@ -1614,13 +1614,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L6_30: +L(6_30): testq $2, M - jz .L6_40 + jz L(6_40) ALIGN_4 -.L6_31: +L(6_31): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1629,7 +1629,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_36 + je L(6_36) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1640,7 +1640,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_32: +L(6_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x3_1(xxx) @@ -1654,7 +1654,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L6_36 + je L(6_36) KERNEL2x3_1(xxx) KERNEL2x3_2(xxx) @@ -1667,16 +1667,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L6_36 + je L(6_36) - jmp .L6_32 + jmp L(6_32) ALIGN_4 -.L6_36: +L(6_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_39 + je L(6_39) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1688,16 +1688,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_37: +L(6_37): KERNEL2x3_SUB(xxx) addq $3, BI addq $2, %rax - jl .L6_37 + jl L(6_37) ALIGN_4 -.L6_39: +L(6_39): vmovss ALPHA, %xmm0 @@ -1719,13 +1719,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L6_40: +L(6_40): testq $1, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N ALIGN_4 -.L6_41: +L(6_41): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1734,7 +1734,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_46 + je L(6_46) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1744,7 +1744,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_42: +L(6_42): KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -1756,7 +1756,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L6_46 + je L(6_46) KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -1768,16 +1768,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L6_46 + je L(6_46) - jmp .L6_42 + jmp L(6_42) ALIGN_4 -.L6_46: +L(6_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_49 + je L(6_49) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1788,16 +1788,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_47: +L(6_47): KERNEL1x3_SUB(xxx) addq $3, BI addq $1, %rax - jl .L6_47 + jl L(6_47) ALIGN_4 -.L6_49: +L(6_49): vmovss ALPHA, %xmm0 @@ -1818,7 +1818,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /***************************************************************************************************************/ -.L7_10: +L(7_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -1829,11 +1829,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L7_20 + je L(7_20) ALIGN_4 -.L7_11: +L(7_11): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -1842,7 +1842,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L7_16 + je L(7_16) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1853,7 +1853,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_12: +L(7_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x3_1(xxx) @@ -1867,7 +1867,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x3_3(xxx) KERNEL16x3_4(xxx) - je .L7_16 + je L(7_16) KERNEL16x3_1(xxx) KERNEL16x3_2(xxx) @@ -1880,16 +1880,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x3_3(xxx) KERNEL16x3_4(xxx) - je .L7_16 + je L(7_16) - jmp .L7_12 + jmp L(7_12) ALIGN_4 -.L7_16: +L(7_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_19 + je L(7_19) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1901,16 +1901,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_17: +L(7_17): KERNEL16x3_SUB(xxx) addq $3, BI addq $16, %rax - jl .L7_17 + jl L(7_17) ALIGN_4 -.L7_19: +L(7_19): vbroadcastss ALPHA, %xmm0 @@ -1948,25 +1948,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L7_11 + jg L(7_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L7_20: +L(7_20): // Test rest of M testq $15, M - jz .L7_60 // to next 3 lines of N + jz L(7_60) // to next 3 lines of N testq $8, M - jz .L7_21pre + jz L(7_21pre) ALIGN_4 /**************************************************************************/ -.L7_20_1: +L(7_20_1): leaq BUFFER2, BO // first buffer to BO addq $6 * SIZE, BO @@ -1975,7 +1975,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_20_6 + je L(7_20_6) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1986,7 +1986,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_20_2: +L(7_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x3_1(xxx) @@ -2000,7 +2000,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L7_20_6 + je L(7_20_6) KERNEL8x3_1(xxx) KERNEL8x3_2(xxx) @@ -2013,16 +2013,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L7_20_6 + je L(7_20_6) - jmp .L7_20_2 + jmp L(7_20_2) ALIGN_4 -.L7_20_6: +L(7_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_20_9 + je L(7_20_9) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2034,15 +2034,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_20_7: +L(7_20_7): KERNEL8x3_SUB(xxx) addq $3, BI addq $8, %rax - jl .L7_20_7 + jl L(7_20_7) ALIGN_4 -.L7_20_9: +L(7_20_9): vbroadcastss ALPHA, %xmm0 @@ -2073,13 +2073,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L7_21pre: +L(7_21pre): testq $4, M - jz .L7_30 + jz L(7_30) ALIGN_4 -.L7_21: +L(7_21): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -2088,7 +2088,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_26 + je L(7_26) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2099,7 +2099,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_22: +L(7_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x3_1(xxx) @@ -2113,7 +2113,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L7_26 + je L(7_26) KERNEL4x3_1(xxx) KERNEL4x3_2(xxx) @@ -2126,16 +2126,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L7_26 + je L(7_26) - jmp .L7_22 + jmp L(7_22) ALIGN_4 -.L7_26: +L(7_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_29 + je L(7_29) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2147,16 +2147,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_27: +L(7_27): KERNEL4x3_SUB(xxx) addq $3, BI addq $4, %rax - jl .L7_27 + jl L(7_27) ALIGN_4 -.L7_29: +L(7_29): vbroadcastss ALPHA, %xmm0 @@ -2172,13 +2172,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L7_30: +L(7_30): testq $2, M - jz .L7_40 + jz L(7_40) ALIGN_4 -.L7_31: +L(7_31): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -2187,7 +2187,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_36 + je L(7_36) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2198,7 +2198,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_32: +L(7_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x3_1(xxx) @@ -2212,7 +2212,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L7_36 + je L(7_36) KERNEL2x3_1(xxx) KERNEL2x3_2(xxx) @@ -2225,16 +2225,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L7_36 + je L(7_36) - jmp .L7_32 + jmp L(7_32) ALIGN_4 -.L7_36: +L(7_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_39 + je L(7_39) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2246,16 +2246,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_37: +L(7_37): KERNEL2x3_SUB(xxx) addq $3, BI addq $2, %rax - jl .L7_37 + jl L(7_37) ALIGN_4 -.L7_39: +L(7_39): vmovss ALPHA, %xmm0 @@ -2276,13 +2276,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L7_40: +L(7_40): testq $1, M - jz .L7_60 // to next 3 lines of N + jz L(7_60) // to next 3 lines of N ALIGN_4 -.L7_41: +L(7_41): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -2291,7 +2291,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_46 + je L(7_46) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2301,7 +2301,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_42: +L(7_42): KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -2313,7 +2313,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L7_46 + je L(7_46) KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -2325,16 +2325,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L7_46 + je L(7_46) - jmp .L7_42 + jmp L(7_42) ALIGN_4 -.L7_46: +L(7_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_49 + je L(7_49) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2345,16 +2345,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_47: +L(7_47): KERNEL1x3_SUB(xxx) addq $3, BI addq $1, %rax - jl .L7_47 + jl L(7_47) ALIGN_4 -.L7_49: +L(7_49): vmovss ALPHA, %xmm0 @@ -2371,15 +2371,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L7_60: +L(7_60): decq J // j -- - jg .L6_01 + jg L(6_01) -.L2_0: +L(2_0): cmpq $0, Nmod6 // N % 6 == 0 - je .L999 + je L(999) /************************************************************************************************ * Loop for Nmod6 / 2 > 0 @@ -2387,30 +2387,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J sarq $1, J // j = j / 2 - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -2420,11 +2420,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2433,7 +2433,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2444,7 +2444,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x2_1(xxx) @@ -2457,7 +2457,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_3(xxx) KERNEL16x2_4(xxx) - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x2_1(xxx) @@ -2470,16 +2470,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_3(xxx) KERNEL16x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2491,16 +2491,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB(xxx) addq $2, BI addq $16, %rax - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vbroadcastss ALPHA, %xmm0 @@ -2526,25 +2526,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 3 lines of N + jz L(2_60) // to next 3 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2553,7 +2553,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2564,7 +2564,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x2_1(xxx) @@ -2577,7 +2577,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_20_6 + je L(2_20_6) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x2_1(xxx) @@ -2590,16 +2590,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2611,16 +2611,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB(xxx) addq $2, BI addq $8, %rax - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): vbroadcastss ALPHA, %xmm0 @@ -2644,13 +2644,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2659,7 +2659,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -2670,7 +2670,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x2_1(xxx) @@ -2683,7 +2683,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x2_1(xxx) @@ -2696,16 +2696,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2717,16 +2717,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB(xxx) addq $2, BI addq $4, %rax - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): vbroadcastss ALPHA, %xmm0 @@ -2741,13 +2741,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2756,7 +2756,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2767,7 +2767,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_32: +L(2_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -2780,7 +2780,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -2793,16 +2793,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2814,16 +2814,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB(xxx) addq $2, BI addq $2, %rax - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): vmovss ALPHA, %xmm0 @@ -2840,13 +2840,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2855,7 +2855,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2865,7 +2865,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -2877,7 +2877,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -2889,16 +2889,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2909,16 +2909,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) addq $2, BI addq $1, %rax - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vmovss ALPHA, %xmm0 @@ -2931,14 +2931,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $1 * SIZE, CO1 # coffset += 1 ALIGN_4 -.L2_60: +L(2_60): decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -2946,30 +2946,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovss (BO1), %xmm0 vmovss %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -2979,11 +2979,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -2992,7 +2992,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -3002,7 +3002,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x1_1(xxx) @@ -3015,7 +3015,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_3(xxx) KERNEL16x1_4(xxx) - je .L1_16 + je L(1_16) KERNEL16x1_1(xxx) KERNEL16x1_2(xxx) @@ -3027,16 +3027,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_3(xxx) KERNEL16x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -3047,16 +3047,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB(xxx) addq $1, BI addq $16, %rax - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vbroadcastss ALPHA, %xmm0 @@ -3072,25 +3072,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3099,7 +3099,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -3109,7 +3109,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x1_1(xxx) @@ -3122,7 +3122,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_20_6 + je L(1_20_6) KERNEL8x1_1(xxx) KERNEL8x1_2(xxx) @@ -3134,16 +3134,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -3154,16 +3154,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB(xxx) addq $1, BI addq $8, %rax - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): vbroadcastss ALPHA, %xmm0 @@ -3180,13 +3180,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3195,7 +3195,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -3205,7 +3205,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x1_1(xxx) @@ -3218,7 +3218,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) KERNEL4x1_1(xxx) KERNEL4x1_2(xxx) @@ -3230,16 +3230,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -3250,16 +3250,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB(xxx) addq $1, BI addq $4, %rax - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): vbroadcastss ALPHA, %xmm0 @@ -3271,13 +3271,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3286,7 +3286,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -3296,7 +3296,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_32: +L(1_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -3309,7 +3309,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) KERNEL2x1_1(xxx) KERNEL2x1_2(xxx) @@ -3321,16 +3321,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -3341,16 +3341,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB(xxx) addq $1, BI addq $2, %rax - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): vmovss ALPHA, %xmm0 @@ -3363,13 +3363,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3378,7 +3378,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -3387,7 +3387,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -3399,7 +3399,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -3411,16 +3411,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -3430,16 +3430,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) addq $1, BI addq $1, %rax - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vmovss ALPHA, %xmm0 @@ -3451,7 +3451,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp @@ -3541,13 +3541,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -3576,30 +3576,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv6, J cmpq $0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -3613,11 +3613,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3654,7 +3654,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3665,7 +3665,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x2_1(xxx) @@ -3678,7 +3678,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_3(xxx) KERNEL16x2_4(xxx) - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x2_1(xxx) @@ -3691,12 +3691,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_3(xxx) KERNEL16x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3704,7 +3704,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3716,16 +3716,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB(xxx) addq $2, BI addq $16, %rax - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vbroadcastss ALPHA, %xmm0 @@ -3783,25 +3783,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 3 lines of N + jz L(2_60) // to next 3 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3839,7 +3839,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3850,7 +3850,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x2_1(xxx) @@ -3863,7 +3863,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_20_6 + je L(2_20_6) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x2_1(xxx) @@ -3876,12 +3876,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -3889,7 +3889,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3901,16 +3901,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB(xxx) addq $2, BI addq $8, %rax - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): vbroadcastss ALPHA, %xmm0 @@ -3961,13 +3961,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4005,7 +4005,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -4016,7 +4016,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x2_1(xxx) @@ -4029,7 +4029,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x2_1(xxx) @@ -4042,12 +4042,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4055,7 +4055,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4067,16 +4067,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB(xxx) addq $2, BI addq $4, %rax - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): vbroadcastss ALPHA, %xmm0 @@ -4115,13 +4115,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4159,7 +4159,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4170,7 +4170,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_32: +L(2_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -4183,7 +4183,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -4196,12 +4196,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -4209,7 +4209,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4221,16 +4221,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB(xxx) addq $2, BI addq $2, %rax - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): vmovss ALPHA, %xmm0 @@ -4273,13 +4273,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4315,7 +4315,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4325,7 +4325,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -4337,7 +4337,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -4349,12 +4349,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -4362,7 +4362,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4373,16 +4373,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) addq $2, BI addq $1, %rax - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vmovss ALPHA, %xmm0 @@ -4422,17 +4422,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -4440,30 +4440,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovss (BO1), %xmm0 vmovss %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -4477,11 +4477,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4517,7 +4517,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -4527,7 +4527,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x1_1(xxx) @@ -4540,7 +4540,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_3(xxx) KERNEL16x1_4(xxx) - je .L1_16 + je L(1_16) KERNEL16x1_1(xxx) KERNEL16x1_2(xxx) @@ -4552,12 +4552,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_3(xxx) KERNEL16x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -4565,7 +4565,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -4576,16 +4576,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB(xxx) addq $1, BI addq $16, %rax - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vbroadcastss ALPHA, %xmm0 @@ -4627,25 +4627,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4682,7 +4682,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -4692,7 +4692,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x1_1(xxx) @@ -4705,7 +4705,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_20_6 + je L(1_20_6) KERNEL8x1_1(xxx) KERNEL8x1_2(xxx) @@ -4717,12 +4717,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -4730,7 +4730,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -4741,16 +4741,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB(xxx) addq $1, BI addq $8, %rax - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): vbroadcastss ALPHA, %xmm0 @@ -4790,13 +4790,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4833,7 +4833,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -4843,7 +4843,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x1_1(xxx) @@ -4856,7 +4856,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) KERNEL4x1_1(xxx) KERNEL4x1_2(xxx) @@ -4868,12 +4868,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4881,7 +4881,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -4892,16 +4892,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB(xxx) addq $1, BI addq $4, %rax - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): vbroadcastss ALPHA, %xmm0 @@ -4936,13 +4936,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4979,7 +4979,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -4989,7 +4989,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_32: +L(1_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -5002,7 +5002,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) KERNEL2x1_1(xxx) KERNEL2x1_2(xxx) @@ -5014,12 +5014,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -5027,7 +5027,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -5038,16 +5038,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB(xxx) addq $1, BI addq $2, %rax - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): vmovss ALPHA, %xmm0 @@ -5083,13 +5083,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5124,7 +5124,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -5133,7 +5133,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -5145,7 +5145,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -5157,12 +5157,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -5170,7 +5170,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -5180,16 +5180,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) addq $1, BI addq $1, %rax - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vmovss ALPHA, %xmm0 @@ -5222,7 +5222,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/sgemm_kernel_16x4_haswell.S b/kernel/x86_64/sgemm_kernel_16x4_haswell.S index 76ea12fee7..e6579f11c2 100644 --- a/kernel/x86_64/sgemm_kernel_16x4_haswell.S +++ b/kernel/x86_64/sgemm_kernel_16x4_haswell.S @@ -1191,13 +1191,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -1216,13 +1216,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv6, J cmpq $0, J - je .L4_00 + je L(4_00) ALIGN_4 /*******************************************************************************************/ -.L6_01: +L(6_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO @@ -1235,7 +1235,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L6_02c: +L(6_02c): vmovups (BO1), %xmm0 vmovsd (BO2), %xmm1 @@ -1245,10 +1245,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4*SIZE,BO2 addq $ 6*SIZE,BO decq %rax - jnz .L6_02c + jnz L(6_02c) -.L6_10: +L(6_10): movq C, CO1 leaq (C, LDC, 2), CO2 leaq (CO2, LDC, 1), CO2 // co2 = c + 3 * ldc @@ -1260,11 +1260,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L6_20 + je L(6_20) ALIGN_4 -.L6_11: +L(6_11): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1273,11 +1273,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L6_16 + je L(6_16) ALIGN_4 -.L6_12: +L(6_12): KERNEL16x6_SUB KERNEL16x6_SUB @@ -1289,7 +1289,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x6_SUB KERNEL16x6_SUB - je .L6_16 + je L(6_16) KERNEL16x6_SUB KERNEL16x6_SUB @@ -1301,53 +1301,53 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x6_SUB KERNEL16x6_SUB - je .L6_16 + je L(6_16) - jmp .L6_12 + jmp L(6_12) ALIGN_4 -.L6_16: +L(6_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_19 + je L(6_19) ALIGN_4 -.L6_17: +L(6_17): KERNEL16x6_SUB - jnz .L6_17 + jnz L(6_17) ALIGN_4 -.L6_19: +L(6_19): SAVE16x6 addq $16 * SIZE, CO1 # coffset += 16 addq $16 * SIZE, CO2 # coffset += 16 decq I # i -- - jg .L6_11 + jg L(6_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L6_20: +L(6_20): // Test rest of M testq $15, M - jz .L6_60 // to next 6 lines of N + jz L(6_60) // to next 6 lines of N testq $8, M - jz .L6_21pre + jz L(6_21pre) ALIGN_4 /**************************************************************************/ -.L6_20_1: +L(6_20_1): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1356,11 +1356,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_20_6 + je L(6_20_6) ALIGN_4 -.L6_20_2: +L(6_20_2): prefetcht0 A_PR1(AO) KERNEL8x6_SUB @@ -1376,7 +1376,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x6_SUB KERNEL8x6_SUB - je .L6_20_6 + je L(6_20_6) prefetcht0 A_PR1(AO) KERNEL8x6_SUB @@ -1392,28 +1392,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x6_SUB KERNEL8x6_SUB - je .L6_20_6 + je L(6_20_6) - jmp .L6_20_2 + jmp L(6_20_2) ALIGN_4 -.L6_20_6: +L(6_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_20_9 + je L(6_20_9) ALIGN_4 -.L6_20_7: +L(6_20_7): KERNEL8x6_SUB - jnz .L6_20_7 + jnz L(6_20_7) ALIGN_4 -.L6_20_9: +L(6_20_9): SAVE8x6 @@ -1425,13 +1425,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L6_21pre: +L(6_21pre): testq $4, M - jz .L6_30 + jz L(6_30) ALIGN_4 -.L6_21: +L(6_21): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1440,11 +1440,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_26 + je L(6_26) ALIGN_4 -.L6_22: +L(6_22): prefetcht0 A_PR1(AO) KERNEL4x6_SUB @@ -1458,7 +1458,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x6_SUB KERNEL4x6_SUB - je .L6_26 + je L(6_26) prefetcht0 A_PR1(AO) KERNEL4x6_SUB @@ -1472,28 +1472,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x6_SUB KERNEL4x6_SUB - je .L6_26 + je L(6_26) - jmp .L6_22 + jmp L(6_22) ALIGN_4 -.L6_26: +L(6_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_29 + je L(6_29) ALIGN_4 -.L6_27: +L(6_27): KERNEL4x6_SUB - jnz .L6_27 + jnz L(6_27) ALIGN_4 -.L6_29: +L(6_29): SAVE4x6 @@ -1502,13 +1502,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L6_30: +L(6_30): testq $2, M - jz .L6_40 + jz L(6_40) ALIGN_4 -.L6_31: +L(6_31): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1517,11 +1517,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_36 + je L(6_36) ALIGN_4 -.L6_32: +L(6_32): prefetcht0 A_PR1(AO) KERNEL2x6_SUB @@ -1534,7 +1534,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x6_SUB KERNEL2x6_SUB - je .L6_36 + je L(6_36) prefetcht0 A_PR1(AO) KERNEL2x6_SUB @@ -1547,28 +1547,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x6_SUB KERNEL2x6_SUB - je .L6_36 + je L(6_36) - jmp .L6_32 + jmp L(6_32) ALIGN_4 -.L6_36: +L(6_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_39 + je L(6_39) ALIGN_4 -.L6_37: +L(6_37): KERNEL2x6_SUB - jnz .L6_37 + jnz L(6_37) ALIGN_4 -.L6_39: +L(6_39): SAVE2x6 @@ -1576,13 +1576,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO2 # coffset += 2 ALIGN_4 -.L6_40: +L(6_40): testq $1, M - jz .L6_60 // to next 4 lines of N + jz L(6_60) // to next 4 lines of N ALIGN_4 -.L6_41: +L(6_41): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1591,11 +1591,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_46 + je L(6_46) ALIGN_4 -.L6_42: +L(6_42): prefetcht0 A_PR1(AO) KERNEL1x6_SUB @@ -1608,7 +1608,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x6_SUB KERNEL1x6_SUB - je .L6_46 + je L(6_46) KERNEL1x6_SUB KERNEL1x6_SUB @@ -1620,28 +1620,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x6_SUB KERNEL1x6_SUB - je .L6_46 + je L(6_46) - jmp .L6_42 + jmp L(6_42) ALIGN_4 -.L6_46: +L(6_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_49 + je L(6_49) ALIGN_4 -.L6_47: +L(6_47): KERNEL1x6_SUB - jnz .L6_47 + jnz L(6_47) ALIGN_4 -.L6_49: +L(6_49): SAVE1x6 @@ -1653,13 +1653,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L6_60: +L(6_60): /*******************************************************************************************/ -.L7_01: +L(7_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO @@ -1671,7 +1671,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L7_02c: +L(7_02c): vmovsd 2*SIZE(BO1), %xmm0 vmovups (BO2), %xmm1 @@ -1681,11 +1681,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4*SIZE,BO2 addq $ 6*SIZE,BO decq %rax - jnz .L7_02c + jnz L(7_02c) movq BO2, B // next offset of B -.L7_10: +L(7_10): movq C, CO1 leaq (C, LDC, 2), CO2 leaq (CO2, LDC, 1), CO2 // co2 = c + 3 * ldc @@ -1697,11 +1697,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L7_20 + je L(7_20) ALIGN_4 -.L7_11: +L(7_11): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1710,11 +1710,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L7_16 + je L(7_16) ALIGN_4 -.L7_12: +L(7_12): KERNEL16x6_SUB KERNEL16x6_SUB @@ -1726,7 +1726,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x6_SUB KERNEL16x6_SUB - je .L7_16 + je L(7_16) KERNEL16x6_SUB KERNEL16x6_SUB @@ -1738,53 +1738,53 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x6_SUB KERNEL16x6_SUB - je .L7_16 + je L(7_16) - jmp .L7_12 + jmp L(7_12) ALIGN_4 -.L7_16: +L(7_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_19 + je L(7_19) ALIGN_4 -.L7_17: +L(7_17): KERNEL16x6_SUB - jnz .L7_17 + jnz L(7_17) ALIGN_4 -.L7_19: +L(7_19): SAVE16x6 addq $16 * SIZE, CO1 # coffset += 16 addq $16 * SIZE, CO2 # coffset += 16 decq I # i -- - jg .L7_11 + jg L(7_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L7_20: +L(7_20): // Test rest of M testq $15, M - jz .L7_60 // to next 6 lines of N + jz L(7_60) // to next 6 lines of N testq $8, M - jz .L7_21pre + jz L(7_21pre) ALIGN_4 /**************************************************************************/ -.L7_20_1: +L(7_20_1): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1793,11 +1793,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_20_6 + je L(7_20_6) ALIGN_4 -.L7_20_2: +L(7_20_2): prefetcht0 A_PR1(AO) KERNEL8x6_SUB @@ -1813,7 +1813,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x6_SUB KERNEL8x6_SUB - je .L7_20_6 + je L(7_20_6) prefetcht0 A_PR1(AO) KERNEL8x6_SUB @@ -1829,28 +1829,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x6_SUB KERNEL8x6_SUB - je .L7_20_6 + je L(7_20_6) - jmp .L7_20_2 + jmp L(7_20_2) ALIGN_4 -.L7_20_6: +L(7_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_20_9 + je L(7_20_9) ALIGN_4 -.L7_20_7: +L(7_20_7): KERNEL8x6_SUB - jnz .L7_20_7 + jnz L(7_20_7) ALIGN_4 -.L7_20_9: +L(7_20_9): SAVE8x6 @@ -1862,13 +1862,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L7_21pre: +L(7_21pre): testq $4, M - jz .L7_30 + jz L(7_30) ALIGN_4 -.L7_21: +L(7_21): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1877,11 +1877,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_26 + je L(7_26) ALIGN_4 -.L7_22: +L(7_22): prefetcht0 A_PR1(AO) KERNEL4x6_SUB @@ -1895,7 +1895,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x6_SUB KERNEL4x6_SUB - je .L7_26 + je L(7_26) prefetcht0 A_PR1(AO) KERNEL4x6_SUB @@ -1909,28 +1909,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x6_SUB KERNEL4x6_SUB - je .L7_26 + je L(7_26) - jmp .L7_22 + jmp L(7_22) ALIGN_4 -.L7_26: +L(7_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_29 + je L(7_29) ALIGN_4 -.L7_27: +L(7_27): KERNEL4x6_SUB - jnz .L7_27 + jnz L(7_27) ALIGN_4 -.L7_29: +L(7_29): SAVE4x6 @@ -1939,13 +1939,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L7_30: +L(7_30): testq $2, M - jz .L7_40 + jz L(7_40) ALIGN_4 -.L7_31: +L(7_31): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1954,11 +1954,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_36 + je L(7_36) ALIGN_4 -.L7_32: +L(7_32): prefetcht0 A_PR1(AO) KERNEL2x6_SUB @@ -1971,7 +1971,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x6_SUB KERNEL2x6_SUB - je .L7_36 + je L(7_36) prefetcht0 A_PR1(AO) KERNEL2x6_SUB @@ -1984,28 +1984,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x6_SUB KERNEL2x6_SUB - je .L7_36 + je L(7_36) - jmp .L7_32 + jmp L(7_32) ALIGN_4 -.L7_36: +L(7_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_39 + je L(7_39) ALIGN_4 -.L7_37: +L(7_37): KERNEL2x6_SUB - jnz .L7_37 + jnz L(7_37) ALIGN_4 -.L7_39: +L(7_39): SAVE2x6 @@ -2013,13 +2013,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO2 # coffset += 2 ALIGN_4 -.L7_40: +L(7_40): testq $1, M - jz .L7_60 // to next 4 lines of N + jz L(7_60) // to next 4 lines of N ALIGN_4 -.L7_41: +L(7_41): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2028,11 +2028,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_46 + je L(7_46) ALIGN_4 -.L7_42: +L(7_42): prefetcht0 A_PR1(AO) KERNEL1x6_SUB @@ -2045,7 +2045,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x6_SUB KERNEL1x6_SUB - je .L7_46 + je L(7_46) KERNEL1x6_SUB KERNEL1x6_SUB @@ -2057,28 +2057,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x6_SUB KERNEL1x6_SUB - je .L7_46 + je L(7_46) - jmp .L7_42 + jmp L(7_42) ALIGN_4 -.L7_46: +L(7_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_49 + je L(7_49) ALIGN_4 -.L7_47: +L(7_47): KERNEL1x6_SUB - jnz .L7_47 + jnz L(7_47) ALIGN_4 -.L7_49: +L(7_49): SAVE1x6 @@ -2090,35 +2090,35 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L7_60: +L(7_60): decq J // j -- - jg .L6_01 // next 12 lines of N + jg L(6_01) // next 12 lines of N /*******************************************************************************************/ -.L4_00: +L(4_00): movq Nmod6, J sarq $2, J // j = j / 4 cmpq $ 0, J - je .L2_00 + je L(2_00) ALIGN_4 -.L4_01: +L(4_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L4_01b + jz L(4_01b) ALIGN_4 -.L4_01a: +L(4_01a): prefetcht0 512(BO1) prefetchw 512(BO) @@ -2135,30 +2135,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 16*SIZE,BO1 addq $ 16*SIZE,BO decq %rax - jnz .L4_01a + jnz L(4_01a) -.L4_01b: +L(4_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L4_02d + jz L(4_02d) ALIGN_4 -.L4_02c: +L(4_02c): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $ 4*SIZE,BO1 addq $ 4*SIZE,BO decq %rax - jnz .L4_02c + jnz L(4_02c) -.L4_02d: +L(4_02d): movq BO1, B // next offset of B -.L4_10: +L(4_10): movq C, CO1 leaq (C, LDC, 2), CO2 leaq (C, LDC, 4), C // c += 4 * ldc @@ -2173,11 +2173,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L4_20 + je L(4_20) ALIGN_4 -.L4_11: +L(4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2214,7 +2214,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L4_16 + je L(4_16) movq %rax, BI // Index for BO leaq (,BI,4) , BI // BI = BI * 4 ; number of values @@ -2225,7 +2225,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_12: +L(4_12): prefetcht0 A_PR1(AO, %rax, SIZE) prefetcht0 B_PR1(BO, BI , SIZE) @@ -2247,7 +2247,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO, %rax, SIZE) KERNEL16x4_SUB - je .L4_16 + je L(4_16) prefetcht0 A_PR1(AO, %rax, SIZE) prefetcht0 B_PR1(BO, BI , SIZE) @@ -2269,12 +2269,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO, %rax, SIZE) KERNEL16x4_SUB - je .L4_16 + je L(4_16) - jmp .L4_12 + jmp L(4_12) ALIGN_4 -.L4_16: +L(4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -2282,7 +2282,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_19 + je L(4_19) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2294,15 +2294,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_17: +L(4_17): KERNEL16x4_SUB - jl .L4_17 + jl L(4_17) ALIGN_4 -.L4_19: +L(4_19): SAVE16x4 @@ -2325,25 +2325,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 addq $16 * SIZE, CO2 # coffset += 16 decq I # i -- - jg .L4_11 + jg L(4_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L4_20: +L(4_20): // Test rest of M testq $15, M - jz .L4_60 // to next 3 lines of N + jz L(4_60) // to next 3 lines of N testq $8, M - jz .L4_21pre + jz L(4_21pre) ALIGN_4 /**************************************************************************/ -.L4_20_1: +L(4_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2381,7 +2381,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_20_6 + je L(4_20_6) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2392,7 +2392,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_20_2: +L(4_20_2): KERNEL8x4_SUB KERNEL8x4_SUB @@ -2404,7 +2404,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_SUB KERNEL8x4_SUB - je .L4_20_6 + je L(4_20_6) KERNEL8x4_SUB KERNEL8x4_SUB @@ -2416,12 +2416,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_SUB KERNEL8x4_SUB - je .L4_20_6 + je L(4_20_6) - jmp .L4_20_2 + jmp L(4_20_2) ALIGN_4 -.L4_20_6: +L(4_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -2429,7 +2429,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_20_9 + je L(4_20_9) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2441,15 +2441,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_20_7: +L(4_20_7): KERNEL8x4_SUB - jl .L4_20_7 + jl L(4_20_7) ALIGN_4 -.L4_20_9: +L(4_20_9): SAVE8x4 @@ -2477,13 +2477,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L4_21pre: +L(4_21pre): testq $4, M - jz .L4_30 + jz L(4_30) ALIGN_4 -.L4_21: +L(4_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2521,7 +2521,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_26 + je L(4_26) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2532,7 +2532,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_22: +L(4_22): KERNEL4x4_SUB KERNEL4x4_SUB @@ -2544,7 +2544,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB KERNEL4x4_SUB - je .L4_26 + je L(4_26) KERNEL4x4_SUB KERNEL4x4_SUB @@ -2556,12 +2556,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB KERNEL4x4_SUB - je .L4_26 + je L(4_26) - jmp .L4_22 + jmp L(4_22) ALIGN_4 -.L4_26: +L(4_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -2569,7 +2569,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_29 + je L(4_29) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2581,15 +2581,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_27: +L(4_27): KERNEL4x4_SUB - jl .L4_27 + jl L(4_27) ALIGN_4 -.L4_29: +L(4_29): SAVE4x4 @@ -2614,13 +2614,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L4_30: +L(4_30): testq $2, M - jz .L4_40 + jz L(4_40) ALIGN_4 -.L4_31: +L(4_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2658,7 +2658,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_36 + je L(4_36) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2669,7 +2669,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_32: +L(4_32): KERNEL2x4_SUB KERNEL2x4_SUB @@ -2681,7 +2681,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB KERNEL2x4_SUB - je .L4_36 + je L(4_36) KERNEL2x4_SUB KERNEL2x4_SUB @@ -2693,12 +2693,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB KERNEL2x4_SUB - je .L4_36 + je L(4_36) - jmp .L4_32 + jmp L(4_32) ALIGN_4 -.L4_36: +L(4_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -2706,7 +2706,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_39 + je L(4_39) movq %rax, BI // Index for BO leaq (,BI, 4), BI // BI = BI * 4 ; number of values @@ -2718,15 +2718,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_37: +L(4_37): KERNEL2x4_SUB - jl .L4_37 + jl L(4_37) ALIGN_4 -.L4_39: +L(4_39): SAVE2x4 @@ -2750,13 +2750,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO2 # coffset += 2 ALIGN_4 -.L4_40: +L(4_40): testq $1, M - jz .L4_60 // to next 4 lines of N + jz L(4_60) // to next 4 lines of N ALIGN_4 -.L4_41: +L(4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2792,7 +2792,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L4_46 + je L(4_46) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2802,7 +2802,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_42: +L(4_42): KERNEL1x4_SUB KERNEL1x4_SUB @@ -2814,7 +2814,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB KERNEL1x4_SUB - je .L4_46 + je L(4_46) KERNEL1x4_SUB KERNEL1x4_SUB @@ -2826,12 +2826,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB KERNEL1x4_SUB - je .L4_46 + je L(4_46) - jmp .L4_42 + jmp L(4_42) ALIGN_4 -.L4_46: +L(4_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -2839,7 +2839,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_49 + je L(4_49) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2850,15 +2850,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_47: +L(4_47): KERNEL1x4_SUB - jl .L4_47 + jl L(4_47) ALIGN_4 -.L4_49: +L(4_49): SAVE1x4 @@ -2885,38 +2885,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L4_60: +L(4_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif decq J // j -- - jg .L4_01 // next 4 lines of N + jg L(4_01) // next 4 lines of N /*******************************************************************************************/ -.L2_00: +L(2_00): movq Nmod6, J andq $3, J // j % 4 - je .L999 + je L(999) movq Nmod6, J andq $2, J // j % 4 - je .L1_0 + je L(1_0) -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L2_01b + jz L(2_01b) ALIGN_4 -.L2_01a: +L(2_01a): vmovsd (BO1), %xmm0 vmovsd 2*SIZE(BO1), %xmm1 @@ -2931,30 +2931,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $8*SIZE,BO1 addq $8*SIZE,BO decq %rax - jnz .L2_01a + jnz L(2_01a) -.L2_01b: +L(2_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L2_02d + jz L(2_02d) ALIGN_4 -.L2_02c: +L(2_02c): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02c + jnz L(2_02c) -.L2_02d: +L(2_02d): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -2968,11 +2968,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3009,7 +3009,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3020,7 +3020,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_12: +L(2_12): KERNEL16x2_SUB KERNEL16x2_SUB @@ -3032,7 +3032,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB KERNEL16x2_SUB - je .L2_16 + je L(2_16) KERNEL16x2_SUB KERNEL16x2_SUB @@ -3044,12 +3044,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB KERNEL16x2_SUB - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3057,7 +3057,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3069,15 +3069,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE16x2 @@ -3099,25 +3099,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3155,7 +3155,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3166,7 +3166,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): KERNEL8x2_SUB @@ -3179,7 +3179,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB KERNEL8x2_SUB - je .L2_20_6 + je L(2_20_6) KERNEL8x2_SUB KERNEL8x2_SUB @@ -3191,12 +3191,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB KERNEL8x2_SUB - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -3204,7 +3204,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3216,15 +3216,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): SAVE8x2 @@ -3251,13 +3251,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3295,7 +3295,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -3306,7 +3306,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_22: +L(2_22): KERNEL4x2_SUB @@ -3319,7 +3319,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_26 + je L(2_26) KERNEL4x2_SUB KERNEL4x2_SUB @@ -3331,12 +3331,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -3344,7 +3344,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3356,15 +3356,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): SAVE4x2 @@ -3388,13 +3388,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3432,7 +3432,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3443,7 +3443,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_SUB KERNEL2x2_SUB @@ -3455,7 +3455,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_36 + je L(2_36) KERNEL2x2_SUB KERNEL2x2_SUB @@ -3467,12 +3467,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -3480,7 +3480,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3492,15 +3492,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): SAVE2x2 @@ -3523,13 +3523,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3565,7 +3565,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3575,7 +3575,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_SUB KERNEL1x2_SUB @@ -3587,7 +3587,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_46 + je L(2_46) KERNEL1x2_SUB KERNEL1x2_SUB @@ -3599,12 +3599,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -3612,7 +3612,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3623,15 +3623,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): SAVE1x2 @@ -3657,7 +3657,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -3665,7 +3665,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -3673,30 +3673,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovss (BO1), %xmm0 vmovss %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -3710,11 +3710,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3750,7 +3750,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -3760,7 +3760,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_12: +L(1_12): KERNEL16x1_SUB KERNEL16x1_SUB @@ -3772,7 +3772,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB KERNEL16x1_SUB - je .L1_16 + je L(1_16) KERNEL16x1_SUB KERNEL16x1_SUB @@ -3784,12 +3784,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB KERNEL16x1_SUB - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3797,7 +3797,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -3808,15 +3808,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE16x1 @@ -3837,25 +3837,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3892,7 +3892,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -3902,7 +3902,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): KERNEL8x1_SUB KERNEL8x1_SUB @@ -3914,7 +3914,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB KERNEL8x1_SUB - je .L1_20_6 + je L(1_20_6) KERNEL8x1_SUB KERNEL8x1_SUB @@ -3926,12 +3926,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB KERNEL8x1_SUB - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -3939,7 +3939,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -3950,15 +3950,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): SAVE8x1 @@ -3984,13 +3984,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4027,7 +4027,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -4037,7 +4037,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_22: +L(1_22): KERNEL4x1_SUB KERNEL4x1_SUB @@ -4049,7 +4049,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_26 + je L(1_26) KERNEL4x1_SUB KERNEL4x1_SUB @@ -4061,12 +4061,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4074,7 +4074,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -4085,15 +4085,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): SAVE4x1 @@ -4116,13 +4116,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4159,7 +4159,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -4169,7 +4169,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_SUB KERNEL2x1_SUB @@ -4181,7 +4181,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_36 + je L(1_36) KERNEL2x1_SUB KERNEL2x1_SUB @@ -4193,12 +4193,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -4206,7 +4206,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -4217,15 +4217,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): SAVE2x1 @@ -4247,13 +4247,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4288,7 +4288,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -4297,7 +4297,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_SUB KERNEL1x1_SUB @@ -4309,7 +4309,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_46 + je L(1_46) KERNEL1x1_SUB KERNEL1x1_SUB @@ -4321,12 +4321,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -4334,7 +4334,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -4344,15 +4344,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): SAVE1x1 @@ -4374,7 +4374,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp @@ -4466,13 +4466,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -4501,22 +4501,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv6, J cmpq $0, J - je .L2_0 + je L(2_0) ALIGN_4 /*******************************************************************************************/ -.L4_01: +L(4_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L4_01b + jz L(4_01b) ALIGN_4 -.L4_01a: +L(4_01a): prefetcht0 512(BO1) prefetchw 512(BO) @@ -4533,30 +4533,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 16*SIZE,BO1 addq $ 16*SIZE,BO decq %rax - jnz .L4_01a + jnz L(4_01a) -.L4_01b: +L(4_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L4_02d + jz L(4_02d) ALIGN_4 -.L4_02c: +L(4_02c): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $ 4*SIZE,BO1 addq $ 4*SIZE,BO decq %rax - jnz .L4_02c + jnz L(4_02c) -.L4_02d: +L(4_02d): movq BO1, B // next offset of B -.L4_10: +L(4_10): movq C, CO1 leaq (C, LDC, 2), CO2 leaq (C, LDC, 4), C // c += 4 * ldc @@ -4571,11 +4571,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L4_20 + je L(4_20) ALIGN_4 -.L4_11: +L(4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4612,7 +4612,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L4_16 + je L(4_16) movq %rax, BI // Index for BO leaq (,BI,4) , BI // BI = BI * 4 ; number of values @@ -4623,7 +4623,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_12: +L(4_12): prefetcht0 A_PR1(AO, %rax, SIZE) prefetcht0 B_PR1(BO, BI , SIZE) @@ -4645,7 +4645,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO, %rax, SIZE) KERNEL16x4_SUB - je .L4_16 + je L(4_16) prefetcht0 A_PR1(AO, %rax, SIZE) prefetcht0 B_PR1(BO, BI , SIZE) @@ -4667,12 +4667,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO, %rax, SIZE) KERNEL16x4_SUB - je .L4_16 + je L(4_16) - jmp .L4_12 + jmp L(4_12) ALIGN_4 -.L4_16: +L(4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -4680,7 +4680,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_19 + je L(4_19) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -4692,15 +4692,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_17: +L(4_17): KERNEL16x4_SUB - jl .L4_17 + jl L(4_17) ALIGN_4 -.L4_19: +L(4_19): SAVE16x4 @@ -4723,25 +4723,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 addq $16 * SIZE, CO2 # coffset += 16 decq I # i -- - jg .L4_11 + jg L(4_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L4_20: +L(4_20): // Test rest of M testq $15, M - jz .L4_60 // to next 3 lines of N + jz L(4_60) // to next 3 lines of N testq $8, M - jz .L4_21pre + jz L(4_21pre) ALIGN_4 /**************************************************************************/ -.L4_20_1: +L(4_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4779,7 +4779,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_20_6 + je L(4_20_6) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -4790,7 +4790,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_20_2: +L(4_20_2): KERNEL8x4_SUB KERNEL8x4_SUB @@ -4802,7 +4802,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_SUB KERNEL8x4_SUB - je .L4_20_6 + je L(4_20_6) KERNEL8x4_SUB KERNEL8x4_SUB @@ -4814,12 +4814,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_SUB KERNEL8x4_SUB - je .L4_20_6 + je L(4_20_6) - jmp .L4_20_2 + jmp L(4_20_2) ALIGN_4 -.L4_20_6: +L(4_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -4827,7 +4827,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_20_9 + je L(4_20_9) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -4839,15 +4839,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_20_7: +L(4_20_7): KERNEL8x4_SUB - jl .L4_20_7 + jl L(4_20_7) ALIGN_4 -.L4_20_9: +L(4_20_9): SAVE8x4 @@ -4875,13 +4875,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L4_21pre: +L(4_21pre): testq $4, M - jz .L4_30 + jz L(4_30) ALIGN_4 -.L4_21: +L(4_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4919,7 +4919,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_26 + je L(4_26) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -4930,7 +4930,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_22: +L(4_22): KERNEL4x4_SUB KERNEL4x4_SUB @@ -4942,7 +4942,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB KERNEL4x4_SUB - je .L4_26 + je L(4_26) KERNEL4x4_SUB KERNEL4x4_SUB @@ -4954,12 +4954,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB KERNEL4x4_SUB - je .L4_26 + je L(4_26) - jmp .L4_22 + jmp L(4_22) ALIGN_4 -.L4_26: +L(4_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4967,7 +4967,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_29 + je L(4_29) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -4979,15 +4979,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_27: +L(4_27): KERNEL4x4_SUB - jl .L4_27 + jl L(4_27) ALIGN_4 -.L4_29: +L(4_29): SAVE4x4 @@ -5012,13 +5012,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L4_30: +L(4_30): testq $2, M - jz .L4_40 + jz L(4_40) ALIGN_4 -.L4_31: +L(4_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5056,7 +5056,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_36 + je L(4_36) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -5067,7 +5067,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_32: +L(4_32): KERNEL2x4_SUB KERNEL2x4_SUB @@ -5079,7 +5079,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB KERNEL2x4_SUB - je .L4_36 + je L(4_36) KERNEL2x4_SUB KERNEL2x4_SUB @@ -5091,12 +5091,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB KERNEL2x4_SUB - je .L4_36 + je L(4_36) - jmp .L4_32 + jmp L(4_32) ALIGN_4 -.L4_36: +L(4_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -5104,7 +5104,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_39 + je L(4_39) movq %rax, BI // Index for BO leaq (,BI, 4), BI // BI = BI * 4 ; number of values @@ -5116,15 +5116,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_37: +L(4_37): KERNEL2x4_SUB - jl .L4_37 + jl L(4_37) ALIGN_4 -.L4_39: +L(4_39): SAVE2x4 @@ -5148,13 +5148,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO2 # coffset += 2 ALIGN_4 -.L4_40: +L(4_40): testq $1, M - jz .L4_60 // to next 4 lines of N + jz L(4_60) // to next 4 lines of N ALIGN_4 -.L4_41: +L(4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5190,7 +5190,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L4_46 + je L(4_46) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -5200,7 +5200,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_42: +L(4_42): KERNEL1x4_SUB KERNEL1x4_SUB @@ -5212,7 +5212,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB KERNEL1x4_SUB - je .L4_46 + je L(4_46) KERNEL1x4_SUB KERNEL1x4_SUB @@ -5224,12 +5224,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB KERNEL1x4_SUB - je .L4_46 + je L(4_46) - jmp .L4_42 + jmp L(4_42) ALIGN_4 -.L4_46: +L(4_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -5237,7 +5237,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_49 + je L(4_49) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -5248,15 +5248,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_47: +L(4_47): KERNEL1x4_SUB - jl .L4_47 + jl L(4_47) ALIGN_4 -.L4_49: +L(4_49): SAVE1x4 @@ -5283,38 +5283,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L4_60: +L(4_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif decq J // j -- - jg .L4_01 // next 4 lines of N + jg L(4_01) // next 4 lines of N /*******************************************************************************************/ -.L2_0: +L(2_0): movq Nmod6, J andq $3, J // j % 4 - je .L999 + je L(999) movq Nmod6, J andq $2, J // j % 4 - je .L1_0 + je L(1_0) -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L2_01b + jz L(2_01b) ALIGN_4 -.L2_01a: +L(2_01a): vmovsd (BO1), %xmm0 vmovsd 2*SIZE(BO1), %xmm1 @@ -5329,30 +5329,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $8*SIZE,BO1 addq $8*SIZE,BO decq %rax - jnz .L2_01a + jnz L(2_01a) -.L2_01b: +L(2_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L2_02d + jz L(2_02d) ALIGN_4 -.L2_02c: +L(2_02c): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02c + jnz L(2_02c) -.L2_02d: +L(2_02d): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -5366,11 +5366,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5407,7 +5407,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5418,7 +5418,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_12: +L(2_12): KERNEL16x2_SUB KERNEL16x2_SUB @@ -5430,7 +5430,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB KERNEL16x2_SUB - je .L2_16 + je L(2_16) KERNEL16x2_SUB KERNEL16x2_SUB @@ -5442,12 +5442,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB KERNEL16x2_SUB - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -5455,7 +5455,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5467,15 +5467,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE16x2 @@ -5497,25 +5497,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5553,7 +5553,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5564,7 +5564,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): KERNEL8x2_SUB @@ -5577,7 +5577,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB KERNEL8x2_SUB - je .L2_20_6 + je L(2_20_6) KERNEL8x2_SUB KERNEL8x2_SUB @@ -5589,12 +5589,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB KERNEL8x2_SUB - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -5602,7 +5602,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5614,15 +5614,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): SAVE8x2 @@ -5649,13 +5649,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5693,7 +5693,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -5704,7 +5704,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_22: +L(2_22): KERNEL4x2_SUB @@ -5717,7 +5717,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_26 + je L(2_26) KERNEL4x2_SUB KERNEL4x2_SUB @@ -5729,12 +5729,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -5742,7 +5742,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5754,15 +5754,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): SAVE4x2 @@ -5786,13 +5786,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5830,7 +5830,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5841,7 +5841,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_SUB KERNEL2x2_SUB @@ -5853,7 +5853,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_36 + je L(2_36) KERNEL2x2_SUB KERNEL2x2_SUB @@ -5865,12 +5865,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -5878,7 +5878,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5890,15 +5890,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): SAVE2x2 @@ -5921,13 +5921,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5963,7 +5963,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5973,7 +5973,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_SUB KERNEL1x2_SUB @@ -5985,7 +5985,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_46 + je L(2_46) KERNEL1x2_SUB KERNEL1x2_SUB @@ -5997,12 +5997,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -6010,7 +6010,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -6021,15 +6021,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): SAVE1x2 @@ -6055,7 +6055,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -6063,7 +6063,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -6071,30 +6071,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovss (BO1), %xmm0 vmovss %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -6108,11 +6108,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -6148,7 +6148,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -6158,7 +6158,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_12: +L(1_12): KERNEL16x1_SUB KERNEL16x1_SUB @@ -6170,7 +6170,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB KERNEL16x1_SUB - je .L1_16 + je L(1_16) KERNEL16x1_SUB KERNEL16x1_SUB @@ -6182,12 +6182,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB KERNEL16x1_SUB - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -6195,7 +6195,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -6206,15 +6206,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE16x1 @@ -6235,25 +6235,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -6290,7 +6290,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -6300,7 +6300,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): KERNEL8x1_SUB KERNEL8x1_SUB @@ -6312,7 +6312,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB KERNEL8x1_SUB - je .L1_20_6 + je L(1_20_6) KERNEL8x1_SUB KERNEL8x1_SUB @@ -6324,12 +6324,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB KERNEL8x1_SUB - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -6337,7 +6337,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -6348,15 +6348,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): SAVE8x1 @@ -6382,13 +6382,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -6425,7 +6425,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -6435,7 +6435,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_22: +L(1_22): KERNEL4x1_SUB KERNEL4x1_SUB @@ -6447,7 +6447,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_26 + je L(1_26) KERNEL4x1_SUB KERNEL4x1_SUB @@ -6459,12 +6459,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -6472,7 +6472,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -6483,15 +6483,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): SAVE4x1 @@ -6514,13 +6514,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -6557,7 +6557,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -6567,7 +6567,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_SUB KERNEL2x1_SUB @@ -6579,7 +6579,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_36 + je L(1_36) KERNEL2x1_SUB KERNEL2x1_SUB @@ -6591,12 +6591,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -6604,7 +6604,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -6615,15 +6615,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): SAVE2x1 @@ -6645,13 +6645,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -6686,7 +6686,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -6695,7 +6695,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_SUB KERNEL1x1_SUB @@ -6707,7 +6707,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_46 + je L(1_46) KERNEL1x1_SUB KERNEL1x1_SUB @@ -6719,12 +6719,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -6732,7 +6732,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -6742,15 +6742,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): SAVE1x1 @@ -6772,7 +6772,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/sgemm_kernel_16x4_sandy.S b/kernel/x86_64/sgemm_kernel_16x4_sandy.S index 2ee4b15548..448921a2cc 100644 --- a/kernel/x86_64/sgemm_kernel_16x4_sandy.S +++ b/kernel/x86_64/sgemm_kernel_16x4_sandy.S @@ -826,13 +826,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -861,22 +861,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv6, J cmpq $0, J - je .L2_0 + je L(2_0) ALIGN_4 /*******************************************************************************************/ -.L4_01: +L(4_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L4_01b + jz L(4_01b) ALIGN_4 -.L4_01a: +L(4_01a): prefetcht0 512(BO1) prefetchw 512(BO) @@ -893,30 +893,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 16*SIZE,BO1 addq $ 16*SIZE,BO decq %rax - jnz .L4_01a + jnz L(4_01a) -.L4_01b: +L(4_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L4_02d + jz L(4_02d) ALIGN_4 -.L4_02c: +L(4_02c): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $ 4*SIZE,BO1 addq $ 4*SIZE,BO decq %rax - jnz .L4_02c + jnz L(4_02c) -.L4_02d: +L(4_02d): movq BO1, B // next offset of B -.L4_10: +L(4_10): movq C, CO1 leaq (C, LDC, 2), CO2 leaq (C, LDC, 4), C // c += 4 * ldc @@ -931,11 +931,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L4_20 + je L(4_20) ALIGN_4 -.L4_11: +L(4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -972,7 +972,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L4_16 + je L(4_16) movq %rax, BI // Index for BO leaq (,BI,4) , BI // BI = BI * 4 ; number of values @@ -983,7 +983,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_12: +L(4_12): prefetcht0 A_PR1(AO, %rax, SIZE) prefetcht0 B_PR1(BO, BI , SIZE) @@ -1005,7 +1005,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO, %rax, SIZE) KERNEL16x4_SUB - je .L4_16 + je L(4_16) prefetcht0 A_PR1(AO, %rax, SIZE) prefetcht0 B_PR1(BO, BI , SIZE) @@ -1027,12 +1027,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO, %rax, SIZE) KERNEL16x4_SUB - je .L4_16 + je L(4_16) - jmp .L4_12 + jmp L(4_12) ALIGN_4 -.L4_16: +L(4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -1040,7 +1040,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_19 + je L(4_19) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -1052,15 +1052,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_17: +L(4_17): KERNEL16x4_SUB - jl .L4_17 + jl L(4_17) ALIGN_4 -.L4_19: +L(4_19): SAVE16x4 @@ -1083,25 +1083,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 addq $16 * SIZE, CO2 # coffset += 16 decq I # i -- - jg .L4_11 + jg L(4_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L4_20: +L(4_20): // Test rest of M testq $15, M - jz .L4_60 // to next 3 lines of N + jz L(4_60) // to next 3 lines of N testq $8, M - jz .L4_21pre + jz L(4_21pre) ALIGN_4 /**************************************************************************/ -.L4_20_1: +L(4_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1139,7 +1139,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_20_6 + je L(4_20_6) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -1150,7 +1150,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_20_2: +L(4_20_2): KERNEL8x4_SUB KERNEL8x4_SUB @@ -1162,7 +1162,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_SUB KERNEL8x4_SUB - je .L4_20_6 + je L(4_20_6) KERNEL8x4_SUB KERNEL8x4_SUB @@ -1174,12 +1174,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_SUB KERNEL8x4_SUB - je .L4_20_6 + je L(4_20_6) - jmp .L4_20_2 + jmp L(4_20_2) ALIGN_4 -.L4_20_6: +L(4_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -1187,7 +1187,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_20_9 + je L(4_20_9) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -1199,15 +1199,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_20_7: +L(4_20_7): KERNEL8x4_SUB - jl .L4_20_7 + jl L(4_20_7) ALIGN_4 -.L4_20_9: +L(4_20_9): SAVE8x4 @@ -1235,13 +1235,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L4_21pre: +L(4_21pre): testq $4, M - jz .L4_30 + jz L(4_30) ALIGN_4 -.L4_21: +L(4_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1279,7 +1279,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_26 + je L(4_26) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -1290,7 +1290,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_22: +L(4_22): KERNEL4x4_SUB KERNEL4x4_SUB @@ -1302,7 +1302,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB KERNEL4x4_SUB - je .L4_26 + je L(4_26) KERNEL4x4_SUB KERNEL4x4_SUB @@ -1314,12 +1314,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB KERNEL4x4_SUB - je .L4_26 + je L(4_26) - jmp .L4_22 + jmp L(4_22) ALIGN_4 -.L4_26: +L(4_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -1327,7 +1327,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_29 + je L(4_29) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -1339,15 +1339,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_27: +L(4_27): KERNEL4x4_SUB - jl .L4_27 + jl L(4_27) ALIGN_4 -.L4_29: +L(4_29): SAVE4x4 @@ -1372,13 +1372,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L4_30: +L(4_30): testq $2, M - jz .L4_40 + jz L(4_40) ALIGN_4 -.L4_31: +L(4_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1416,7 +1416,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_36 + je L(4_36) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -1427,7 +1427,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_32: +L(4_32): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1439,7 +1439,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB KERNEL2x4_SUB - je .L4_36 + je L(4_36) KERNEL2x4_SUB KERNEL2x4_SUB @@ -1451,12 +1451,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB KERNEL2x4_SUB - je .L4_36 + je L(4_36) - jmp .L4_32 + jmp L(4_32) ALIGN_4 -.L4_36: +L(4_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -1464,7 +1464,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_39 + je L(4_39) movq %rax, BI // Index for BO leaq (,BI, 4), BI // BI = BI * 4 ; number of values @@ -1476,15 +1476,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_37: +L(4_37): KERNEL2x4_SUB - jl .L4_37 + jl L(4_37) ALIGN_4 -.L4_39: +L(4_39): SAVE2x4 @@ -1508,13 +1508,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO2 # coffset += 2 ALIGN_4 -.L4_40: +L(4_40): testq $1, M - jz .L4_60 // to next 4 lines of N + jz L(4_60) // to next 4 lines of N ALIGN_4 -.L4_41: +L(4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1550,7 +1550,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L4_46 + je L(4_46) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -1560,7 +1560,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_42: +L(4_42): KERNEL1x4_SUB KERNEL1x4_SUB @@ -1572,7 +1572,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB KERNEL1x4_SUB - je .L4_46 + je L(4_46) KERNEL1x4_SUB KERNEL1x4_SUB @@ -1584,12 +1584,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB KERNEL1x4_SUB - je .L4_46 + je L(4_46) - jmp .L4_42 + jmp L(4_42) ALIGN_4 -.L4_46: +L(4_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -1597,7 +1597,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_49 + je L(4_49) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -1608,15 +1608,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_47: +L(4_47): KERNEL1x4_SUB - jl .L4_47 + jl L(4_47) ALIGN_4 -.L4_49: +L(4_49): SAVE1x4 @@ -1643,38 +1643,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L4_60: +L(4_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif decq J // j -- - jg .L4_01 // next 4 lines of N + jg L(4_01) // next 4 lines of N /*******************************************************************************************/ -.L2_0: +L(2_0): movq Nmod6, J andq $3, J // j % 4 - je .L999 + je L(999) movq Nmod6, J andq $2, J // j % 4 - je .L1_0 + je L(1_0) -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L2_01b + jz L(2_01b) ALIGN_4 -.L2_01a: +L(2_01a): vmovsd (BO1), %xmm0 vmovsd 2*SIZE(BO1), %xmm1 @@ -1689,30 +1689,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $8*SIZE,BO1 addq $8*SIZE,BO decq %rax - jnz .L2_01a + jnz L(2_01a) -.L2_01b: +L(2_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L2_02d + jz L(2_02d) ALIGN_4 -.L2_02c: +L(2_02c): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02c + jnz L(2_02c) -.L2_02d: +L(2_02d): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -1726,11 +1726,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1767,7 +1767,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -1778,7 +1778,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_12: +L(2_12): KERNEL16x2_SUB KERNEL16x2_SUB @@ -1790,7 +1790,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB KERNEL16x2_SUB - je .L2_16 + je L(2_16) KERNEL16x2_SUB KERNEL16x2_SUB @@ -1802,12 +1802,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB KERNEL16x2_SUB - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -1815,7 +1815,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -1827,15 +1827,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE16x2 @@ -1857,25 +1857,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1913,7 +1913,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -1924,7 +1924,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): KERNEL8x2_SUB @@ -1937,7 +1937,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB KERNEL8x2_SUB - je .L2_20_6 + je L(2_20_6) KERNEL8x2_SUB KERNEL8x2_SUB @@ -1949,12 +1949,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB KERNEL8x2_SUB - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -1962,7 +1962,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -1974,15 +1974,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): SAVE8x2 @@ -2009,13 +2009,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2053,7 +2053,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -2064,7 +2064,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_22: +L(2_22): KERNEL4x2_SUB @@ -2077,7 +2077,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_26 + je L(2_26) KERNEL4x2_SUB KERNEL4x2_SUB @@ -2089,12 +2089,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -2102,7 +2102,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2114,15 +2114,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): SAVE4x2 @@ -2146,13 +2146,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2190,7 +2190,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2201,7 +2201,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_SUB KERNEL2x2_SUB @@ -2213,7 +2213,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_36 + je L(2_36) KERNEL2x2_SUB KERNEL2x2_SUB @@ -2225,12 +2225,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -2238,7 +2238,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2250,15 +2250,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): SAVE2x2 @@ -2281,13 +2281,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2323,7 +2323,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2333,7 +2333,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_SUB KERNEL1x2_SUB @@ -2345,7 +2345,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_46 + je L(2_46) KERNEL1x2_SUB KERNEL1x2_SUB @@ -2357,12 +2357,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -2370,7 +2370,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2381,15 +2381,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): SAVE1x2 @@ -2415,7 +2415,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -2423,7 +2423,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -2431,30 +2431,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovss (BO1), %xmm0 vmovss %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -2468,11 +2468,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2508,7 +2508,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -2518,7 +2518,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_12: +L(1_12): KERNEL16x1_SUB KERNEL16x1_SUB @@ -2530,7 +2530,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB KERNEL16x1_SUB - je .L1_16 + je L(1_16) KERNEL16x1_SUB KERNEL16x1_SUB @@ -2542,12 +2542,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB KERNEL16x1_SUB - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -2555,7 +2555,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -2566,15 +2566,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE16x1 @@ -2595,25 +2595,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2650,7 +2650,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -2660,7 +2660,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): KERNEL8x1_SUB KERNEL8x1_SUB @@ -2672,7 +2672,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB KERNEL8x1_SUB - je .L1_20_6 + je L(1_20_6) KERNEL8x1_SUB KERNEL8x1_SUB @@ -2684,12 +2684,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB KERNEL8x1_SUB - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -2697,7 +2697,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -2708,15 +2708,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): SAVE8x1 @@ -2742,13 +2742,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2785,7 +2785,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -2795,7 +2795,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_22: +L(1_22): KERNEL4x1_SUB KERNEL4x1_SUB @@ -2807,7 +2807,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_26 + je L(1_26) KERNEL4x1_SUB KERNEL4x1_SUB @@ -2819,12 +2819,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -2832,7 +2832,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -2843,15 +2843,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): SAVE4x1 @@ -2874,13 +2874,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2917,7 +2917,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -2927,7 +2927,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_SUB KERNEL2x1_SUB @@ -2939,7 +2939,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_36 + je L(1_36) KERNEL2x1_SUB KERNEL2x1_SUB @@ -2951,12 +2951,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -2964,7 +2964,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -2975,15 +2975,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): SAVE2x1 @@ -3005,13 +3005,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3046,7 +3046,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -3055,7 +3055,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_SUB KERNEL1x1_SUB @@ -3067,7 +3067,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_46 + je L(1_46) KERNEL1x1_SUB KERNEL1x1_SUB @@ -3079,12 +3079,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -3092,7 +3092,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -3102,15 +3102,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): SAVE1x1 @@ -3132,7 +3132,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/sgemm_kernel_16x4_skylakex.S b/kernel/x86_64/sgemm_kernel_16x4_skylakex.S index ac4421252d..73c8239adb 100644 --- a/kernel/x86_64/sgemm_kernel_16x4_skylakex.S +++ b/kernel/x86_64/sgemm_kernel_16x4_skylakex.S @@ -1220,13 +1220,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -1245,13 +1245,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv6, J cmpq $0, J - je .L4_00 + je L(4_00) ALIGN_4 /*******************************************************************************************/ -.L6_01: +L(6_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO @@ -1264,7 +1264,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L6_02c: +L(6_02c): vmovups (BO1), %xmm0 vmovsd (BO2), %xmm1 @@ -1274,10 +1274,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4*SIZE,BO2 addq $ 6*SIZE,BO decq %rax - jnz .L6_02c + jnz L(6_02c) -.L6_10: +L(6_10): movq C, CO1 leaq (C, LDC, 2), CO2 leaq (CO2, LDC, 1), CO2 // co2 = c + 3 * ldc @@ -1289,11 +1289,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L6_20 + je L(6_20) ALIGN_4 -.L6_11: +L(6_11): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1302,69 +1302,69 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L6_16 + je L(6_16) ALIGN_4 -.L6_12: +L(6_12): KERNEL16x6_SUB4 KERNEL16x6_SUB4 - je .L6_16 + je L(6_16) KERNEL16x6_SUB4 KERNEL16x6_SUB4 - je .L6_16 + je L(6_16) - jmp .L6_12 + jmp L(6_12) ALIGN_4 -.L6_16: +L(6_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_19 + je L(6_19) ALIGN_4 -.L6_17: +L(6_17): KERNEL16x6_SUB - jnz .L6_17 + jnz L(6_17) ALIGN_4 -.L6_19: +L(6_19): SAVE16x6 addq $16 * SIZE, CO1 # coffset += 16 addq $16 * SIZE, CO2 # coffset += 16 decq I # i -- - jg .L6_11 + jg L(6_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L6_20: +L(6_20): // Test rest of M testq $15, M - jz .L6_60 // to next 6 lines of N + jz L(6_60) // to next 6 lines of N testq $8, M - jz .L6_21pre + jz L(6_21pre) ALIGN_4 /**************************************************************************/ -.L6_20_1: +L(6_20_1): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1373,11 +1373,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_20_6 + je L(6_20_6) ALIGN_4 -.L6_20_2: +L(6_20_2): prefetcht0 A_PR1(AO) KERNEL8x6_SUB @@ -1393,7 +1393,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x6_SUB KERNEL8x6_SUB - je .L6_20_6 + je L(6_20_6) prefetcht0 A_PR1(AO) KERNEL8x6_SUB @@ -1409,28 +1409,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x6_SUB KERNEL8x6_SUB - je .L6_20_6 + je L(6_20_6) - jmp .L6_20_2 + jmp L(6_20_2) ALIGN_4 -.L6_20_6: +L(6_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_20_9 + je L(6_20_9) ALIGN_4 -.L6_20_7: +L(6_20_7): KERNEL8x6_SUB - jnz .L6_20_7 + jnz L(6_20_7) ALIGN_4 -.L6_20_9: +L(6_20_9): SAVE8x6 @@ -1442,13 +1442,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L6_21pre: +L(6_21pre): testq $4, M - jz .L6_30 + jz L(6_30) ALIGN_4 -.L6_21: +L(6_21): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1457,11 +1457,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_26 + je L(6_26) ALIGN_4 -.L6_22: +L(6_22): prefetcht0 A_PR1(AO) KERNEL4x6_SUB @@ -1475,7 +1475,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x6_SUB KERNEL4x6_SUB - je .L6_26 + je L(6_26) prefetcht0 A_PR1(AO) KERNEL4x6_SUB @@ -1489,28 +1489,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x6_SUB KERNEL4x6_SUB - je .L6_26 + je L(6_26) - jmp .L6_22 + jmp L(6_22) ALIGN_4 -.L6_26: +L(6_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_29 + je L(6_29) ALIGN_4 -.L6_27: +L(6_27): KERNEL4x6_SUB - jnz .L6_27 + jnz L(6_27) ALIGN_4 -.L6_29: +L(6_29): SAVE4x6 @@ -1519,13 +1519,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L6_30: +L(6_30): testq $2, M - jz .L6_40 + jz L(6_40) ALIGN_4 -.L6_31: +L(6_31): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1534,11 +1534,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_36 + je L(6_36) ALIGN_4 -.L6_32: +L(6_32): prefetcht0 A_PR1(AO) KERNEL2x6_SUB @@ -1551,7 +1551,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x6_SUB KERNEL2x6_SUB - je .L6_36 + je L(6_36) prefetcht0 A_PR1(AO) KERNEL2x6_SUB @@ -1564,28 +1564,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x6_SUB KERNEL2x6_SUB - je .L6_36 + je L(6_36) - jmp .L6_32 + jmp L(6_32) ALIGN_4 -.L6_36: +L(6_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_39 + je L(6_39) ALIGN_4 -.L6_37: +L(6_37): KERNEL2x6_SUB - jnz .L6_37 + jnz L(6_37) ALIGN_4 -.L6_39: +L(6_39): SAVE2x6 @@ -1593,13 +1593,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO2 # coffset += 2 ALIGN_4 -.L6_40: +L(6_40): testq $1, M - jz .L6_60 // to next 4 lines of N + jz L(6_60) // to next 4 lines of N ALIGN_4 -.L6_41: +L(6_41): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1608,11 +1608,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_46 + je L(6_46) ALIGN_4 -.L6_42: +L(6_42): prefetcht0 A_PR1(AO) KERNEL1x6_SUB @@ -1625,7 +1625,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x6_SUB KERNEL1x6_SUB - je .L6_46 + je L(6_46) KERNEL1x6_SUB KERNEL1x6_SUB @@ -1637,28 +1637,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x6_SUB KERNEL1x6_SUB - je .L6_46 + je L(6_46) - jmp .L6_42 + jmp L(6_42) ALIGN_4 -.L6_46: +L(6_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_49 + je L(6_49) ALIGN_4 -.L6_47: +L(6_47): KERNEL1x6_SUB - jnz .L6_47 + jnz L(6_47) ALIGN_4 -.L6_49: +L(6_49): SAVE1x6 @@ -1670,13 +1670,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L6_60: +L(6_60): /*******************************************************************************************/ -.L7_01: +L(7_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO @@ -1688,7 +1688,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L7_02c: +L(7_02c): vmovsd 2*SIZE(BO1), %xmm0 vmovups (BO2), %xmm1 @@ -1698,11 +1698,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4*SIZE,BO2 addq $ 6*SIZE,BO decq %rax - jnz .L7_02c + jnz L(7_02c) movq BO2, B // next offset of B -.L7_10: +L(7_10): movq C, CO1 leaq (C, LDC, 2), CO2 leaq (CO2, LDC, 1), CO2 // co2 = c + 3 * ldc @@ -1714,11 +1714,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L7_20 + je L(7_20) ALIGN_4 -.L7_11: +L(7_11): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1727,69 +1727,69 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L7_16 + je L(7_16) ALIGN_4 -.L7_12: +L(7_12): KERNEL16x6_SUB4 KERNEL16x6_SUB4 - je .L7_16 + je L(7_16) KERNEL16x6_SUB4 KERNEL16x6_SUB4 - je .L7_16 + je L(7_16) - jmp .L7_12 + jmp L(7_12) ALIGN_4 -.L7_16: +L(7_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_19 + je L(7_19) ALIGN_4 -.L7_17: +L(7_17): KERNEL16x6_SUB - jnz .L7_17 + jnz L(7_17) ALIGN_4 -.L7_19: +L(7_19): SAVE16x6 addq $16 * SIZE, CO1 # coffset += 16 addq $16 * SIZE, CO2 # coffset += 16 decq I # i -- - jg .L7_11 + jg L(7_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L7_20: +L(7_20): // Test rest of M testq $15, M - jz .L7_60 // to next 6 lines of N + jz L(7_60) // to next 6 lines of N testq $8, M - jz .L7_21pre + jz L(7_21pre) ALIGN_4 /**************************************************************************/ -.L7_20_1: +L(7_20_1): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1798,11 +1798,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_20_6 + je L(7_20_6) ALIGN_4 -.L7_20_2: +L(7_20_2): prefetcht0 A_PR1(AO) KERNEL8x6_SUB @@ -1818,7 +1818,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x6_SUB KERNEL8x6_SUB - je .L7_20_6 + je L(7_20_6) prefetcht0 A_PR1(AO) KERNEL8x6_SUB @@ -1834,28 +1834,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x6_SUB KERNEL8x6_SUB - je .L7_20_6 + je L(7_20_6) - jmp .L7_20_2 + jmp L(7_20_2) ALIGN_4 -.L7_20_6: +L(7_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_20_9 + je L(7_20_9) ALIGN_4 -.L7_20_7: +L(7_20_7): KERNEL8x6_SUB - jnz .L7_20_7 + jnz L(7_20_7) ALIGN_4 -.L7_20_9: +L(7_20_9): SAVE8x6 @@ -1867,13 +1867,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L7_21pre: +L(7_21pre): testq $4, M - jz .L7_30 + jz L(7_30) ALIGN_4 -.L7_21: +L(7_21): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1882,11 +1882,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_26 + je L(7_26) ALIGN_4 -.L7_22: +L(7_22): prefetcht0 A_PR1(AO) KERNEL4x6_SUB @@ -1900,7 +1900,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x6_SUB KERNEL4x6_SUB - je .L7_26 + je L(7_26) prefetcht0 A_PR1(AO) KERNEL4x6_SUB @@ -1914,28 +1914,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x6_SUB KERNEL4x6_SUB - je .L7_26 + je L(7_26) - jmp .L7_22 + jmp L(7_22) ALIGN_4 -.L7_26: +L(7_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_29 + je L(7_29) ALIGN_4 -.L7_27: +L(7_27): KERNEL4x6_SUB - jnz .L7_27 + jnz L(7_27) ALIGN_4 -.L7_29: +L(7_29): SAVE4x6 @@ -1944,13 +1944,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L7_30: +L(7_30): testq $2, M - jz .L7_40 + jz L(7_40) ALIGN_4 -.L7_31: +L(7_31): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1959,11 +1959,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_36 + je L(7_36) ALIGN_4 -.L7_32: +L(7_32): prefetcht0 A_PR1(AO) KERNEL2x6_SUB @@ -1976,7 +1976,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x6_SUB KERNEL2x6_SUB - je .L7_36 + je L(7_36) prefetcht0 A_PR1(AO) KERNEL2x6_SUB @@ -1989,28 +1989,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x6_SUB KERNEL2x6_SUB - je .L7_36 + je L(7_36) - jmp .L7_32 + jmp L(7_32) ALIGN_4 -.L7_36: +L(7_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_39 + je L(7_39) ALIGN_4 -.L7_37: +L(7_37): KERNEL2x6_SUB - jnz .L7_37 + jnz L(7_37) ALIGN_4 -.L7_39: +L(7_39): SAVE2x6 @@ -2018,13 +2018,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO2 # coffset += 2 ALIGN_4 -.L7_40: +L(7_40): testq $1, M - jz .L7_60 // to next 4 lines of N + jz L(7_60) // to next 4 lines of N ALIGN_4 -.L7_41: +L(7_41): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2033,11 +2033,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_46 + je L(7_46) ALIGN_4 -.L7_42: +L(7_42): prefetcht0 A_PR1(AO) KERNEL1x6_SUB @@ -2050,7 +2050,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x6_SUB KERNEL1x6_SUB - je .L7_46 + je L(7_46) KERNEL1x6_SUB KERNEL1x6_SUB @@ -2062,28 +2062,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x6_SUB KERNEL1x6_SUB - je .L7_46 + je L(7_46) - jmp .L7_42 + jmp L(7_42) ALIGN_4 -.L7_46: +L(7_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_49 + je L(7_49) ALIGN_4 -.L7_47: +L(7_47): KERNEL1x6_SUB - jnz .L7_47 + jnz L(7_47) ALIGN_4 -.L7_49: +L(7_49): SAVE1x6 @@ -2095,35 +2095,35 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L7_60: +L(7_60): decq J // j -- - jg .L6_01 // next 12 lines of N + jg L(6_01) // next 12 lines of N /*******************************************************************************************/ -.L4_00: +L(4_00): movq Nmod6, J sarq $2, J // j = j / 4 cmpq $ 0, J - je .L2_00 + je L(2_00) ALIGN_4 -.L4_01: +L(4_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L4_01b + jz L(4_01b) ALIGN_4 -.L4_01a: +L(4_01a): prefetcht0 512(BO1) prefetchw 512(BO) @@ -2140,30 +2140,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 16*SIZE,BO1 addq $ 16*SIZE,BO decq %rax - jnz .L4_01a + jnz L(4_01a) -.L4_01b: +L(4_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L4_02d + jz L(4_02d) ALIGN_4 -.L4_02c: +L(4_02c): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $ 4*SIZE,BO1 addq $ 4*SIZE,BO decq %rax - jnz .L4_02c + jnz L(4_02c) -.L4_02d: +L(4_02d): movq BO1, B // next offset of B -.L4_10: +L(4_10): movq C, CO1 leaq (C, LDC, 2), CO2 leaq (C, LDC, 4), C // c += 4 * ldc @@ -2178,11 +2178,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L4_20 + je L(4_20) ALIGN_4 -.L4_11: +L(4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2219,7 +2219,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L4_16 + je L(4_16) movq %rax, BI // Index for BO leaq (,BI,4) , BI // BI = BI * 4 ; number of values @@ -2230,7 +2230,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_12: +L(4_12): prefetcht0 A_PR1(AO, %rax, SIZE) prefetcht0 B_PR1(BO, BI , SIZE) @@ -2252,7 +2252,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO, %rax, SIZE) KERNEL16x4_SUB - je .L4_16 + je L(4_16) prefetcht0 A_PR1(AO, %rax, SIZE) prefetcht0 B_PR1(BO, BI , SIZE) @@ -2274,12 +2274,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO, %rax, SIZE) KERNEL16x4_SUB - je .L4_16 + je L(4_16) - jmp .L4_12 + jmp L(4_12) ALIGN_4 -.L4_16: +L(4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -2287,7 +2287,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_19 + je L(4_19) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2299,15 +2299,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_17: +L(4_17): KERNEL16x4_SUB - jl .L4_17 + jl L(4_17) ALIGN_4 -.L4_19: +L(4_19): SAVE16x4 @@ -2330,25 +2330,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 addq $16 * SIZE, CO2 # coffset += 16 decq I # i -- - jg .L4_11 + jg L(4_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L4_20: +L(4_20): // Test rest of M testq $15, M - jz .L4_60 // to next 3 lines of N + jz L(4_60) // to next 3 lines of N testq $8, M - jz .L4_21pre + jz L(4_21pre) ALIGN_4 /**************************************************************************/ -.L4_20_1: +L(4_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2386,7 +2386,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_20_6 + je L(4_20_6) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2397,7 +2397,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_20_2: +L(4_20_2): KERNEL8x4_SUB KERNEL8x4_SUB @@ -2409,7 +2409,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_SUB KERNEL8x4_SUB - je .L4_20_6 + je L(4_20_6) KERNEL8x4_SUB KERNEL8x4_SUB @@ -2421,12 +2421,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_SUB KERNEL8x4_SUB - je .L4_20_6 + je L(4_20_6) - jmp .L4_20_2 + jmp L(4_20_2) ALIGN_4 -.L4_20_6: +L(4_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -2434,7 +2434,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_20_9 + je L(4_20_9) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2446,15 +2446,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_20_7: +L(4_20_7): KERNEL8x4_SUB - jl .L4_20_7 + jl L(4_20_7) ALIGN_4 -.L4_20_9: +L(4_20_9): SAVE8x4 @@ -2482,13 +2482,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L4_21pre: +L(4_21pre): testq $4, M - jz .L4_30 + jz L(4_30) ALIGN_4 -.L4_21: +L(4_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2526,7 +2526,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_26 + je L(4_26) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2537,7 +2537,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_22: +L(4_22): KERNEL4x4_SUB KERNEL4x4_SUB @@ -2549,7 +2549,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB KERNEL4x4_SUB - je .L4_26 + je L(4_26) KERNEL4x4_SUB KERNEL4x4_SUB @@ -2561,12 +2561,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB KERNEL4x4_SUB - je .L4_26 + je L(4_26) - jmp .L4_22 + jmp L(4_22) ALIGN_4 -.L4_26: +L(4_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -2574,7 +2574,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_29 + je L(4_29) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2586,15 +2586,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_27: +L(4_27): KERNEL4x4_SUB - jl .L4_27 + jl L(4_27) ALIGN_4 -.L4_29: +L(4_29): SAVE4x4 @@ -2619,13 +2619,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L4_30: +L(4_30): testq $2, M - jz .L4_40 + jz L(4_40) ALIGN_4 -.L4_31: +L(4_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2663,7 +2663,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_36 + je L(4_36) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2674,7 +2674,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_32: +L(4_32): KERNEL2x4_SUB KERNEL2x4_SUB @@ -2686,7 +2686,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB KERNEL2x4_SUB - je .L4_36 + je L(4_36) KERNEL2x4_SUB KERNEL2x4_SUB @@ -2698,12 +2698,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB KERNEL2x4_SUB - je .L4_36 + je L(4_36) - jmp .L4_32 + jmp L(4_32) ALIGN_4 -.L4_36: +L(4_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -2711,7 +2711,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_39 + je L(4_39) movq %rax, BI // Index for BO leaq (,BI, 4), BI // BI = BI * 4 ; number of values @@ -2723,15 +2723,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_37: +L(4_37): KERNEL2x4_SUB - jl .L4_37 + jl L(4_37) ALIGN_4 -.L4_39: +L(4_39): SAVE2x4 @@ -2755,13 +2755,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO2 # coffset += 2 ALIGN_4 -.L4_40: +L(4_40): testq $1, M - jz .L4_60 // to next 4 lines of N + jz L(4_60) // to next 4 lines of N ALIGN_4 -.L4_41: +L(4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2797,7 +2797,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L4_46 + je L(4_46) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2807,7 +2807,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_42: +L(4_42): KERNEL1x4_SUB KERNEL1x4_SUB @@ -2819,7 +2819,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB KERNEL1x4_SUB - je .L4_46 + je L(4_46) KERNEL1x4_SUB KERNEL1x4_SUB @@ -2831,12 +2831,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB KERNEL1x4_SUB - je .L4_46 + je L(4_46) - jmp .L4_42 + jmp L(4_42) ALIGN_4 -.L4_46: +L(4_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -2844,7 +2844,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_49 + je L(4_49) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2855,15 +2855,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_47: +L(4_47): KERNEL1x4_SUB - jl .L4_47 + jl L(4_47) ALIGN_4 -.L4_49: +L(4_49): SAVE1x4 @@ -2890,38 +2890,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L4_60: +L(4_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif decq J // j -- - jg .L4_01 // next 4 lines of N + jg L(4_01) // next 4 lines of N /*******************************************************************************************/ -.L2_00: +L(2_00): movq Nmod6, J andq $3, J // j % 4 - je .L999 + je L(999) movq Nmod6, J andq $2, J // j % 4 - je .L1_0 + je L(1_0) -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L2_01b + jz L(2_01b) ALIGN_4 -.L2_01a: +L(2_01a): vmovsd (BO1), %xmm0 vmovsd 2*SIZE(BO1), %xmm1 @@ -2936,30 +2936,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $8*SIZE,BO1 addq $8*SIZE,BO decq %rax - jnz .L2_01a + jnz L(2_01a) -.L2_01b: +L(2_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L2_02d + jz L(2_02d) ALIGN_4 -.L2_02c: +L(2_02c): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02c + jnz L(2_02c) -.L2_02d: +L(2_02d): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -2973,11 +2973,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3014,7 +3014,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3025,7 +3025,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_12: +L(2_12): KERNEL16x2_SUB KERNEL16x2_SUB @@ -3037,7 +3037,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB KERNEL16x2_SUB - je .L2_16 + je L(2_16) KERNEL16x2_SUB KERNEL16x2_SUB @@ -3049,12 +3049,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB KERNEL16x2_SUB - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3062,7 +3062,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3074,15 +3074,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE16x2 @@ -3104,25 +3104,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3160,7 +3160,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3171,7 +3171,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): KERNEL8x2_SUB @@ -3184,7 +3184,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB KERNEL8x2_SUB - je .L2_20_6 + je L(2_20_6) KERNEL8x2_SUB KERNEL8x2_SUB @@ -3196,12 +3196,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB KERNEL8x2_SUB - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -3209,7 +3209,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3221,15 +3221,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): SAVE8x2 @@ -3256,13 +3256,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3300,7 +3300,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -3311,7 +3311,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_22: +L(2_22): KERNEL4x2_SUB @@ -3324,7 +3324,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_26 + je L(2_26) KERNEL4x2_SUB KERNEL4x2_SUB @@ -3336,12 +3336,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -3349,7 +3349,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3361,15 +3361,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): SAVE4x2 @@ -3393,13 +3393,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3437,7 +3437,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3448,7 +3448,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_SUB KERNEL2x2_SUB @@ -3460,7 +3460,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_36 + je L(2_36) KERNEL2x2_SUB KERNEL2x2_SUB @@ -3472,12 +3472,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -3485,7 +3485,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3497,15 +3497,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): SAVE2x2 @@ -3528,13 +3528,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3570,7 +3570,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3580,7 +3580,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_SUB KERNEL1x2_SUB @@ -3592,7 +3592,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_46 + je L(2_46) KERNEL1x2_SUB KERNEL1x2_SUB @@ -3604,12 +3604,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -3617,7 +3617,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3628,15 +3628,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): SAVE1x2 @@ -3662,7 +3662,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -3670,7 +3670,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -3678,30 +3678,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovss (BO1), %xmm0 vmovss %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -3715,11 +3715,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3755,7 +3755,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -3765,7 +3765,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_12: +L(1_12): KERNEL16x1_SUB KERNEL16x1_SUB @@ -3777,7 +3777,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB KERNEL16x1_SUB - je .L1_16 + je L(1_16) KERNEL16x1_SUB KERNEL16x1_SUB @@ -3789,12 +3789,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB KERNEL16x1_SUB - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3802,7 +3802,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -3813,15 +3813,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE16x1 @@ -3842,25 +3842,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3897,7 +3897,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -3907,7 +3907,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): KERNEL8x1_SUB KERNEL8x1_SUB @@ -3919,7 +3919,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB KERNEL8x1_SUB - je .L1_20_6 + je L(1_20_6) KERNEL8x1_SUB KERNEL8x1_SUB @@ -3931,12 +3931,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB KERNEL8x1_SUB - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -3944,7 +3944,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -3955,15 +3955,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): SAVE8x1 @@ -3989,13 +3989,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4032,7 +4032,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -4042,7 +4042,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_22: +L(1_22): KERNEL4x1_SUB KERNEL4x1_SUB @@ -4054,7 +4054,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_26 + je L(1_26) KERNEL4x1_SUB KERNEL4x1_SUB @@ -4066,12 +4066,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4079,7 +4079,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -4090,15 +4090,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): SAVE4x1 @@ -4121,13 +4121,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4164,7 +4164,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -4174,7 +4174,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_SUB KERNEL2x1_SUB @@ -4186,7 +4186,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_36 + je L(1_36) KERNEL2x1_SUB KERNEL2x1_SUB @@ -4198,12 +4198,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -4211,7 +4211,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -4222,15 +4222,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): SAVE2x1 @@ -4252,13 +4252,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4293,7 +4293,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -4302,7 +4302,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_SUB KERNEL1x1_SUB @@ -4314,7 +4314,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_46 + je L(1_46) KERNEL1x1_SUB KERNEL1x1_SUB @@ -4326,12 +4326,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -4339,7 +4339,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -4349,15 +4349,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): SAVE1x1 @@ -4379,7 +4379,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp @@ -4471,13 +4471,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -4506,22 +4506,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv6, J cmpq $0, J - je .L2_0 + je L(2_0) ALIGN_4 /*******************************************************************************************/ -.L4_01: +L(4_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L4_01b + jz L(4_01b) ALIGN_4 -.L4_01a: +L(4_01a): prefetcht0 512(BO1) prefetchw 512(BO) @@ -4538,30 +4538,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 16*SIZE,BO1 addq $ 16*SIZE,BO decq %rax - jnz .L4_01a + jnz L(4_01a) -.L4_01b: +L(4_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L4_02d + jz L(4_02d) ALIGN_4 -.L4_02c: +L(4_02c): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $ 4*SIZE,BO1 addq $ 4*SIZE,BO decq %rax - jnz .L4_02c + jnz L(4_02c) -.L4_02d: +L(4_02d): movq BO1, B // next offset of B -.L4_10: +L(4_10): movq C, CO1 leaq (C, LDC, 2), CO2 leaq (C, LDC, 4), C // c += 4 * ldc @@ -4576,11 +4576,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L4_20 + je L(4_20) ALIGN_4 -.L4_11: +L(4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4617,7 +4617,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L4_16 + je L(4_16) movq %rax, BI // Index for BO leaq (,BI,4) , BI // BI = BI * 4 ; number of values @@ -4628,7 +4628,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_12: +L(4_12): prefetcht0 A_PR1(AO, %rax, SIZE) prefetcht0 B_PR1(BO, BI , SIZE) @@ -4650,7 +4650,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO, %rax, SIZE) KERNEL16x4_SUB - je .L4_16 + je L(4_16) prefetcht0 A_PR1(AO, %rax, SIZE) prefetcht0 B_PR1(BO, BI , SIZE) @@ -4672,12 +4672,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO, %rax, SIZE) KERNEL16x4_SUB - je .L4_16 + je L(4_16) - jmp .L4_12 + jmp L(4_12) ALIGN_4 -.L4_16: +L(4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -4685,7 +4685,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_19 + je L(4_19) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -4697,15 +4697,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_17: +L(4_17): KERNEL16x4_SUB - jl .L4_17 + jl L(4_17) ALIGN_4 -.L4_19: +L(4_19): SAVE16x4 @@ -4728,25 +4728,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 addq $16 * SIZE, CO2 # coffset += 16 decq I # i -- - jg .L4_11 + jg L(4_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L4_20: +L(4_20): // Test rest of M testq $15, M - jz .L4_60 // to next 3 lines of N + jz L(4_60) // to next 3 lines of N testq $8, M - jz .L4_21pre + jz L(4_21pre) ALIGN_4 /**************************************************************************/ -.L4_20_1: +L(4_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4784,7 +4784,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_20_6 + je L(4_20_6) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -4795,7 +4795,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_20_2: +L(4_20_2): KERNEL8x4_SUB KERNEL8x4_SUB @@ -4807,7 +4807,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_SUB KERNEL8x4_SUB - je .L4_20_6 + je L(4_20_6) KERNEL8x4_SUB KERNEL8x4_SUB @@ -4819,12 +4819,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_SUB KERNEL8x4_SUB - je .L4_20_6 + je L(4_20_6) - jmp .L4_20_2 + jmp L(4_20_2) ALIGN_4 -.L4_20_6: +L(4_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -4832,7 +4832,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_20_9 + je L(4_20_9) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -4844,15 +4844,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_20_7: +L(4_20_7): KERNEL8x4_SUB - jl .L4_20_7 + jl L(4_20_7) ALIGN_4 -.L4_20_9: +L(4_20_9): SAVE8x4 @@ -4880,13 +4880,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L4_21pre: +L(4_21pre): testq $4, M - jz .L4_30 + jz L(4_30) ALIGN_4 -.L4_21: +L(4_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4924,7 +4924,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_26 + je L(4_26) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -4935,7 +4935,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_22: +L(4_22): KERNEL4x4_SUB KERNEL4x4_SUB @@ -4947,7 +4947,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB KERNEL4x4_SUB - je .L4_26 + je L(4_26) KERNEL4x4_SUB KERNEL4x4_SUB @@ -4959,12 +4959,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB KERNEL4x4_SUB - je .L4_26 + je L(4_26) - jmp .L4_22 + jmp L(4_22) ALIGN_4 -.L4_26: +L(4_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4972,7 +4972,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_29 + je L(4_29) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -4984,15 +4984,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_27: +L(4_27): KERNEL4x4_SUB - jl .L4_27 + jl L(4_27) ALIGN_4 -.L4_29: +L(4_29): SAVE4x4 @@ -5017,13 +5017,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L4_30: +L(4_30): testq $2, M - jz .L4_40 + jz L(4_40) ALIGN_4 -.L4_31: +L(4_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5061,7 +5061,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_36 + je L(4_36) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -5072,7 +5072,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_32: +L(4_32): KERNEL2x4_SUB KERNEL2x4_SUB @@ -5084,7 +5084,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB KERNEL2x4_SUB - je .L4_36 + je L(4_36) KERNEL2x4_SUB KERNEL2x4_SUB @@ -5096,12 +5096,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB KERNEL2x4_SUB - je .L4_36 + je L(4_36) - jmp .L4_32 + jmp L(4_32) ALIGN_4 -.L4_36: +L(4_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -5109,7 +5109,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_39 + je L(4_39) movq %rax, BI // Index for BO leaq (,BI, 4), BI // BI = BI * 4 ; number of values @@ -5121,15 +5121,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_37: +L(4_37): KERNEL2x4_SUB - jl .L4_37 + jl L(4_37) ALIGN_4 -.L4_39: +L(4_39): SAVE2x4 @@ -5153,13 +5153,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO2 # coffset += 2 ALIGN_4 -.L4_40: +L(4_40): testq $1, M - jz .L4_60 // to next 4 lines of N + jz L(4_60) // to next 4 lines of N ALIGN_4 -.L4_41: +L(4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5195,7 +5195,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L4_46 + je L(4_46) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -5205,7 +5205,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_42: +L(4_42): KERNEL1x4_SUB KERNEL1x4_SUB @@ -5217,7 +5217,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB KERNEL1x4_SUB - je .L4_46 + je L(4_46) KERNEL1x4_SUB KERNEL1x4_SUB @@ -5229,12 +5229,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB KERNEL1x4_SUB - je .L4_46 + je L(4_46) - jmp .L4_42 + jmp L(4_42) ALIGN_4 -.L4_46: +L(4_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -5242,7 +5242,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_49 + je L(4_49) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -5253,15 +5253,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_47: +L(4_47): KERNEL1x4_SUB - jl .L4_47 + jl L(4_47) ALIGN_4 -.L4_49: +L(4_49): SAVE1x4 @@ -5288,38 +5288,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L4_60: +L(4_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif decq J // j -- - jg .L4_01 // next 4 lines of N + jg L(4_01) // next 4 lines of N /*******************************************************************************************/ -.L2_0: +L(2_0): movq Nmod6, J andq $3, J // j % 4 - je .L999 + je L(999) movq Nmod6, J andq $2, J // j % 4 - je .L1_0 + je L(1_0) -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L2_01b + jz L(2_01b) ALIGN_4 -.L2_01a: +L(2_01a): vmovsd (BO1), %xmm0 vmovsd 2*SIZE(BO1), %xmm1 @@ -5334,30 +5334,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $8*SIZE,BO1 addq $8*SIZE,BO decq %rax - jnz .L2_01a + jnz L(2_01a) -.L2_01b: +L(2_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L2_02d + jz L(2_02d) ALIGN_4 -.L2_02c: +L(2_02c): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02c + jnz L(2_02c) -.L2_02d: +L(2_02d): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -5371,11 +5371,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5412,7 +5412,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5423,7 +5423,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_12: +L(2_12): KERNEL16x2_SUB KERNEL16x2_SUB @@ -5435,7 +5435,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB KERNEL16x2_SUB - je .L2_16 + je L(2_16) KERNEL16x2_SUB KERNEL16x2_SUB @@ -5447,12 +5447,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB KERNEL16x2_SUB - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -5460,7 +5460,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5472,15 +5472,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE16x2 @@ -5502,25 +5502,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5558,7 +5558,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5569,7 +5569,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): KERNEL8x2_SUB @@ -5582,7 +5582,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB KERNEL8x2_SUB - je .L2_20_6 + je L(2_20_6) KERNEL8x2_SUB KERNEL8x2_SUB @@ -5594,12 +5594,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB KERNEL8x2_SUB - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -5607,7 +5607,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5619,15 +5619,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): SAVE8x2 @@ -5654,13 +5654,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5698,7 +5698,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -5709,7 +5709,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_22: +L(2_22): KERNEL4x2_SUB @@ -5722,7 +5722,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_26 + je L(2_26) KERNEL4x2_SUB KERNEL4x2_SUB @@ -5734,12 +5734,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -5747,7 +5747,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5759,15 +5759,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): SAVE4x2 @@ -5791,13 +5791,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5835,7 +5835,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5846,7 +5846,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_SUB KERNEL2x2_SUB @@ -5858,7 +5858,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_36 + je L(2_36) KERNEL2x2_SUB KERNEL2x2_SUB @@ -5870,12 +5870,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -5883,7 +5883,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5895,15 +5895,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): SAVE2x2 @@ -5926,13 +5926,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5968,7 +5968,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5978,7 +5978,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_SUB KERNEL1x2_SUB @@ -5990,7 +5990,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_46 + je L(2_46) KERNEL1x2_SUB KERNEL1x2_SUB @@ -6002,12 +6002,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -6015,7 +6015,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -6026,15 +6026,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): SAVE1x2 @@ -6060,7 +6060,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -6068,7 +6068,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -6076,30 +6076,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovss (BO1), %xmm0 vmovss %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -6113,11 +6113,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -6153,7 +6153,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -6163,7 +6163,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_12: +L(1_12): KERNEL16x1_SUB KERNEL16x1_SUB @@ -6175,7 +6175,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB KERNEL16x1_SUB - je .L1_16 + je L(1_16) KERNEL16x1_SUB KERNEL16x1_SUB @@ -6187,12 +6187,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB KERNEL16x1_SUB - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -6200,7 +6200,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -6211,15 +6211,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE16x1 @@ -6240,25 +6240,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -6295,7 +6295,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -6305,7 +6305,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): KERNEL8x1_SUB KERNEL8x1_SUB @@ -6317,7 +6317,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB KERNEL8x1_SUB - je .L1_20_6 + je L(1_20_6) KERNEL8x1_SUB KERNEL8x1_SUB @@ -6329,12 +6329,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB KERNEL8x1_SUB - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -6342,7 +6342,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -6353,15 +6353,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): SAVE8x1 @@ -6387,13 +6387,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -6430,7 +6430,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -6440,7 +6440,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_22: +L(1_22): KERNEL4x1_SUB KERNEL4x1_SUB @@ -6452,7 +6452,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_26 + je L(1_26) KERNEL4x1_SUB KERNEL4x1_SUB @@ -6464,12 +6464,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -6477,7 +6477,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -6488,15 +6488,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): SAVE4x1 @@ -6519,13 +6519,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -6562,7 +6562,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -6572,7 +6572,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_SUB KERNEL2x1_SUB @@ -6584,7 +6584,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_36 + je L(1_36) KERNEL2x1_SUB KERNEL2x1_SUB @@ -6596,12 +6596,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -6609,7 +6609,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -6620,15 +6620,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): SAVE2x1 @@ -6650,13 +6650,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -6691,7 +6691,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -6700,7 +6700,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_SUB KERNEL1x1_SUB @@ -6712,7 +6712,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_46 + je L(1_46) KERNEL1x1_SUB KERNEL1x1_SUB @@ -6724,12 +6724,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -6737,7 +6737,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -6747,15 +6747,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): SAVE1x1 @@ -6777,7 +6777,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/sgemm_kernel_8x4_bulldozer.S b/kernel/x86_64/sgemm_kernel_8x4_bulldozer.S index 2194fd6c1b..2b3128562c 100644 --- a/kernel/x86_64/sgemm_kernel_8x4_bulldozer.S +++ b/kernel/x86_64/sgemm_kernel_8x4_bulldozer.S @@ -404,9 +404,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L50 + jle L(50) -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -417,10 +417,10 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): prefetcht0 192(B) prefetcht0 256(B) @@ -478,17 +478,17 @@ addq $64 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movaps 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -504,10 +504,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): movq C, CO1 leaq (C, LDC, 1), CO2 movq A, AO @@ -515,10 +515,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -575,10 +575,10 @@ leaq (BO, %rax, 8), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_3 -.L12: +L(12): KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -588,7 +588,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -598,7 +598,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -608,7 +608,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -618,7 +618,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -628,7 +628,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -638,7 +638,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -648,7 +648,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -658,10 +658,10 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) BRANCH - jl .L12 + jl L(12) ALIGN_4 -.L15: +L(15): movaps ALPHA, %xmm7 @@ -671,7 +671,7 @@ movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_3 @@ -684,14 +684,14 @@ addq $64 * SIZE, BO ALIGN_3 -.L16: +L(16): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L18 + je L(18) leaq (, %rax, 8), %rax leaq (AO, %rax, 4), AO @@ -699,7 +699,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulps %xmm1, %xmm0 mulps -28 * SIZE(AO, %rax, 4), %xmm1 addps %xmm0, %xmm8 @@ -726,10 +726,10 @@ movaps %xmm0, %xmm2 addq $SIZE * 2, %rax - jl .L17 + jl L(17) ALIGN_4 -.L18: +L(18): #ifndef TRMMKERNEL vfmaddps 0 * SIZE(CO1),%xmm7, %xmm8, %xmm8 @@ -785,12 +785,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -834,10 +834,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -945,10 +945,10 @@ addq $ 32 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -957,10 +957,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -977,10 +977,10 @@ addq $ 4 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): mulps %xmm15, %xmm0 mulps %xmm15, %xmm1 mulps %xmm15, %xmm2 @@ -1026,9 +1026,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $2, M - je .L40 + je L(40) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1072,10 +1072,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movsd 4 * SIZE(BO), %xmm9 @@ -1191,10 +1191,10 @@ addq $ 16 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1203,10 +1203,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movsd 4 * SIZE(BO), %xmm9 @@ -1224,10 +1224,10 @@ addq $ 2 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): mulps %xmm15, %xmm0 mulps %xmm15, %xmm1 mulps %xmm15, %xmm2 @@ -1267,9 +1267,9 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L40: +L(40): testq $1, M - je .L49 + je L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1314,10 +1314,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): mulss %xmm8, %xmm9 addss %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -1433,10 +1433,10 @@ addq $ 8 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1445,10 +1445,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -1466,10 +1466,10 @@ addq $ 1 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): mulss %xmm15, %xmm0 mulss %xmm15, %xmm1 mulss %xmm15, %xmm2 @@ -1507,19 +1507,19 @@ #endif ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C # c += 4 * ldc decq J # j -- - jg .L01 + jg L(01) -.L50: +L(50): testq $2, N - je .L100 + je L(100) -.L51: +L(51): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1530,10 +1530,10 @@ movq K, %rax sarq $2, %rax - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 @@ -1563,17 +1563,17 @@ addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): movq K, %rax andq $3, %rax BRANCH - jle .L60 + jle L(60) ALIGN_4 -.L54: +L(54): movsd 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -1590,20 +1590,20 @@ addq $ 2 * SIZE, B addq $ 8 * SIZE, BO decq %rax - jne .L54 + jne L(54) ALIGN_4 -.L60: +L(60): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L70 + jle L(70) ALIGN_4 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1651,10 +1651,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -1762,10 +1762,10 @@ addq $64 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1774,10 +1774,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -1794,10 +1794,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -1844,12 +1844,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L61 + jg L(61) ALIGN_4 -.L70: +L(70): testq $4, M - je .L80 + je L(80) #if !defined(TRMMKERNEL) || \ @@ -1894,10 +1894,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 @@ -1958,10 +1958,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -1970,10 +1970,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -1984,10 +1984,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2026,9 +2026,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L80: +L(80): testq $2, M - je .L90 + je L(90) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2072,10 +2072,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L85 + je L(85) ALIGN_4 -.L82: +L(82): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movsd 4 * SIZE(BO), %xmm9 @@ -2143,10 +2143,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L85: +L(85): #ifndef TRMMKERNEL movq K, %rax #else @@ -2155,10 +2155,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movsd 4 * SIZE(BO), %xmm9 @@ -2170,10 +2170,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movsd 0 * SIZE(CO2), %xmm10 @@ -2210,9 +2210,9 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L90: +L(90): testq $1, M - je .L99 + je L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2256,10 +2256,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -2327,10 +2327,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2339,10 +2339,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -2354,10 +2354,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): #ifndef TRMMKERNEL movss 0 * SIZE(CO1), %xmm8 movss 0 * SIZE(CO2), %xmm10 @@ -2390,7 +2390,7 @@ #endif ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -2398,11 +2398,11 @@ ALIGN_4 -.L100: +L(100): testq $1, N - je .L999 + je L(999) -.L101: +L(101): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -2413,11 +2413,11 @@ movq K, %rax sarq $3, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movups 0 * SIZE(B), %xmm3 movups 4 * SIZE(B), %xmm7 @@ -2447,17 +2447,17 @@ addq $32 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): movq K, %rax andq $7, %rax BRANCH - jle .L110 + jle L(110) ALIGN_4 -.L104: +L(104): movss 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -2467,19 +2467,19 @@ addq $ 1 * SIZE, B addq $ 4 * SIZE, BO decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L110: +L(110): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L120 + jle L(120) ALIGN_4 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2526,10 +2526,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulps %xmm9, %xmm8 mulps -28 * SIZE(AO), %xmm9 @@ -2590,10 +2590,10 @@ addq $64 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2602,10 +2602,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulps %xmm9, %xmm8 mulps -28 * SIZE(AO), %xmm9 addps %xmm8, %xmm0 @@ -2616,10 +2616,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2652,12 +2652,12 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L111 + jg L(111) ALIGN_4 -.L120: +L(120): testq $4, M - je .L130 + je L(130) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2699,10 +2699,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L125 + je L(125) ALIGN_4 -.L122: +L(122): mulps %xmm8, %xmm9 movaps -28 * SIZE(AO), %xmm8 mulps 4 * SIZE(BO), %xmm8 @@ -2734,10 +2734,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L122 + jne L(122) ALIGN_4 -.L125: +L(125): #ifndef TRMMKERNEL movq K, %rax #else @@ -2746,10 +2746,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): mulps %xmm8, %xmm9 movaps -28 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -2758,10 +2758,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2794,9 +2794,9 @@ addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L130: +L(130): testq $2, M - je .L140 + je L(140) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2838,10 +2838,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L135 + je L(135) ALIGN_4 -.L132: +L(132): mulps %xmm8, %xmm9 movsd -30 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -2884,10 +2884,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L132 + jne L(132) ALIGN_4 -.L135: +L(135): #ifndef TRMMKERNEL movq K, %rax #else @@ -2896,10 +2896,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): mulps %xmm8, %xmm9 movsd -30 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -2908,10 +2908,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): addps %xmm1, %xmm0 mulps %xmm15, %xmm0 @@ -2938,9 +2938,9 @@ addq $2 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L140: +L(140): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2982,10 +2982,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L145 + je L(145) ALIGN_4 -.L142: +L(142): mulss %xmm8, %xmm9 movss -31 * SIZE(AO), %xmm8 mulss 4 * SIZE(BO), %xmm8 @@ -3016,10 +3016,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L142 + jne L(142) ALIGN_4 -.L145: +L(145): #ifndef TRMMKERNEL movq K, %rax #else @@ -3028,10 +3028,10 @@ movss ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): mulss %xmm8, %xmm9 movss -31 * SIZE(AO), %xmm8 addss %xmm9, %xmm0 @@ -3040,10 +3040,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): addss %xmm1, %xmm0 addss %xmm3, %xmm2 addss %xmm2, %xmm0 @@ -3057,7 +3057,7 @@ movss %xmm0, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/sgemm_kernel_8x8_sandy.S b/kernel/x86_64/sgemm_kernel_8x8_sandy.S index 2843efbaeb..2fc6a724ec 100644 --- a/kernel/x86_64/sgemm_kernel_8x8_sandy.S +++ b/kernel/x86_64/sgemm_kernel_8x8_sandy.S @@ -255,9 +255,9 @@ movq %r11, kk #endif MOVQ bn,j; SARQ $3,j; -JLE .L0_loopE; +JLE L(0_loopE); ALIGN_4; -.L0_bodyB:; +L(0_bodyB):; #if defined(TRMMKERNEL) && defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk; @@ -271,9 +271,9 @@ LEAQ (bb, k, 1), prebb; MOVQ ba,ptrba; MOVQ bm,i; SARQ $3,i; -JLE .L1_loopE; +JLE L(1_loopE); ALIGN_4; -.L1_bodyB:; +L(1_bodyB):; #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -324,9 +324,9 @@ ADDQ $8, %rax; MOVQ %rax, kkk; #endif SARQ $2,k; -JLE .L2_loopE; +JLE L(2_loopE); ALIGN_4; -.L2_bodyB:; +L(2_bodyB):; # Computing kernel #### Unroll times 1 #### @@ -452,19 +452,19 @@ MUL_SY yvec1, yvec4, yvec6; MUL_SY yvec1, yvec5, yvec7; ADD_SY yvec10, yvec6, yvec10; ADD_SY yvec8, yvec7, yvec8; -.L2_bodyE:; +L(2_bodyE):; DECQ k; -JG .L2_bodyB; +JG L(2_bodyB); ALIGN_4 -.L2_loopE:; +L(2_loopE):; #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L3_loopE; +JLE L(3_loopE); ALIGN_4 -.L3_loobB: +L(3_loobB): #### Unroll times 1 #### MUL_SY yvec0, yvec2, yvec6; SHUF_SY $0x03, yvec2, yvec2, yvec4; @@ -526,15 +526,15 @@ MUL_SY yvec1, yvec4, yvec6; MUL_SY yvec1, yvec5, yvec7; ADD_SY yvec10, yvec6, yvec10; ADD_SY yvec8, yvec7, yvec8; -.L3_loopE: +L(3_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L4_loopE; +JLE L(4_loopE); ALIGN_4 -.L4_loopB:; +L(4_loopB):; #### Unroll times 1 #### MUL_SY yvec0, yvec2, yvec6; SHUF_SY $0x03, yvec2, yvec2, yvec4; @@ -564,7 +564,7 @@ MUL_SY yvec0, yvec5, yvec7; ADD_SY yvec10, yvec6, yvec10; ADD_SY yvec8, yvec7, yvec8; -.L4_loopE:; +L(4_loopE):; #### Load Alpha #### BROAD_SY MEMALPHA,yvec7; MUL_SY yvec7,yvec15,yvec15; @@ -591,7 +591,7 @@ REVS_SY $0xe4,yvec7,yvec8,yvec8; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L4_loopEx; +JNE L(4_loopEx); ALIGN_4 LEAQ (ldc,ldc,2),%rax; EXTRA_SY $1,yvec15,xvec7; @@ -648,12 +648,12 @@ ADDQ $8, kk #endif ADDQ $8*SIZE,C0; ADDQ $8*SIZE,C1; -.L1_bodyE:; +L(1_bodyE):; DECQ i; -JG .L1_bodyB; -JMP .L1_loopE; +JG L(1_bodyB); +JMP L(1_loopE); ALIGN_4; -.L4_loopEx: +L(4_loopEx): LEAQ (ldc,ldc,2),%rax; EXTRA_SY $1, yvec15, xvec7; #ifndef TRMMKERNEL @@ -795,13 +795,13 @@ ADDQ $8, kk ADDQ $8*SIZE, C0; ADDQ $8*SIZE, C1; DECQ i; -JG .L1_bodyB; +JG L(1_bodyB); ALIGN_4 -.L1_loopE:; +L(1_loopE):; TEST $4, bm; -JLE .L5_loopE; +JLE L(5_loopE); ALIGN_4 -.L5_bodyB: +L(5_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -839,9 +839,9 @@ ADDQ $8, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L8_loopE; +JLE L(8_loopE); ALIGN_4 -.L8_bodyB: +L(8_bodyB): #### Unroll time 1 #### SHUF_SX $0x4e, xvec2, xvec4; @@ -965,17 +965,17 @@ ODUP_SX 0*SIZE(ptrbb), xvec3; MUL_SX xvec1, xvec5, xvec5; ADD_SX xvec5, xvec8, xvec8; DECQ k; -JG .L8_bodyB; +JG L(8_bodyB); ALIGN_4 -.L8_loopE: +L(8_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L9_loopE; +JLE L(9_loopE); ALIGN_4 -.L9_bodyB: +L(9_bodyB): #### Unroll time 1 #### SHUF_SX $0x4e, xvec2, xvec4; MUL_SX xvec0, xvec2, xvec2; @@ -1038,15 +1038,15 @@ ODUP_SX 0*SIZE(ptrbb), xvec3; MUL_SX xvec1, xvec5, xvec5; ADD_SX xvec5, xvec8, xvec8; -.L9_loopE: +L(9_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L10_loopE; +JLE L(10_loopE); ALIGN_4 -.L10_bodyB: +L(10_bodyB): #### Unroll time 1 #### SHUF_SX $0x4e, xvec2, xvec4; MUL_SX xvec0, xvec2, xvec2; @@ -1076,7 +1076,7 @@ ADD_SX xvec4, xvec9, xvec9; MUL_SX xvec0, xvec5, xvec5; ADD_SX xvec5, xvec8, xvec8; -.L10_loopE: +L(10_loopE): #### Multiply Alpha #### BROAD_SX MEMALPHA, xvec7; MUL_SX xvec7, xvec15, xvec15; @@ -1104,7 +1104,7 @@ REVS_SX $0xe4, xvec7, xvec8, xvec8; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L10_loopEx; +JNE L(10_loopEx); ALIGN_4 LEAQ (ldc,ldc,2),%rax; #ifndef TRMMKERNEL @@ -1137,9 +1137,9 @@ ADDQ $4, kk #endif ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -JMP .L5_loopE; +JMP L(5_loopE); ALIGN_4 -.L10_loopEx: +L(10_loopEx): LEAQ (ldc,ldc,2),%rax; #ifndef TRMMKERNEL LDL_SX 0*SIZE(C0), xvec7, xvec7; @@ -1195,11 +1195,11 @@ ADDQ $4, kk #endif ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -.L5_loopE: +L(5_loopE): TEST $2, bm; -JLE .L6_loopE; +JLE L(6_loopE); ALIGN_4 -.L6_bodyB: +L(6_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -1231,9 +1231,9 @@ ADDQ $8, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L11_loopE; +JLE L(11_loopE); ALIGN_4 -.L11_bodyB: +L(11_bodyB): #### Computing kernel LD_SX 0*SIZE(ptrba), xvec0; # a1, a2, a3, a4 SHUF_SX $0x44, xvec0, xvec1; # a1, a2, a1, a2 @@ -1300,17 +1300,17 @@ ADD_SX xvec5, xvec12, xvec12; ADDQ $8*SIZE, ptrba; ADDQ $32*SIZE, ptrbb; DECQ k; -JG .L11_bodyB; +JG L(11_bodyB); ALIGN_4 -.L11_loopE: +L(11_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L12_loopE; +JLE L(12_loopE); ALIGN_4 -.L12_bodyB: +L(12_bodyB): LD_SX 0*SIZE(ptrba), xvec0; # a1, a2, a3, a4 SHUF_SX $0x44, xvec0, xvec1; # a1, a2, a1, a2 EDUP_SX 0*SIZE(ptrbb), xvec2; @@ -1344,15 +1344,15 @@ ADD_SX xvec5, xvec12, xvec12; ADDQ $4*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; -.L12_loopE: +L(12_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L13_loopE; +JLE L(13_loopE); ALIGN_4 -.L13_bodyB: +L(13_bodyB): LD_SX 0*SIZE(ptrba), xvec0; # a1, a2, a3, a4 SHUF_SX $0x44, xvec0, xvec1; # a1, a2, a1, a2 EDUP_SX 0*SIZE(ptrbb), xvec2; @@ -1370,7 +1370,7 @@ MUL_SX xvec1, xvec5, xvec5; ADD_SX xvec5, xvec12, xvec12; ADDQ $2*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L13_loopE: +L(13_loopE): LEAQ (ldc,ldc,2),%rax; #### Multiply Alpha #### BROAD_SX MEMALPHA, xvec7; @@ -1413,11 +1413,11 @@ ADDQ $2, kk ADDQ $2*SIZE, C0; ADDQ $2*SIZE, C1; #### Writing Back #### -.L6_loopE: +L(6_loopE): TEST $1, bm; -JLE .L7_loopE; +JLE L(7_loopE); ALIGN_4 -.L7_bodyB: +L(7_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -1447,9 +1447,9 @@ ADDQ $8, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L14_loopE; +JLE L(14_loopE); ALIGN_4 -.L14_bodyB: +L(14_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; MUL_SX xvec0, xvec2, xvec2; @@ -1485,17 +1485,17 @@ ADD_SX xvec5, xvec14, xvec14; ADDQ $4*SIZE, ptrba; ADDQ $32*SIZE, ptrbb; DECQ k; -JG .L14_bodyB; +JG L(14_bodyB); ALIGN_4 -.L14_loopE: +L(14_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L15_loopE; +JLE L(15_loopE); ALIGN_4 -.L15_bodyB: +L(15_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; MUL_SX xvec0, xvec2, xvec2; @@ -1514,15 +1514,15 @@ ADD_SX xvec5, xvec14, xvec14; ADDQ $2*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; -.L15_loopE: +L(15_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L16_loopE; +JLE L(16_loopE); ALIGN_4 -.L16_bodyB: +L(16_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; MUL_SX xvec0, xvec2, xvec2; @@ -1533,7 +1533,7 @@ ADD_SX xvec3, xvec14, xvec14; ADDQ $1, ptrba; ADDQ $4, ptrbb; -.L16_loopE: +L(16_loopE): BROAD_SX MEMALPHA, xvec7; MUL_SX xvec7, xvec15, xvec15; MUL_SX xvec7, xvec14, xvec14; @@ -1583,7 +1583,7 @@ ADDQ $1, kk ADDQ $1*SIZE, C0; ADDQ $1*SIZE, C1; #### Writing Back #### -.L7_loopE: +L(7_loopE): #if defined(TRMMKERNEL)&&!defined(LEFT) ADDQ $8, kk #endif @@ -1591,15 +1591,15 @@ MOVQ bk,k; SALQ $5,k; ADDQ k,bb; LEAQ (C,ldc,8),C; -.L0_bodyE:; +L(0_bodyE):; DECQ j; -JG .L0_bodyB; +JG L(0_bodyB); ALIGN_4; -.L0_loopE:; +L(0_loopE):; TEST $4, bn; # Rn = 4 -JLE .L20_loopE; +JLE L(20_loopE); ALIGN_4; -.L20_bodyB: +L(20_bodyB): #if defined(TRMMKERNEL) && defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk; @@ -1610,9 +1610,9 @@ LEAQ (C, ldc, 2), C1; MOVQ ba, ptrba; MOVQ bm, i; SARQ $3, i; -JLE .L21_loopE; +JLE L(21_loopE); ALIGN_4 -.L21_bodyB: +L(21_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -1650,9 +1650,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2,k; -JLE .L211_loopE; +JLE L(211_loopE); ALIGN_4 -.L211_bodyB: +L(211_bodyB): #### Unroll time 1 #### ODUP_SX 0*SIZE(ptrbb), xvec3; SHUF_SX $0x4e, xvec2, xvec4; @@ -1782,17 +1782,17 @@ MUL_SX xvec1, xvec7, xvec7; ADD_SX xvec7, xvec8, xvec8; LD_SX 4*SIZE(ptrba), xvec1; DECQ k; -JG .L211_bodyB; +JG L(211_bodyB); ALIGN_4 -.L211_loopE: +L(211_loopE): #ifndef TRMMKERNEL TEST $2, bk #else TEST $2, kkk; #endif -JLE .L212_loopE; +JLE L(212_loopE); ALIGN_4 -.L212_bodyB: +L(212_bodyB): #### Unroll time 1 #### ODUP_SX 0*SIZE(ptrbb), xvec3; SHUF_SX $0x4e, xvec2, xvec4; @@ -1858,15 +1858,15 @@ MUL_SX xvec1, xvec7, xvec7; ADD_SX xvec7, xvec8, xvec8; LD_SX 4*SIZE(ptrba), xvec1; -.L212_loopE: +L(212_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L213_loopE; +JLE L(213_loopE); ALIGN_4 -.L213_bodyB: +L(213_bodyB): ODUP_SX 0*SIZE(ptrbb), xvec3; SHUF_SX $0x4e, xvec2, xvec4; MOV_SX xvec2, xvec6; @@ -1897,7 +1897,7 @@ ADD_SX xvec5, xvec9, xvec9; MUL_SX xvec1, xvec7, xvec7; ADD_SX xvec7, xvec8, xvec8; -.L213_loopE: +L(213_loopE): #### Multiply Alpha #### BROAD_SX MEMALPHA, xvec7; MUL_SX xvec7, xvec15, xvec15; @@ -1964,13 +1964,13 @@ ADDQ $8, kk ADDQ $8*SIZE, C0; ADDQ $8*SIZE, C1; DECQ i; -JG .L21_bodyB; +JG L(21_bodyB); ALIGN_4 -.L21_loopE: +L(21_loopE): TEST $4, bm; -JLE .L22_loopE; +JLE L(22_loopE); ALIGN_4 -.L22_bodyB: +L(22_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -2001,9 +2001,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L221_loopE; +JLE L(221_loopE); ALIGN_4 -.L221_bodyB: +L(221_bodyB): LD_SX 0*SIZE(ptrba), xvec0; EDUP_SX 0*SIZE(ptrbb), xvec2; ODUP_SX 0*SIZE(ptrbb), xvec3; @@ -2071,17 +2071,17 @@ ADDQ $16*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; DECQ k; -JG .L221_bodyB; +JG L(221_bodyB); ALIGN_4 -.L221_loopE: +L(221_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L222_loopE; +JLE L(222_loopE); ALIGN_4 -.L222_bodyB: +L(222_bodyB): LD_SX 0*SIZE(ptrba), xvec0; EDUP_SX 0*SIZE(ptrbb), xvec2; ODUP_SX 0*SIZE(ptrbb), xvec3; @@ -2115,15 +2115,15 @@ MUL_SX xvec1, xvec5, xvec5; ADD_SX xvec5, xvec12, xvec12; ADDQ $8*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L222_loopE: +L(222_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L223_loopE; +JLE L(223_loopE); ALIGN_4 -.L223_bodyB: +L(223_bodyB): LD_SX 0*SIZE(ptrba), xvec0; EDUP_SX 0*SIZE(ptrbb), xvec2; ODUP_SX 0*SIZE(ptrbb), xvec3; @@ -2141,7 +2141,7 @@ MUL_SX xvec0, xvec5, xvec5; ADD_SX xvec5, xvec12, xvec12; ADDQ $4*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L223_loopE: +L(223_loopE): #### Multiply Alpha #### BROAD_SX MEMALPHA, xvec7; MUL_SX xvec7, xvec15, xvec15; @@ -2183,11 +2183,11 @@ ADDQ $4, kk #endif ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -.L22_loopE: +L(22_loopE): TEST $2, bm; -JLE .L23_loopE; +JLE L(23_loopE); ALIGN_4 -.L23_bodyB: +L(23_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -2216,9 +2216,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L231_loopE; +JLE L(231_loopE); ALIGN_4 -.L231_bodyB: +L(231_bodyB): LD_SX 0*SIZE(ptrba), xvec0; EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; @@ -2256,17 +2256,17 @@ ADD_SX xvec7, xvec14, xvec14; ADDQ $8*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; DECQ k; -JG .L231_bodyB; +JG L(231_bodyB); ALIGN_4 -.L231_loopE: +L(231_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L232_loopE; +JLE L(232_loopE); ALIGN_4 -.L232_bodyB: +L(232_bodyB): LD_SX 0*SIZE(ptrba), xvec0; EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; @@ -2286,15 +2286,15 @@ ADD_SX xvec7, xvec14, xvec14; ADDQ $4*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L232_loopE: +L(232_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L233_loopE; +JLE L(233_loopE); ALIGN_4 -.L233_bodyB: +L(233_bodyB): LD_SX 0*SIZE(ptrba), xvec0; EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; @@ -2306,7 +2306,7 @@ ADD_SX xvec5, xvec14, xvec14; ADDQ $2*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L233_loopE: +L(233_loopE): #### Multiply Alpha #### BROAD_SY MEMALPHA, yvec7; MUL_SY xvec7, xvec15, xvec15; @@ -2336,11 +2336,11 @@ ADDQ $2, kk #endif ADDQ $2*SIZE, C0; ADDQ $2*SIZE, C1; -.L23_loopE: +L(23_loopE): TEST $1, bm; -JLE .L24_loopE; +JLE L(24_loopE); ALIGN_4 -.L24_bodyB: +L(24_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -2368,9 +2368,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L241_loopE; +JLE L(241_loopE); ALIGN_4 -.L241_bodyB: +L(241_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec1; MUL_SX xvec0, xvec1, xvec1; @@ -2393,17 +2393,17 @@ ADD_SX xvec7, xvec15, xvec15; ADDQ $4*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; DECQ k; -JG .L241_bodyB; +JG L(241_bodyB); ALIGN_4 -.L241_loopE: +L(241_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L242_loopE; +JLE L(242_loopE); ALIGN_4 -.L242_bodyB: +L(242_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec1; MUL_SX xvec0, xvec1, xvec1; @@ -2416,22 +2416,22 @@ ADD_SX xvec3, xvec15, xvec15; ADDQ $2*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L242_loopE: +L(242_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L243_loopE; +JLE L(243_loopE); ALIGN_4; -.L243_bodyB: +L(243_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec1; MUL_SX xvec0, xvec1, xvec1; ADD_SX xvec1, xvec15, xvec15; ADDQ $1*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L243_loopE: +L(243_loopE): #### Multiply Alpha #### BROAD_SX MEMALPHA, xvec7; MUL_SX xvec7, xvec15, xvec15; @@ -2463,7 +2463,7 @@ ADDQ $1, kk #endif ADDQ $1*SIZE, C0; ADDQ $1*SIZE, C1; -.L24_loopE: +L(24_loopE): #if defined(TRMMKERNEL)&&!defined(LEFT) ADDQ $4, kk #endif @@ -2471,11 +2471,11 @@ MOVQ bk, k; SALQ $4, k; ADDQ k, bb; LEAQ (C, ldc, 4), C; -.L20_loopE: +L(20_loopE): TEST $2, bn; -JLE .L30_loopE; +JLE L(30_loopE); ALIGN_4 -.L30_bodyB: +L(30_bodyB): #if defined(TRMMKERNEL) && defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk @@ -2485,9 +2485,9 @@ LEAQ (C, ldc, 1), C1; MOVQ ba, ptrba; MOVQ bm, i; SARQ $3, i; -JLE .L31_loopE; +JLE L(31_loopE); ALIGN_4 -.L31_bodyB: +L(31_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -2518,9 +2518,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L311_loopE; +JLE L(311_loopE); ALIGN_4 -.L311_bodyB: +L(311_bodyB): LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0x50, xvec2, xvec3; LD_SX 0*SIZE(ptrba), xvec0; @@ -2594,17 +2594,17 @@ ADD_SX xvec6, xvec12, xvec12; ADDQ $32*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L311_bodyB; +JG L(311_bodyB); ALIGN_4 -.L311_loopE: +L(311_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L312_loopE; +JLE L(312_loopE); ALIGN_4 -.L312_bodyB: +L(312_bodyB): LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0x50, xvec2, xvec3; LD_SX 0*SIZE(ptrba), xvec0; @@ -2642,15 +2642,15 @@ ADD_SX xvec6, xvec12, xvec12; ADDQ $16*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L312_loopE: +L(312_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L313_loopE; +JLE L(313_loopE); ALIGN_4 -.L313_bodyB: +L(313_bodyB): LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0x50, xvec2, xvec3; LD_SX 0*SIZE(ptrba), xvec0; @@ -2671,7 +2671,7 @@ ADD_SX xvec6, xvec12, xvec12; ADDQ $8*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L313_loopE: +L(313_loopE): BROAD_SX MEMALPHA, xvec7; MUL_SX xvec7, xvec15, xvec15; MUL_SX xvec7, xvec14, xvec14; @@ -2713,13 +2713,13 @@ ADDQ $8, kk ADDQ $8*SIZE, C0; ADDQ $8*SIZE, C1; DECQ i; -JG .L31_bodyB; +JG L(31_bodyB); ALIGN_4 -.L31_loopE: +L(31_loopE): TEST $4, bm; -JLE .L32_loopE; +JLE L(32_loopE); ALIGN_4 -.L32_bodyB: +L(32_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -2748,9 +2748,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L321_loopE; +JLE L(321_loopE); ALIGN_4 -.L321_bodyB: +L(321_bodyB): LD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0x50, xvec2, xvec3; @@ -2788,17 +2788,17 @@ ADD_SX xvec6, xvec14, xvec14; ADDQ $16*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L321_bodyB; +JG L(321_bodyB); ALIGN_4 -.L321_loopE: +L(321_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L322_loopE; +JLE L(322_loopE); ALIGN_4 -.L322_bodyB: +L(322_bodyB): LD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0x50, xvec2, xvec3; @@ -2818,15 +2818,15 @@ ADD_SX xvec6, xvec14, xvec14; ADDQ $8*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L322_loopE: +L(322_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L323_loopE; +JLE L(323_loopE); ALIGN_4 -.L323_bodyB: +L(323_bodyB): LD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0x50, xvec2, xvec3; @@ -2838,7 +2838,7 @@ ADD_SX xvec4, xvec14, xvec14; ADDQ $4*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L323_loopE: +L(323_loopE): BROAD_SX MEMALPHA, xvec7; MUL_SX xvec7, xvec15, xvec15; MUL_SX xvec7, xvec14, xvec14; @@ -2867,11 +2867,11 @@ ADDQ $4, kk #endif ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -.L32_loopE: +L(32_loopE): TEST $2, bm; -JLE .L33_loopE; +JLE L(33_loopE); ALIGN_4 -.L33_bodyB: +L(33_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -2902,9 +2902,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L331_loopE; +JLE L(331_loopE); ALIGN_4 -.L331_bodyB: +L(331_bodyB): LD_SX 0*SIZE(ptrba), xvec0; # a0, a1, a2, a3 EDUP_SX 0*SIZE(ptrbb), xvec2; # b0, b0, b2, b2 ODUP_SX 0*SIZE(ptrbb), xvec3; # b1, b1, b3, b3 @@ -2925,17 +2925,17 @@ ADD_SX xvec3, xvec14, xvec14; ADDQ $8*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L331_bodyB; +JG L(331_bodyB); ALIGN_4 -.L331_loopE: +L(331_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L332_loopE; +JLE L(332_loopE); ALIGN_4 -.L332_bodyB: +L(332_bodyB): LD_SX 0*SIZE(ptrba), xvec0; # a0, a1, a2, a3 EDUP_SX 0*SIZE(ptrbb), xvec2; # b0, b0, b2, b2 ODUP_SX 0*SIZE(ptrbb), xvec3; # b1, b1, b3, b3 @@ -2948,15 +2948,15 @@ ADD_SX xvec3, xvec14, xvec14; ADDQ $4*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L332_loopE: +L(332_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L333_loopE; +JLE L(333_loopE); ALIGN_4 -.L333_bodyB: +L(333_bodyB): movss 0*SIZE(ptrba), xvec0; movss 1*SIZE(ptrba), xvec1; movss 0*SIZE(ptrbb), xvec2; @@ -2979,7 +2979,7 @@ ADD_SX xvec7, xvec14, xvec14 ADDQ $2*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L333_loopE: +L(333_loopE): BROAD_SX MEMALPHA, xvec7; MUL_SX xvec7, xvec15, xvec15; MUL_SX xvec7, xvec14, xvec14; @@ -3011,11 +3011,11 @@ ADDQ $2, kk; ADDQ $2*SIZE, C0; ADDQ $2*SIZE, C1; #### Writing Back #### -.L33_loopE: +L(33_loopE): TEST $1, bm; -JLE .L34_loopE; +JLE L(34_loopE); ALIGN_4 -.L34_bodyB: +L(34_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -3044,9 +3044,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L341_loopE; +JLE L(341_loopE); ALIGN_4 -.L341_bodyB: +L(341_bodyB): movss 0*SIZE(ptrba), xvec0; movss 0*SIZE(ptrbb), xvec1; mulss xvec0, xvec1; @@ -3086,17 +3086,17 @@ addss xvec2, xvec14; addq $4*SIZE, ptrba; addq $8*SIZE, ptrbb; decq k; -jg .L341_bodyB; +jg L(341_bodyB); ALIGN_4 -.L341_loopE: +L(341_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L342_loopE; +JLE L(342_loopE); ALIGN_4 -.L342_bodyB: +L(342_bodyB): movss 0*SIZE(ptrba), xvec0; movss 0*SIZE(ptrbb), xvec1; mulss xvec0, xvec1; @@ -3116,15 +3116,15 @@ mulss xvec0, xvec2; addss xvec2, xvec14; addq $2*SIZE, ptrba; addq $4*SIZE, ptrbb; -.L342_loopE: +L(342_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L343_loopE; +JLE L(343_loopE); ALIGN_4 -.L343_bodyB: +L(343_bodyB): movss 0*SIZE(ptrba), xvec0; movss 0*SIZE(ptrbb), xvec1; mulss xvec0, xvec1; @@ -3136,7 +3136,7 @@ addss xvec2, xvec14; addq $1*SIZE, ptrba; addq $2*SIZE, ptrbb -.L343_loopE: +L(343_loopE): #### Writing back #### movss MEMALPHA, xvec7; mulss xvec7, xvec15; @@ -3161,7 +3161,7 @@ ADDQ $1, kk; #endif addq $1*SIZE, C0; addq $1*SIZE, C1; -.L34_loopE: +L(34_loopE): #if defined(TRMMKERNEL) && !defined(LEFT) ADDQ $2, kk; #endif @@ -3169,11 +3169,11 @@ MOVQ bk, k; SALQ $3, k; ADDQ k, bb; LEAQ (C, ldc, 2), C; -.L30_loopE: +L(30_loopE): TEST $1, bn; -JLE .L40_loopE; +JLE L(40_loopE); ALIGN_4 -.L40_bodyB: +L(40_bodyB): #if defined(TRMMKERNEL)&&defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk; @@ -3182,9 +3182,9 @@ MOVQ C, C0; MOVQ ba, ptrba; MOVQ bm, i; SARQ $3, i; -JLE .L41_loopE; +JLE L(41_loopE); ALIGN_4 -.L41_bodyB: +L(41_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -3212,9 +3212,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L411_loopE; +JLE L(411_loopE); ALIGN_4 -.L411_bodyB: +L(411_bodyB): LD_SY 0*SIZE(ptrba), yvec0; BROAD_SY 0*SIZE(ptrbb), yvec1; MUL_SY yvec0, yvec1, yvec2; @@ -3238,17 +3238,17 @@ ADD_SY yvec2, yvec15, yvec15; ADDQ $32*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; DECQ k; -JG .L411_bodyB; +JG L(411_bodyB); ALIGN_4 -.L411_loopE: +L(411_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L412_loopE; +JLE L(412_loopE); ALIGN_4 -.L412_bodyB: +L(412_bodyB): LD_SY 0*SIZE(ptrba), yvec0; BROAD_SY 0*SIZE(ptrbb), yvec1; MUL_SY yvec0, yvec1, yvec2; @@ -3261,22 +3261,22 @@ ADD_SY yvec2, yvec15, yvec15; ADDQ $16*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L412_loopE: +L(412_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L413_loopE; +JLE L(413_loopE); ALIGN_4 -.L413_bodyB: +L(413_bodyB): LD_SY 0*SIZE(ptrba), yvec0; BROAD_SY 0*SIZE(ptrbb), yvec1; MUL_SY yvec0, yvec1, yvec2; ADD_SY yvec2, yvec15, yvec15; ADDQ $8*SIZE, ptrba; ADDQ $1*SIZE, ptrbb; -.L413_loopE: +L(413_loopE): #### Writing #### BROAD_SY MEMALPHA, yvec7; MUL_SY yvec7, yvec15, yvec15; @@ -3311,13 +3311,13 @@ ADDQ $8, kk; #endif ADDQ $8*SIZE, C0; DECQ i; -JG .L41_bodyB; +JG L(41_bodyB); ALIGN_4 -.L41_loopE: +L(41_loopE): TEST $4, bm; -JLE .L42_loopE; +JLE L(42_loopE); ALIGN_4 -.L42_bodyB: +L(42_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -3344,9 +3344,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk #endif SARQ $2, k; -JLE .L421_loopE; +JLE L(421_loopE); ALIGN_4 -.L421_bodyB: +L(421_bodyB): LD_SX 0*SIZE(ptrba), xvec0; BROAD_SX 0*SIZE(ptrbb), xvec1; MUL_SX xvec0, xvec1, xvec1; @@ -3369,17 +3369,17 @@ ADD_SX xvec1, xvec15, xvec15; ADDQ $16*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; DECQ k; -JG .L421_bodyB; +JG L(421_bodyB); ALIGN_4 -.L421_loopE: +L(421_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L422_loopE; +JLE L(422_loopE); ALIGN_4 -.L422_bodyB: +L(422_bodyB): LD_SX 0*SIZE(ptrba), xvec0; BROAD_SX 0*SIZE(ptrbb), xvec1; MUL_SX xvec0, xvec1, xvec1; @@ -3392,15 +3392,15 @@ ADD_SX xvec1, xvec15, xvec15; ADDQ $8*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L422_loopE: +L(422_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L423_loopE; +JLE L(423_loopE); ALIGN_4 -.L423_bodyB: +L(423_bodyB): LD_SX 0*SIZE(ptrba), xvec0; BROAD_SX 0*SIZE(ptrbb), xvec1; MUL_SX xvec0, xvec1, xvec1; @@ -3408,7 +3408,7 @@ ADD_SX xvec1, xvec15, xvec15; ADDQ $4*SIZE, ptrba; ADDQ $1*SIZE, ptrbb; -.L423_loopE: +L(423_loopE): #### Writing back #### BROAD_SX MEMALPHA, xvec7; MUL_SX xvec7, xvec15, xvec15; @@ -3431,11 +3431,11 @@ ADDQ $4, kk #endif ADDQ $4*SIZE, C0; -.L42_loopE: +L(42_loopE): TEST $2, bm; -JLE .L43_loopE; +JLE L(43_loopE); ALIGN_4 -.L43_bodyB: +L(43_bodyB): #if !defined(TRMMKERNEL) || (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -3463,9 +3463,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L431_loopE; +JLE L(431_loopE); ALIGN_4 -.L431_bodyB: +L(431_bodyB): vmovss 0*SIZE(ptrba), xvec0; vmovss 1*SIZE(ptrba), xvec1; vmovss 0*SIZE(ptrbb), xvec2; @@ -3500,17 +3500,17 @@ vaddss xvec4, xvec14, xvec14; addq $8*SIZE, ptrba; addq $4*SIZE, ptrbb; decq k; -JG .L431_bodyB; +JG L(431_bodyB); ALIGN_4 -.L431_loopE: +L(431_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L432_loopE; +JLE L(432_loopE); ALIGN_4 -.L432_bodyB: +L(432_bodyB): vmovss 0*SIZE(ptrba), xvec0; vmovss 1*SIZE(ptrba), xvec1; vmovss 0*SIZE(ptrbb), xvec2; @@ -3529,15 +3529,15 @@ vaddss xvec4, xvec14, xvec14; addq $4*SIZE, ptrba; addq $2*SIZE, ptrbb; -.L432_loopE: +L(432_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L433_loopE; +JLE L(433_loopE); ALIGN_4 -.L433_bodyB: +L(433_bodyB): vmovss 0*SIZE(ptrba), xvec0; vmovss 1*SIZE(ptrba), xvec1; vmovss 0*SIZE(ptrbb), xvec2; @@ -3548,7 +3548,7 @@ vaddss xvec1, xvec14, xvec14; addq $2*SIZE, ptrba; addq $1*SIZE, ptrbb; -.L433_loopE: +L(433_loopE): #### Writing Back #### vmovss MEMALPHA, xvec7; vmulss xvec7, xvec15, xvec15; @@ -3572,11 +3572,11 @@ addq $2, kk #endif addq $2*SIZE, C0; -.L43_loopE: +L(43_loopE): TEST $1, bm; -JLE .L44_loopE; +JLE L(44_loopE); ALIGN_4 -.L44_bodyB: +L(44_bodyB): #if !defined(TRMMKERNEL) || (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -3603,9 +3603,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L441_loopE; +JLE L(441_loopE); ALIGN_4 -.L441_bodyB: +L(441_bodyB): vmovss 0*SIZE(ptrba), xvec0; vmovss 0*SIZE(ptrbb), xvec1; vmulss xvec0, xvec1, xvec1; @@ -3628,17 +3628,17 @@ vaddss xvec1, xvec15, xvec15; addq $4*SIZE, ptrba; addq $4*SIZE, ptrbb; decq k; -JG .L441_bodyB; +JG L(441_bodyB); ALIGN_4 -.L441_loopE: +L(441_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L442_loopE; +JLE L(442_loopE); ALIGN_4 -.L442_bodyB: +L(442_bodyB): vmovss 0*SIZE(ptrba), xvec0; vmovss 0*SIZE(ptrbb), xvec1; vmulss xvec0, xvec1, xvec1; @@ -3651,15 +3651,15 @@ vaddss xvec1, xvec15, xvec15; addq $2*SIZE, ptrba; addq $2*SIZE, ptrbb; -.L442_loopE: +L(442_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L443_loopE; +JLE L(443_loopE); ALIGN_4 -.L443_bodyB: +L(443_bodyB): vmovss 0*SIZE(ptrba), xvec0; vmovss 0*SIZE(ptrbb), xvec1; vmulss xvec0, xvec1, xvec1; @@ -3667,7 +3667,7 @@ vaddss xvec1, xvec15, xvec15; addq $1*SIZE, ptrba; addq $1*SIZE, ptrbb; -.L443_loopE: +L(443_loopE): #### Writing Back #### vmovss MEMALPHA, xvec7; vmulss xvec7, xvec15, xvec15; @@ -3687,13 +3687,13 @@ addq $1, kk #endif addq $1*SIZE, C0; -.L44_loopE: +L(44_loopE): MOV bk, k; SALQ $2, k; ADDQ k, bb; ADDQ ldc, C; -.L40_loopE: +L(40_loopE): movq 0(%rsp), %rbx; movq 8(%rsp), %rbp; movq 16(%rsp), %r12; diff --git a/kernel/x86_64/sgemv_n.S b/kernel/x86_64/sgemv_n.S index 8f64fe5e2d..95d92cb757 100644 --- a/kernel/x86_64/sgemv_n.S +++ b/kernel/x86_64/sgemv_n.S @@ -160,19 +160,19 @@ movq LDA,LDAX movq X,XX movq STACK_Y, Y -.L0t: +L(0t): xorq I,I addq $1,I salq $22,I subq I,MMM movq I,M - jge .L00t + jge L(00t) movq MMM,M addq I,M - jle .L999x + jle L(999x) -.L00t: +L(00t): movq AA,A movq NN,N movq LDAX,LDA @@ -191,9 +191,9 @@ #ifdef ALIGNED_ACCESS movq M, MM testq $4 * SIZE - 1, A - je .L0X + je L(0X) cmpq $3, M - jle .L0X + jle L(0X) movq A, MM sarq $BASE_SHIFT, MM @@ -201,13 +201,13 @@ subq $4, MM addq M, MM -.L0X: +L(0X): #endif testq N, N # if n <= 0 goto END - jle .L999 + jle L(999) testq M, M # if n <= 0 goto END - jle .L999 + jle L(999) subq $-32 * SIZE, A @@ -224,33 +224,33 @@ sarq $4, %rax ALIGN_3 -.L01: +L(01): movaps %xmm0, 0 * SIZE(Y1) movaps %xmm0, 4 * SIZE(Y1) movaps %xmm0, 8 * SIZE(Y1) movaps %xmm0, 12 * SIZE(Y1) addq $16 * SIZE, Y1 decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L10: +L(10): #ifdef ALIGNED_ACCESS movq A, %rax andq $4 * SIZE - 1, %rax addq %rax, BUFFER testq $4 * SIZE - 1, LDA - jne .L100 + jne L(100) #endif #if GEMV_UNROLL >= 8 cmpq $8, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $8, N leaq 32 * SIZE(BUFFER), Y1 @@ -297,10 +297,10 @@ #ifdef ALIGNED_ACCESS cmpq $3, M - jle .L17 + jle L(17) testq $SIZE, A1 - je .L1X + je L(1X) movss -32 * SIZE(A1), %xmm4 movss -32 * SIZE(A1, LDA, 1), %xmm5 @@ -338,9 +338,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L1X: +L(1X): testq $2 * SIZE, A1 - je .L1XX + je L(1XX) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA, 1), %xmm5 @@ -378,11 +378,11 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L1XX: +L(1XX): #endif movq MM, I sarq $4, I - jle .L15 + jle L(15) MOVUPS_A1 (-32 * SIZE, A1, %xmm4) MOVUPS_A1 (-28 * SIZE, A1, %xmm5) @@ -395,10 +395,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L14 + jle L(14) ALIGN_3 -.L13: +L(13): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -555,10 +555,10 @@ subq $1, I BRANCH - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): mulps %xmm8, %xmm4 addps %xmm4, %xmm0 MOVUPS_A2 (-32 * SIZE, A1, LDA, 1, %xmm4) @@ -669,9 +669,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L15: +L(15): testq $8, MM - je .L16 + je L(16) MOVUPS_A1 (-32 * SIZE, A1, %xmm4) MOVUPS_A1 (-28 * SIZE, A1, %xmm5) @@ -741,9 +741,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L16: +L(16): testq $4, MM - je .L17 + je L(17) MOVUPS_A1 (-32 * SIZE, A1, %xmm4) MOVUPS_A2 (-32 * SIZE, A1, LDA, 1, %xmm5) @@ -781,9 +781,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L17: +L(17): testq $2, MM - je .L18 + je L(18) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA, 1), %xmm5 @@ -821,9 +821,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L18: +L(18): testq $1, MM - je .L19 + je L(19) movss -32 * SIZE(A1), %xmm4 movss -32 * SIZE(A1, LDA, 1), %xmm5 @@ -857,21 +857,21 @@ movss %xmm0, -32 * SIZE(Y1) ALIGN_3 -.L19: +L(19): cmpq $8, N - jge .L11 + jge L(11) ALIGN_3 -.L20: +L(20): #endif cmpq $4, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 4 ALIGN_3 -.L21: +L(21): #endif subq $4, N @@ -903,10 +903,10 @@ #ifdef ALIGNED_ACCESS cmpq $3, M - jle .L27 + jle L(27) testq $SIZE, A1 - je .L2X + je L(2X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(A1, LDA), %xmm1 @@ -931,9 +931,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L2X: +L(2X): testq $2 * SIZE, A1 - je .L2XX + je L(2XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A1, LDA), %xmm1 @@ -958,12 +958,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L2XX: +L(2XX): #endif movq MM, I sarq $4, I - jle .L25 + jle L(25) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) MOVUPS_A1 (-28 * SIZE, A1, %xmm9) @@ -981,10 +981,10 @@ MOVUPS_A2(-20 * SIZE, A1, LDA, 1, %xmm7) decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -1073,10 +1073,10 @@ subq $1, I BRANCH - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): mulps %xmm12, %xmm8 addps %xmm8, %xmm0 MOVUPS_A1(-32 * SIZE, A2, %xmm8) @@ -1130,9 +1130,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L25: +L(25): testq $8, MM - je .L26 + je L(26) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) MOVUPS_A1 (-28 * SIZE, A1, %xmm9) @@ -1176,9 +1176,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L26: +L(26): testq $4, MM - je .L27 + je L(27) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) @@ -1206,9 +1206,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L27: +L(27): testq $2, MM - je .L28 + je L(28) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A1, LDA), %xmm1 @@ -1233,12 +1233,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L28: +L(28): testq $1, MM #if GEMV_UNROLL == 4 - je .L29 + je L(29) #else - je .L30 + je L(30) #endif movss -32 * SIZE(Y1), %xmm8 @@ -1261,18 +1261,18 @@ ALIGN_3 #if GEMV_UNROLL == 4 -.L29: +L(29): cmpq $4, N - jge .L21 + jge L(21) #endif ALIGN_3 -.L30: +L(30): testq N, N - jle .L990 + jle L(990) cmpq $3, N - jne .L40 + jne L(40) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -1297,10 +1297,10 @@ #ifdef ALIGNED_ACCESS cmpq $3, M - jle .L37 + jle L(37) testq $SIZE, A1 - je .L3X + je L(3X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(A1, LDA), %xmm1 @@ -1322,9 +1322,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L3X: +L(3X): testq $2 * SIZE, A1 - je .L3XX + je L(3XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A1, LDA), %xmm1 @@ -1346,12 +1346,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L3XX: +L(3XX): #endif movq MM, I sarq $4, I - jle .L35 + jle L(35) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) MOVUPS_A1 (-28 * SIZE, A1, %xmm9) @@ -1369,10 +1369,10 @@ MOVUPS_A2(-20 * SIZE, A1, LDA, 1, %xmm7) decq I - jle .L34 + jle L(34) ALIGN_3 -.L33: +L(33): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 3 - 128 + PREOFFSET(A1) #endif @@ -1444,10 +1444,10 @@ subq $1, I BRANCH - jg .L33 + jg L(33) ALIGN_3 -.L34: +L(34): mulps %xmm12, %xmm8 addps %xmm8, %xmm0 MOVUPS_A1(-32 * SIZE, A2, %xmm8) @@ -1488,9 +1488,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L35: +L(35): testq $8, MM - je .L36 + je L(36) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) MOVUPS_A1 (-28 * SIZE, A1, %xmm9) @@ -1528,9 +1528,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L36: +L(36): testq $4, MM - je .L37 + je L(37) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) @@ -1555,9 +1555,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L37: +L(37): testq $2, MM - je .L38 + je L(38) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A1, LDA), %xmm1 @@ -1579,9 +1579,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L38: +L(38): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm8 @@ -1597,12 +1597,12 @@ addss %xmm2, %xmm8 movss %xmm8, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_3 -.L40: +L(40): cmpq $2, N - jne .L50 + jne L(50) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -1624,10 +1624,10 @@ #ifdef ALIGNED_ACCESS cmpq $3, M - jle .L47 + jle L(47) testq $SIZE, A1 - je .L4X + je L(4X) movss -32 * SIZE(Y1), %xmm8 @@ -1646,9 +1646,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L4X: +L(4X): testq $2 * SIZE, A1 - je .L4XX + je L(4XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A2), %xmm1 @@ -1667,12 +1667,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L4XX: +L(4XX): #endif movq MM, I sarq $4, I - jle .L45 + jle L(45) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) MOVUPS_A1 (-28 * SIZE, A1, %xmm9) @@ -1690,10 +1690,10 @@ MOVUPS_A1(-20 * SIZE, A2, %xmm7) decq I - jle .L44 + jle L(44) ALIGN_3 -.L43: +L(43): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -1748,10 +1748,10 @@ subq $1, I BRANCH - jg .L43 + jg L(43) ALIGN_3 -.L44: +L(44): mulps %xmm12, %xmm8 addps %xmm8, %xmm0 mulps %xmm12, %xmm9 @@ -1779,9 +1779,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L45: +L(45): testq $8, MM - je .L46 + je L(46) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) MOVUPS_A1 (-28 * SIZE, A1, %xmm9) @@ -1808,9 +1808,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L46: +L(46): testq $4, MM - je .L47 + je L(47) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) @@ -1828,9 +1828,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L47: +L(47): testq $2, MM - je .L48 + je L(48) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A2), %xmm1 @@ -1849,9 +1849,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L48: +L(48): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm8 @@ -1864,12 +1864,12 @@ addss %xmm1, %xmm8 movss %xmm8, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_3 -.L50: +L(50): cmpq $1, N - jne .L990 + jne L(990) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -1881,10 +1881,10 @@ #ifdef ALIGNED_ACCESS cmpq $3, M - jle .L57 + jle L(57) testq $SIZE, A1 - je .L5X + je L(5X) movss -32 * SIZE(Y1), %xmm8 movss -32 * SIZE(A1), %xmm0 @@ -1899,9 +1899,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L5X: +L(5X): testq $2 * SIZE, A1 - je .L5XX + je L(5XX) movsd -32 * SIZE(Y1), %xmm8 movsd -32 * SIZE(A1), %xmm0 @@ -1916,12 +1916,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L5XX: +L(5XX): #endif movq MM, I sarq $4, I - jle .L55 + jle L(55) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) MOVUPS_A1 (-28 * SIZE, A1, %xmm9) @@ -1934,10 +1934,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L54 + jle L(54) ALIGN_3 -.L53: +L(53): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -1974,10 +1974,10 @@ subq $1, I BRANCH - jg .L53 + jg L(53) ALIGN_3 -.L54: +L(54): mulps %xmm12, %xmm8 addps %xmm8, %xmm0 MOVUPS_YS1(-32 * SIZE, Y1, %xmm0) @@ -1995,9 +1995,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L55: +L(55): testq $8, MM - je .L56 + je L(56) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) MOVUPS_A1 (-28 * SIZE, A1, %xmm9) @@ -2016,9 +2016,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L56: +L(56): testq $4, MM - je .L57 + je L(57) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) MOVUPS_YL1(-32 * SIZE, Y1, %xmm0) @@ -2031,9 +2031,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L57: +L(57): testq $2, MM - je .L58 + je L(58) movsd -32 * SIZE(Y1), %xmm8 movsd -32 * SIZE(A1), %xmm0 @@ -2047,9 +2047,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L58: +L(58): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm8 movss -32 * SIZE(A1), %xmm0 @@ -2060,18 +2060,18 @@ movss %xmm8, -32 * SIZE(Y1) #ifdef ALIGNED_ACCESS - jmp .L990 + jmp L(990) ALIGN_3 -.L100: +L(100): testq $2 * SIZE - 1, LDA - jne .L200 + jne L(200) cmpq $4, N - jl .L110 + jl L(110) ALIGN_3 -.L101: +L(101): subq $4, N leaq 32 * SIZE(BUFFER), Y1 @@ -2101,10 +2101,10 @@ shufps $0, %xmm15, %xmm15 cmpq $3, M - jle .L107 + jle L(107) testq $SIZE, A1 - je .L10X + je L(10X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(A1, LDA), %xmm1 @@ -2129,9 +2129,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L10X: +L(10X): testq $2 * SIZE, A1 - je .L10XX + je L(10XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A1, LDA), %xmm1 @@ -2156,13 +2156,13 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L10XX: +L(10XX): movhps -32 * SIZE(A1, LDA), %xmm8 movhps -32 * SIZE(A2, LDA), %xmm9 movq MM, I sarq $4, I - jle .L105 + jle L(105) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -2174,10 +2174,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L104 + jle L(104) ALIGN_3 -.L103: +L(103): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -2274,10 +2274,10 @@ subq $1, I BRANCH - jg .L103 + jg L(103) ALIGN_3 -.L104: +L(104): mulps %xmm12, %xmm4 addps %xmm4, %xmm0 movaps -20 * SIZE(A1), %xmm7 @@ -2345,9 +2345,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L105: +L(105): testq $8, MM - je .L106 + je L(106) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -2395,9 +2395,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L106: +L(106): testq $4, MM - je .L107 + je L(107) movaps -32 * SIZE(A1), %xmm4 movaps -30 * SIZE(A1, LDA), %xmm5 @@ -2424,9 +2424,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L107: +L(107): testq $2, MM - je .L108 + je L(108) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA), %xmm5 @@ -2451,9 +2451,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L108: +L(108): testq $1, MM - je .L109 + je L(109) movss -32 * SIZE(Y1), %xmm0 @@ -2474,17 +2474,17 @@ movss %xmm0, -32 * SIZE(Y1) ALIGN_3 -.L109: +L(109): cmpq $4, N - jge .L101 + jge L(101) ALIGN_3 -.L110: +L(110): testq N, N - jle .L990 + jle L(990) cmpq $3, N - jne .L120 + jne L(120) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -2509,10 +2509,10 @@ shufps $0, %xmm14, %xmm14 cmpq $3, M - jle .L117 + jle L(117) testq $SIZE, A1 - je .L11X + je L(11X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(A1, LDA), %xmm1 @@ -2534,9 +2534,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L11X: +L(11X): testq $2 * SIZE, A1 - je .L11XX + je L(11XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A1, LDA), %xmm1 @@ -2558,13 +2558,13 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L11XX: +L(11XX): movhps -32 * SIZE(A1, LDA), %xmm8 movhps -32 * SIZE(A2, LDA), %xmm9 movq MM, I sarq $4, I - jle .L115 + jle L(115) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -2576,10 +2576,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L114 + jle L(114) ALIGN_3 -.L113: +L(113): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 3 - 128 + PREOFFSET(A1) #endif @@ -2655,10 +2655,10 @@ subq $1, I BRANCH - jg .L113 + jg L(113) ALIGN_3 -.L114: +L(114): mulps %xmm12, %xmm4 addps %xmm4, %xmm0 movaps -20 * SIZE(A1), %xmm7 @@ -2709,9 +2709,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L115: +L(115): testq $8, MM - je .L116 + je L(116) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -2749,9 +2749,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L116: +L(116): testq $4, MM - je .L117 + je L(117) movaps -32 * SIZE(A1), %xmm4 movaps -30 * SIZE(A1, LDA), %xmm5 @@ -2774,9 +2774,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L117: +L(117): testq $2, MM - je .L118 + je L(118) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA), %xmm5 @@ -2798,9 +2798,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L118: +L(118): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm0 @@ -2816,12 +2816,12 @@ addss %xmm6, %xmm0 movss %xmm0, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_3 -.L120: +L(120): cmpq $2, N - jl .L130 + jl L(130) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -2842,10 +2842,10 @@ shufps $0, %xmm13, %xmm13 cmpq $3, M - jle .L127 + jle L(127) testq $SIZE, A1 - je .L12X + je L(12X) movss -32 * SIZE(Y1), %xmm8 @@ -2864,9 +2864,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L12X: +L(12X): testq $2 * SIZE, A1 - je .L12XX + je L(12XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A2), %xmm1 @@ -2885,12 +2885,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L12XX: +L(12XX): movhps -32 * SIZE(A2), %xmm8 movq MM, I sarq $4, I - jle .L125 + jle L(125) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -2902,10 +2902,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L124 + jle L(124) ALIGN_3 -.L123: +L(123): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -2964,10 +2964,10 @@ subq $1, I BRANCH - jg .L123 + jg L(123) ALIGN_3 -.L124: +L(124): mulps %xmm12, %xmm4 addps %xmm4, %xmm0 movaps -20 * SIZE(A1), %xmm7 @@ -3005,9 +3005,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L125: +L(125): testq $8, MM - je .L126 + je L(126) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -3038,9 +3038,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L126: +L(126): testq $4, MM - je .L127 + je L(127) movaps -32 * SIZE(A1), %xmm4 movaps -30 * SIZE(A2), %xmm5 @@ -3060,9 +3060,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L127: +L(127): testq $2, MM - je .L128 + je L(128) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A2), %xmm1 @@ -3081,9 +3081,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L128: +L(128): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm8 @@ -3096,12 +3096,12 @@ addss %xmm1, %xmm8 movss %xmm8, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_3 -.L130: +L(130): cmpq $1, N - jne .L990 + jne L(990) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -3112,10 +3112,10 @@ shufps $0, %xmm12, %xmm12 cmpq $3, M - jle .L137 + jle L(137) testq $SIZE, A1 - je .L13X + je L(13X) movss -32 * SIZE(Y1), %xmm8 movss -32 * SIZE(A1), %xmm0 @@ -3130,9 +3130,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L13X: +L(13X): testq $2 * SIZE, A1 - je .L13XX + je L(13XX) movsd -32 * SIZE(Y1), %xmm8 movsd -32 * SIZE(A1), %xmm0 @@ -3147,10 +3147,10 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L13XX: +L(13XX): movq MM, I sarq $4, I - jle .L135 + jle L(135) movaps -32 * SIZE(A1), %xmm8 movaps -28 * SIZE(A1), %xmm9 @@ -3163,10 +3163,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L134 + jle L(134) ALIGN_3 -.L133: +L(133): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -3203,10 +3203,10 @@ subq $1, I BRANCH - jg .L133 + jg L(133) ALIGN_3 -.L134: +L(134): mulps %xmm12, %xmm8 addps %xmm8, %xmm0 MOVUPS_YS1(-32 * SIZE, Y1, %xmm0) @@ -3224,9 +3224,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L135: +L(135): testq $8, MM - je .L136 + je L(136) movaps -32 * SIZE(A1), %xmm8 movaps -28 * SIZE(A1), %xmm9 @@ -3245,9 +3245,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L136: +L(136): testq $4, MM - je .L137 + je L(137) movaps -32 * SIZE(A1), %xmm8 MOVUPS_YL1(-32 * SIZE, Y1, %xmm0) @@ -3260,9 +3260,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L137: +L(137): testq $2, MM - je .L138 + je L(138) movsd -32 * SIZE(Y1), %xmm8 movsd -32 * SIZE(A1), %xmm0 @@ -3276,9 +3276,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L138: +L(138): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm8 movss -32 * SIZE(A1), %xmm0 @@ -3287,18 +3287,18 @@ addss %xmm0, %xmm8 movss %xmm8, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_3 -.L200: +L(200): testq $2 * SIZE, LDA - jne .L300 + jne L(300) cmpq $4, N - jl .L210 + jl L(210) ALIGN_3 -.L201: +L(201): subq $4, N leaq 32 * SIZE(BUFFER), Y1 @@ -3328,10 +3328,10 @@ shufps $0, %xmm15, %xmm15 cmpq $3, M - jle .L207 + jle L(207) testq $SIZE, A1 - je .L20X + je L(20X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(A1, LDA), %xmm1 @@ -3356,9 +3356,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L20X: +L(20X): testq $2 * SIZE, A1 - je .L20XX + je L(20XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A1, LDA), %xmm1 @@ -3383,14 +3383,14 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L20XX: +L(20XX): movaps -33 * SIZE(A1, LDA), %xmm8 movaps -34 * SIZE(A2), %xmm9 movaps -35 * SIZE(A2, LDA), %xmm10 movq MM, I sarq $4, I - jle .L205 + jle L(205) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -3402,10 +3402,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L204 + jle L(204) ALIGN_3 -.L203: +L(203): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -3514,10 +3514,10 @@ subq $1, I BRANCH - jg .L203 + jg L(203) ALIGN_3 -.L204: +L(204): mulps %xmm12, %xmm4 addps %xmm4, %xmm0 movaps -20 * SIZE(A1), %xmm7 @@ -3599,9 +3599,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L205: +L(205): testq $8, MM - je .L206 + je L(206) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -3657,9 +3657,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L206: +L(206): testq $4, MM - je .L207 + je L(207) movaps -32 * SIZE(A1), %xmm4 movaps -29 * SIZE(A1, LDA), %xmm5 @@ -3690,9 +3690,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L207: +L(207): testq $2, MM - je .L208 + je L(208) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA), %xmm5 @@ -3717,9 +3717,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L208: +L(208): testq $1, MM - je .L209 + je L(209) movss -32 * SIZE(Y1), %xmm0 @@ -3740,14 +3740,14 @@ movss %xmm0, -32 * SIZE(Y1) ALIGN_3 -.L209: +L(209): cmpq $4, N - jge .L201 + jge L(201) ALIGN_3 -.L210: +L(210): cmpq $3, N - jne .L220 + jne L(220) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -3772,10 +3772,10 @@ shufps $0, %xmm14, %xmm14 cmpq $3, M - jle .L217 + jle L(217) testq $SIZE, A1 - je .L21X + je L(21X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(A1, LDA), %xmm1 @@ -3797,9 +3797,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L21X: +L(21X): testq $2 * SIZE, A1 - je .L21XX + je L(21XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A1, LDA), %xmm1 @@ -3821,14 +3821,14 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L21XX: +L(21XX): movaps -33 * SIZE(A1, LDA), %xmm8 movaps -34 * SIZE(A2), %xmm9 movaps -35 * SIZE(A2, LDA), %xmm10 movq MM, I sarq $4, I - jle .L215 + jle L(215) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -3840,10 +3840,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L214 + jle L(214) ALIGN_3 -.L213: +L(213): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 3 - 128 + PREOFFSET(A1) #endif @@ -3927,10 +3927,10 @@ subq $1, I BRANCH - jg .L213 + jg L(213) ALIGN_3 -.L214: +L(214): mulps %xmm12, %xmm4 addps %xmm4, %xmm0 movaps -20 * SIZE(A1), %xmm7 @@ -3990,9 +3990,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L215: +L(215): testq $8, MM - je .L216 + je L(216) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -4035,9 +4035,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L216: +L(216): testq $4, MM - je .L217 + je L(217) movaps -32 * SIZE(A1), %xmm4 movaps -29 * SIZE(A1, LDA), %xmm5 @@ -4064,9 +4064,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L217: +L(217): testq $2, MM - je .L218 + je L(218) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA), %xmm5 @@ -4088,9 +4088,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L218: +L(218): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm0 @@ -4106,15 +4106,15 @@ addss %xmm6, %xmm0 movss %xmm0, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_4 -.L220: +L(220): testq N, N - jle .L990 + jle L(990) cmpq $2, N - jne .L230 + jne L(230) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -4135,10 +4135,10 @@ shufps $0, %xmm13, %xmm13 cmpq $3, M - jle .L227 + jle L(227) testq $SIZE, A1 - je .L22X + je L(22X) movss -32 * SIZE(Y1), %xmm9 @@ -4157,9 +4157,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L22X: +L(22X): testq $2 * SIZE, A1 - je .L22XX + je L(22XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A2), %xmm1 @@ -4178,12 +4178,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L22XX: +L(22XX): movaps -33 * SIZE(A1, LDA), %xmm8 movq MM, I sarq $4, I - jle .L225 + jle L(225) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -4195,10 +4195,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L224 + jle L(224) ALIGN_3 -.L223: +L(223): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -4262,10 +4262,10 @@ subq $1, I BRANCH - jg .L223 + jg L(223) ALIGN_3 -.L224: +L(224): mulps %xmm12, %xmm4 addps %xmm4, %xmm0 movaps -20 * SIZE(A1), %xmm7 @@ -4308,9 +4308,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L225: +L(225): testq $8, MM - je .L226 + je L(226) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -4344,9 +4344,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L226: +L(226): testq $4, MM - je .L227 + je L(227) movaps -32 * SIZE(A1), %xmm4 movaps -29 * SIZE(A2), %xmm5 @@ -4367,9 +4367,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L227: +L(227): testq $2, MM - je .L228 + je L(228) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A2), %xmm1 @@ -4388,9 +4388,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L228: +L(228): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm9 @@ -4403,12 +4403,12 @@ addss %xmm1, %xmm9 movss %xmm9, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_3 -.L230: +L(230): cmpq $1, N - jne .L990 + jne L(990) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -4419,10 +4419,10 @@ shufps $0, %xmm12, %xmm12 cmpq $3, M - jle .L237 + jle L(237) testq $SIZE, A1 - je .L23X + je L(23X) movss -32 * SIZE(Y1), %xmm8 movss -32 * SIZE(A1), %xmm0 @@ -4436,9 +4436,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L23X: +L(23X): testq $2 * SIZE, A1 - je .L23XX + je L(23XX) movsd -32 * SIZE(Y1), %xmm8 movsd -32 * SIZE(A1), %xmm0 @@ -4452,13 +4452,13 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L23XX: +L(23XX): testq $2 * SIZE, A1 - jne .L230 + jne L(230) movq MM, I sarq $4, I - jle .L235 + jle L(235) movaps -32 * SIZE(A1), %xmm8 movaps -28 * SIZE(A1), %xmm9 @@ -4471,10 +4471,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L234 + jle L(234) ALIGN_3 -.L233: +L(233): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -4511,10 +4511,10 @@ subq $1, I BRANCH - jg .L233 + jg L(233) ALIGN_3 -.L234: +L(234): mulps %xmm12, %xmm8 addps %xmm8, %xmm0 MOVUPS_YS1(-32 * SIZE, Y1, %xmm0) @@ -4532,9 +4532,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L235: +L(235): testq $8, MM - je .L236 + je L(236) movaps -32 * SIZE(A1), %xmm8 movaps -28 * SIZE(A1), %xmm9 @@ -4553,9 +4553,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L236: +L(236): testq $4, MM - je .L237 + je L(237) movaps -32 * SIZE(A1), %xmm8 MOVUPS_YL1(-32 * SIZE, Y1, %xmm0) @@ -4568,9 +4568,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L237: +L(237): testq $2, MM - je .L238 + je L(238) movsd -32 * SIZE(Y1), %xmm8 movsd -32 * SIZE(A1), %xmm0 @@ -4584,9 +4584,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L238: +L(238): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm8 movss -32 * SIZE(A1), %xmm0 @@ -4595,15 +4595,15 @@ addss %xmm0, %xmm8 movss %xmm8, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_4 -.L300: +L(300): cmpq $4, N - jl .L310 + jl L(310) ALIGN_3 -.L301: +L(301): subq $4, N leaq 32 * SIZE(BUFFER), Y1 @@ -4633,10 +4633,10 @@ shufps $0, %xmm15, %xmm15 cmpq $3, M - jle .L307 + jle L(307) testq $SIZE, A1 - je .L30X + je L(30X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(A1, LDA), %xmm1 @@ -4661,9 +4661,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L30X: +L(30X): testq $2 * SIZE, A1 - je .L30XX + je L(30XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A1, LDA), %xmm1 @@ -4688,14 +4688,14 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L30XX: +L(30XX): movaps -35 * SIZE(A1, LDA), %xmm8 movaps -34 * SIZE(A2), %xmm9 movaps -33 * SIZE(A2, LDA), %xmm10 movq MM, I sarq $4, I - jle .L305 + jle L(305) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -4707,10 +4707,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L304 + jle L(304) ALIGN_3 -.L303: +L(303): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -4819,10 +4819,10 @@ subq $1, I BRANCH - jg .L303 + jg L(303) ALIGN_3 -.L304: +L(304): mulps %xmm12, %xmm4 addps %xmm4, %xmm0 movaps -20 * SIZE(A1), %xmm7 @@ -4904,9 +4904,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L305: +L(305): testq $8, MM - je .L306 + je L(306) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -4961,9 +4961,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L306: +L(306): testq $4, MM - je .L307 + je L(307) movaps -32 * SIZE(A1), %xmm4 movaps -31 * SIZE(A1, LDA), %xmm5 @@ -4994,9 +4994,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L307: +L(307): testq $2, MM - je .L308 + je L(308) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA), %xmm5 @@ -5021,9 +5021,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L308: +L(308): testq $1, MM - je .L309 + je L(309) movss -32 * SIZE(Y1), %xmm0 @@ -5044,14 +5044,14 @@ movss %xmm0, -32 * SIZE(Y1) ALIGN_3 -.L309: +L(309): cmpq $4, N - jge .L301 + jge L(301) ALIGN_3 -.L310: +L(310): cmpq $3, N - jne .L320 + jne L(320) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -5076,10 +5076,10 @@ shufps $0, %xmm14, %xmm14 cmpq $3, M - jle .L317 + jle L(317) testq $SIZE, A1 - je .L31X + je L(31X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(A1, LDA), %xmm1 @@ -5101,9 +5101,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L31X: +L(31X): testq $2 * SIZE, A1 - je .L31XX + je L(31XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A1, LDA), %xmm1 @@ -5125,14 +5125,14 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L31XX: +L(31XX): movaps -35 * SIZE(A1, LDA), %xmm8 movaps -34 * SIZE(A2), %xmm9 movaps -33 * SIZE(A2, LDA), %xmm10 movq MM, I sarq $4, I - jle .L315 + jle L(315) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -5144,10 +5144,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L314 + jle L(314) ALIGN_3 -.L313: +L(313): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 3 - 128 + PREOFFSET(A1) #endif @@ -5231,10 +5231,10 @@ subq $1, I BRANCH - jg .L313 + jg L(313) ALIGN_3 -.L314: +L(314): mulps %xmm12, %xmm4 addps %xmm4, %xmm0 movaps -20 * SIZE(A1), %xmm7 @@ -5295,9 +5295,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L315: +L(315): testq $8, MM - je .L316 + je L(316) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -5340,9 +5340,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L316: +L(316): testq $4, MM - je .L317 + je L(317) movaps -32 * SIZE(A1), %xmm4 movaps -31 * SIZE(A1, LDA), %xmm5 @@ -5368,9 +5368,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L317: +L(317): testq $2, MM - je .L318 + je L(318) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA), %xmm5 @@ -5392,9 +5392,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L318: +L(318): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm0 @@ -5410,12 +5410,12 @@ addss %xmm6, %xmm0 movss %xmm0, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_3 -.L320: +L(320): cmpq $2, N - jne .L330 + jne L(330) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -5435,10 +5435,10 @@ shufps $0, %xmm13, %xmm13 cmpq $3, M - jle .L327 + jle L(327) testq $SIZE, A1 - je .L32X + je L(32X) movss -32 * SIZE(Y1), %xmm9 @@ -5457,9 +5457,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L32X: +L(32X): testq $2 * SIZE, A1 - je .L32XX + je L(32XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A2), %xmm1 @@ -5478,12 +5478,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L32XX: +L(32XX): movaps -35 * SIZE(A1, LDA), %xmm8 movq MM, I sarq $4, I - jle .L325 + jle L(325) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -5495,10 +5495,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L324 + jle L(324) ALIGN_3 -.L323: +L(323): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -5561,10 +5561,10 @@ subq $1, I BRANCH - jg .L323 + jg L(323) ALIGN_3 -.L324: +L(324): mulps %xmm12, %xmm4 addps %xmm4, %xmm0 movaps -20 * SIZE(A1), %xmm7 @@ -5608,9 +5608,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L325: +L(325): testq $8, MM - je .L326 + je L(326) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -5643,9 +5643,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L326: +L(326): testq $4, MM - je .L327 + je L(327) movaps -32 * SIZE(A1), %xmm4 movaps -31 * SIZE(A2), %xmm5 @@ -5666,9 +5666,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L327: +L(327): testq $2, MM - je .L328 + je L(328) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A2), %xmm1 @@ -5687,9 +5687,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L328: +L(328): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm8 @@ -5702,12 +5702,12 @@ addss %xmm1, %xmm8 movss %xmm8, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_3 -.L330: +L(330): cmpq $1, N - jne .L990 + jne L(990) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -5718,10 +5718,10 @@ shufps $0, %xmm12, %xmm12 cmpq $3, M - jle .L337 + jle L(337) testq $SIZE, A1 - je .L33X + je L(33X) movss -32 * SIZE(Y1), %xmm8 movss -32 * SIZE(A1), %xmm0 @@ -5735,9 +5735,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L33X: +L(33X): testq $2 * SIZE, A1 - je .L33XX + je L(33XX) movsd -32 * SIZE(Y1), %xmm8 movsd -32 * SIZE(A1), %xmm0 @@ -5751,10 +5751,10 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L33XX: +L(33XX): movq MM, I sarq $4, I - jle .L335 + jle L(335) movaps -32 * SIZE(A1), %xmm8 movaps -28 * SIZE(A1), %xmm9 @@ -5767,10 +5767,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L334 + jle L(334) ALIGN_3 -.L333: +L(333): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -5807,10 +5807,10 @@ subq $1, I BRANCH - jg .L333 + jg L(333) ALIGN_3 -.L334: +L(334): mulps %xmm12, %xmm8 addps %xmm8, %xmm0 MOVUPS_YS1(-32 * SIZE, Y1, %xmm0) @@ -5828,9 +5828,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L335: +L(335): testq $8, MM - je .L336 + je L(336) movaps -32 * SIZE(A1), %xmm8 movaps -28 * SIZE(A1), %xmm9 @@ -5849,9 +5849,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L336: +L(336): testq $4, MM - je .L337 + je L(337) movaps -32 * SIZE(A1), %xmm8 MOVUPS_YL1(-32 * SIZE, Y1, %xmm0) @@ -5864,9 +5864,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L337: +L(337): testq $2, MM - je .L338 + je L(338) movsd -32 * SIZE(Y1), %xmm8 movsd -32 * SIZE(A1), %xmm0 @@ -5880,9 +5880,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L338: +L(338): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm8 movss -32 * SIZE(A1), %xmm0 @@ -5891,19 +5891,19 @@ addss %xmm0, %xmm8 movss %xmm8, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) #endif ALIGN_4 -.L990: +L(990): movq Y, Y1 movq M, %rax sarq $3, %rax - jle .L994 + jle L(994) ALIGN_3 -.L992: +L(992): movsd 0 * SIZE(BUFFER), %xmm0 movhps 2 * SIZE(BUFFER), %xmm0 movsd 4 * SIZE(BUFFER), %xmm4 @@ -5953,15 +5953,15 @@ addq $8 * SIZE, BUFFER decq %rax - jg .L992 + jg L(992) ALIGN_3 -.L994: +L(994): testq $7, M - jle .L999 + jle L(999) testq $4, M - jle .L995 + jle L(995) movsd 0 * SIZE(BUFFER), %xmm0 movhps 2 * SIZE(BUFFER), %xmm0 @@ -5991,9 +5991,9 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L995: +L(995): testq $2, M - jle .L996 + jle L(996) movsd (BUFFER), %xmm0 @@ -6012,9 +6012,9 @@ addq $2 * SIZE, BUFFER ALIGN_3 -.L996: +L(996): testq $1, M - jle .L999 + jle L(999) movss (BUFFER), %xmm0 @@ -6023,13 +6023,13 @@ movss %xmm0, (Y1) ALIGN_3 -.L999: +L(999): leaq (,M,SIZE),%rax addq %rax,AA - jmp .L0t + jmp L(0t) ALIGN_4 -.L999x: +L(999x): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/sgemv_t.S b/kernel/x86_64/sgemv_t.S index 33e2fa86cc..013806ef8a 100644 --- a/kernel/x86_64/sgemv_t.S +++ b/kernel/x86_64/sgemv_t.S @@ -154,20 +154,20 @@ #endif -.L0t: +L(0t): xorq M,M addq $1,M salq $22,M subq M,MMM - jge .L00t + jge L(00t) ALIGN_4 movq MMM,%rax addq M,%rax - jle .L999x + jle L(999x) movq %rax,M -.L00t: +L(00t): movq LDAX,LDA movq NN,N movq AA,A @@ -185,9 +185,9 @@ #ifdef ALIGNED_ACCESS movq M, MM testq $4 * SIZE - 1, A - je .L0X + je L(0X) cmpq $3, M - jle .L0X + jle L(0X) movq A, MM sarq $BASE_SHIFT, MM @@ -195,13 +195,13 @@ subq $4, MM addq M, MM -.L0X: +L(0X): #endif testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) ALIGN_4 subq $-32 * SIZE, A @@ -216,10 +216,10 @@ movq M, I sarq $3, I - jle .L05 + jle L(05) ALIGN_4 -.L02: +L(02): movss (X), %xmm0 addq INCX, X movss (X), %xmm1 @@ -251,39 +251,39 @@ addq $8 * SIZE, X1 decq I - jg .L02 + jg L(02) ALIGN_4 -.L05: +L(05): movq M, I andq $7, I - jle .L10 + jle L(10) ALIGN_2 -.L06: +L(06): movss (X), %xmm0 addq INCX, X movss %xmm0, 0 * SIZE(X1) addq $SIZE, X1 decq I - jg .L06 + jg L(06) ALIGN_4 -.L10: +L(10): movq Y, Y1 #ifdef ALIGNED_ACCESS testq $4 * SIZE - 1, LDA - jne .L100 + jne L(100) #endif #if GEMV_UNROLL >= 8 cmpq $8, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $8, N leaq 32 * SIZE(BUFFER), X1 @@ -303,10 +303,10 @@ #ifdef ALIGNED_ACCESS cmpq $3, M - jle .L17 + jle L(17) testq $SIZE, A1 - je .L1X + je L(1X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -339,9 +339,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L1X: +L(1X): testq $2 * SIZE, A1 - je .L1XX + je L(1XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -387,7 +387,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L1XX: +L(1XX): #endif MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) @@ -399,7 +399,7 @@ movq MM, I sarq $4, I - jle .L15 + jle L(15) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-32 * SIZE, A1, LDA, 1, %xmm1) @@ -407,10 +407,10 @@ MOVUPS_A2 (-32 * SIZE, A1, LDA3, 1, %xmm3) decq I - jle .L13 + jle L(13) ALIGN_4 -.L12: +L(12): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -561,10 +561,10 @@ addq $16 * SIZE, X1 decq I - jg .L12 + jg L(12) ALIGN_4 -.L13: +L(13): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-32 * SIZE, A2, %xmm0) @@ -670,9 +670,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L15: +L(15): testq $8, MM - jle .L16 + jle L(16) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-32 * SIZE, A1, LDA, 1, %xmm1) @@ -731,9 +731,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L16: +L(16): testq $4, MM - jle .L17 + jle L(17) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-32 * SIZE, A1, LDA, 1, %xmm1) @@ -766,9 +766,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L17: +L(17): testq $2, MM - jle .L18 + jle L(18) #ifdef movsd xorps %xmm0, %xmm0 @@ -816,9 +816,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L18: +L(18): testq $1, MM - jle .L19 + jle L(19) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -847,7 +847,7 @@ addss %xmm3, %xmm15 ALIGN_4 -.L19: +L(19): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm11, %xmm10 @@ -958,19 +958,19 @@ addq INCY, Y1 cmpq $8, N - jge .L11 + jge L(11) ALIGN_4 -.L20: +L(20): #endif cmpq $4, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 4 ALIGN_3 -.L21: +L(21): #endif subq $4, N @@ -987,10 +987,10 @@ #ifdef ALIGNED_ACCESS cmpq $3, M - jle .L27 + jle L(27) testq $SIZE, A1 - je .L2X + je L(2X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -1011,9 +1011,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L2X: +L(2X): testq $2 * SIZE, A1 - je .L2XX + je L(2XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -1047,7 +1047,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L2XX: +L(2XX): #endif MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) @@ -1059,7 +1059,7 @@ movq MM, I sarq $4, I - jle .L25 + jle L(25) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-32 * SIZE, A1, LDA, 1, %xmm1) @@ -1072,10 +1072,10 @@ MOVUPS_A2 (-28 * SIZE, A2, LDA, 1, %xmm15) decq I - jle .L23 + jle L(23) ALIGN_4 -.L22: +L(22): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -1158,10 +1158,10 @@ addq $16 * SIZE, X1 decq I - jg .L22 + jg L(22) ALIGN_4 -.L23: +L(23): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-24 * SIZE, A1, %xmm0) @@ -1215,9 +1215,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L25: +L(25): testq $8, MM - jle .L26 + jle L(26) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -1251,9 +1251,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L26: +L(26): testq $4, MM - jle .L27 + jle L(27) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -1273,9 +1273,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L27: +L(27): testq $2, MM - jle .L28 + jle L(28) #ifdef movsd xorps %xmm0, %xmm0 @@ -1312,9 +1312,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L28: +L(28): testq $1, MM - jle .L29 + jle L(29) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -1331,7 +1331,7 @@ addss %xmm3, %xmm11 ALIGN_4 -.L29: +L(29): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm11, %xmm10 @@ -1391,13 +1391,13 @@ #if GEMV_UNROLL == 4 cmpq $4, N - jge .L21 + jge L(21) #endif ALIGN_4 -.L30: +L(30): cmpq $3, N - jne .L40 + jne L(40) leaq 32 * SIZE(BUFFER), X1 @@ -1411,10 +1411,10 @@ #ifdef ALIGNED_ACCESS cmpq $3, M - jle .L37 + jle L(37) testq $SIZE, A1 - je .L3X + je L(3X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -1435,9 +1435,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L3X: +L(3X): testq $2 * SIZE, A1 - je .L3XX + je L(3XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -1471,7 +1471,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L3XX: +L(3XX): #endif MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) @@ -1483,7 +1483,7 @@ movq MM, I sarq $4, I - jle .L35 + jle L(35) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-32 * SIZE, A1, LDA, 1, %xmm1) @@ -1494,10 +1494,10 @@ MOVUPS_A1 (-28 * SIZE, A2, %xmm14) decq I - jle .L33 + jle L(33) ALIGN_4 -.L32: +L(32): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 3 - 128 + PREOFFSET(A1) #endif @@ -1563,10 +1563,10 @@ addq $16 * SIZE, X1 decq I - jg .L32 + jg L(32) ALIGN_4 -.L33: +L(33): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-24 * SIZE, A1, %xmm0) @@ -1610,9 +1610,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L35: +L(35): testq $8, MM - jle .L36 + jle L(36) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -1640,9 +1640,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L36: +L(36): testq $4, MM - jle .L37 + jle L(37) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -1659,9 +1659,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L37: +L(37): testq $2, MM - jle .L38 + jle L(38) #ifdef movsd xorps %xmm0, %xmm0 @@ -1694,9 +1694,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L38: +L(38): testq $1, MM - jle .L39 + jle L(39) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -1710,7 +1710,7 @@ addss %xmm2, %xmm10 ALIGN_4 -.L39: +L(39): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm11, %xmm10 @@ -1760,12 +1760,12 @@ addq INCY, Y1 movss %xmm10, (Y1) addq INCY, Y1 - jmp .L999 + jmp L(999) ALIGN_4 -.L40: +L(40): cmpq $2, N - jne .L50 + jne L(50) leaq 32 * SIZE(BUFFER), X1 @@ -1778,10 +1778,10 @@ #ifdef ALIGNED_ACCESS cmpq $3, M - jle .L47 + jle L(47) testq $SIZE, A1 - je .L4X + je L(4X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -1796,9 +1796,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L4X: +L(4X): testq $2 * SIZE, A1 - je .L4XX + je L(4XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -1820,7 +1820,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L4XX: +L(4XX): #endif MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) @@ -1828,7 +1828,7 @@ movq MM, I sarq $4, I - jle .L45 + jle L(45) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-32 * SIZE, A2, %xmm1) @@ -1836,10 +1836,10 @@ MOVUPS_A1 (-28 * SIZE, A2, %xmm13) decq I - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -1889,10 +1889,10 @@ addq $16 * SIZE, X1 decq I - jg .L42 + jg L(42) ALIGN_4 -.L43: +L(43): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-24 * SIZE, A1, %xmm0) @@ -1926,9 +1926,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L45: +L(45): testq $8, MM - jle .L46 + jle L(46) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -1950,9 +1950,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L46: +L(46): testq $4, MM - jle .L47 + jle L(47) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -1966,9 +1966,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L47: +L(47): testq $2, MM - jle .L48 + jle L(48) #ifdef movsd xorps %xmm0, %xmm0 @@ -1993,9 +1993,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L48: +L(48): testq $1, MM - jle .L49 + jle L(49) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -2006,7 +2006,7 @@ addss %xmm1, %xmm9 ALIGN_4 -.L49: +L(49): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm8, %xmm8 @@ -2034,12 +2034,12 @@ addq INCY, Y1 movss %xmm9, (Y1) addq INCY, Y1 - jmp .L999 + jmp L(999) ALIGN_4 -.L50: +L(50): cmpq $1, N - jne .L999 + jne L(999) leaq 32 * SIZE(BUFFER), X1 @@ -2050,10 +2050,10 @@ #ifdef ALIGNED_ACCESS cmpq $3, M - jle .L57 + jle L(57) testq $SIZE, A1 - je .L5X + je L(5X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -2064,9 +2064,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L5X: +L(5X): testq $2 * SIZE, A1 - je .L5XX + je L(5XX) #ifdef movsd @@ -2083,7 +2083,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L5XX: +L(5XX): #endif MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) @@ -2091,16 +2091,16 @@ movq MM, I sarq $4, I - jle .L55 + jle L(55) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-28 * SIZE, A1, %xmm12) decq I - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -2133,10 +2133,10 @@ addq $16 * SIZE, X1 decq I - jg .L52 + jg L(52) ALIGN_4 -.L53: +L(53): mulps %xmm4, %xmm0 MOVUPS_XL1 (-24 * SIZE, X1, %xmm4) addps %xmm0, %xmm8 @@ -2159,9 +2159,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L55: +L(55): testq $8, MM - jle .L56 + jle L(56) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -2176,9 +2176,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L56: +L(56): testq $4, MM - jle .L57 + jle L(57) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -2188,9 +2188,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L57: +L(57): testq $2, MM - jle .L58 + jle L(58) #ifdef movsd xorps %xmm0, %xmm0 @@ -2208,9 +2208,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L58: +L(58): testq $1, MM - jle .L59 + jle L(59) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -2218,7 +2218,7 @@ addss %xmm0, %xmm8 ALIGN_4 -.L59: +L(59): addps %xmm9, %xmm8 #ifdef HAVE_SSE3 @@ -2240,18 +2240,18 @@ movss %xmm8, (Y1) #ifdef ALIGNED_ACCESS - jmp .L999 + jmp L(999) ALIGN_4 -.L100: +L(100): testq $2 * SIZE - 1, LDA - jne .L200 + jne L(200) cmpq $4, N - jl .L110 + jl L(110) ALIGN_3 -.L101: +L(101): subq $4, N leaq 32 * SIZE(BUFFER), X1 @@ -2266,10 +2266,10 @@ xorps %xmm11, %xmm11 cmpq $3, M - jle .L107 + jle L(107) testq $SIZE, A1 - je .L10X + je L(10X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -2290,9 +2290,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L10X: +L(10X): testq $2 * SIZE, A1 - je .L10XX + je L(10XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -2326,7 +2326,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L10XX: +L(10XX): MOVUPS_A2 (-34 * SIZE, A1, LDA, 1, %xmm12) MOVUPS_A2 (-34 * SIZE, A2, LDA, 1, %xmm13) @@ -2339,7 +2339,7 @@ movq MM, I sarq $4, I - jle .L105 + jle L(105) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-30 * SIZE, A1, LDA, 1, %xmm1) @@ -2347,10 +2347,10 @@ MOVUPS_A2 (-30 * SIZE, A2, LDA, 1, %xmm3) decq I - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -2441,10 +2441,10 @@ addq $16 * SIZE, X1 decq I - jg .L102 + jg L(102) ALIGN_4 -.L103: +L(103): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-28 * SIZE, A1, %xmm0) @@ -2510,9 +2510,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L105: +L(105): testq $8, MM - jle .L106 + jle L(106) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-30 * SIZE, A1, LDA, 1, %xmm1) @@ -2551,9 +2551,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L106: +L(106): testq $4, MM - jle .L107 + jle L(107) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -2576,9 +2576,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L107: +L(107): testq $2, MM - jle .L108 + jle L(108) #ifdef movsd xorps %xmm0, %xmm0 @@ -2615,9 +2615,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L108: +L(108): testq $1, MM - jle .L109 + jle L(109) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -2634,7 +2634,7 @@ addss %xmm3, %xmm11 ALIGN_4 -.L109: +L(109): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm11, %xmm10 @@ -2693,12 +2693,12 @@ addq INCY, Y1 cmpq $4, N - jge .L101 + jge L(101) ALIGN_4 -.L110: +L(110): cmpq $3, N - jne .L120 + jne L(120) leaq 32 * SIZE(BUFFER), X1 @@ -2711,10 +2711,10 @@ xorps %xmm10, %xmm10 cmpq $3, M - jle .L117 + jle L(117) testq $SIZE, A1 - je .L11X + je L(11X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -2732,9 +2732,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L11X: +L(11X): testq $2 * SIZE, A1 - je .L11XX + je L(11XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -2762,7 +2762,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L11XX: +L(11XX): MOVUPS_A2 (-34 * SIZE, A1, LDA, 1, %xmm12) MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) @@ -2770,17 +2770,17 @@ movq MM, I sarq $4, I - jle .L115 + jle L(115) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-30 * SIZE, A1, LDA, 1, %xmm1) MOVUPS_A1 (-32 * SIZE, A2, %xmm2) decq I - jle .L113 + jle L(113) ALIGN_4 -.L112: +L(112): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 3 - 128 + PREOFFSET(A1) #endif @@ -2850,10 +2850,10 @@ addq $16 * SIZE, X1 decq I - jg .L112 + jg L(112) ALIGN_4 -.L113: +L(113): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-28 * SIZE, A1, %xmm0) @@ -2904,9 +2904,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L115: +L(115): testq $8, MM - jle .L116 + jle L(116) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-30 * SIZE, A1, LDA, 1, %xmm1) @@ -2937,9 +2937,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L116: +L(116): testq $4, MM - jle .L117 + jle L(117) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -2958,9 +2958,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L117: +L(117): testq $2, MM - jle .L118 + jle L(118) #ifdef movsd xorps %xmm0, %xmm0 @@ -2990,9 +2990,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L118: +L(118): testq $1, MM - jle .L119 + jle L(119) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -3006,7 +3006,7 @@ addss %xmm2, %xmm10 ALIGN_4 -.L119: +L(119): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm11, %xmm10 @@ -3054,12 +3054,12 @@ movss %xmm9, (Y1) addq INCY, Y1 movss %xmm10, (Y1) - jmp .L999 + jmp L(999) ALIGN_4 -.L120: +L(120): cmpq $2, N - jne .L130 + jne L(130) leaq 32 * SIZE(BUFFER), X1 @@ -3071,10 +3071,10 @@ xorps %xmm9, %xmm9 cmpq $3, M - jle .L127 + jle L(127) testq $SIZE, A1 - je .L12X + je L(12X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -3089,9 +3089,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L12X: +L(12X): testq $2 * SIZE, A1 - je .L12XX + je L(12XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -3113,7 +3113,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L12XX: +L(12XX): MOVUPS_A1 (-34 * SIZE, A2, %xmm12) MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) @@ -3121,16 +3121,16 @@ movq MM, I sarq $4, I - jle .L125 + jle L(125) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-30 * SIZE, A2, %xmm1) decq I - jle .L123 + jle L(123) ALIGN_4 -.L122: +L(122): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -3184,10 +3184,10 @@ addq $16 * SIZE, X1 decq I - jg .L122 + jg L(122) ALIGN_4 -.L123: +L(123): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-28 * SIZE, A1, %xmm0) @@ -3227,9 +3227,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L125: +L(125): testq $8, MM - jle .L126 + jle L(126) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-30 * SIZE, A2, %xmm1) @@ -3254,9 +3254,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L126: +L(126): testq $4, MM - jle .L127 + jle L(127) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -3271,9 +3271,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L127: +L(127): testq $2, MM - jle .L128 + jle L(128) #ifdef movsd xorps %xmm0, %xmm0 @@ -3298,9 +3298,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L128: +L(128): testq $1, MM - jle .L129 + jle L(129) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -3311,7 +3311,7 @@ addss %xmm1, %xmm9 ALIGN_4 -.L129: +L(129): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm8, %xmm8 @@ -3339,12 +3339,12 @@ addq INCY, Y1 movss %xmm9, (Y1) addq INCY, Y1 - jmp .L999 + jmp L(999) ALIGN_4 -.L130: +L(130): cmpq $1, N - jne .L999 + jne L(999) leaq 32 * SIZE(BUFFER), X1 @@ -3354,10 +3354,10 @@ xorps %xmm9, %xmm9 cmpq $3, M - jle .L137 + jle L(137) testq $SIZE, A1 - je .L13X + je L(13X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -3368,9 +3368,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L13X: +L(13X): testq $2 * SIZE, A1 - je .L13XX + je L(13XX) #ifdef movsd @@ -3387,23 +3387,23 @@ addq $2 * SIZE, X1 ALIGN_3 -.L13XX: +L(13XX): MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) MOVUPS_XL1 (-28 * SIZE, X1, %xmm5) movq MM, I sarq $4, I - jle .L135 + jle L(135) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-28 * SIZE, A1, %xmm12) decq I - jle .L133 + jle L(133) ALIGN_4 -.L132: +L(132): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -3436,10 +3436,10 @@ addq $16 * SIZE, X1 decq I - jg .L132 + jg L(132) ALIGN_4 -.L133: +L(133): mulps %xmm4, %xmm0 MOVUPS_XL1 (-24 * SIZE, X1, %xmm4) addps %xmm0, %xmm8 @@ -3462,9 +3462,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L135: +L(135): testq $8, MM - jle .L136 + jle L(136) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -3479,9 +3479,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L136: +L(136): testq $4, MM - jle .L137 + jle L(137) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -3491,9 +3491,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L137: +L(137): testq $2, MM - jle .L138 + jle L(138) #ifdef movsd xorps %xmm0, %xmm0 @@ -3511,9 +3511,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L138: +L(138): testq $1, MM - jle .L139 + jle L(139) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -3521,7 +3521,7 @@ addss %xmm0, %xmm8 ALIGN_4 -.L139: +L(139): addps %xmm9, %xmm8 #ifdef HAVE_SSE3 @@ -3541,18 +3541,18 @@ addss (Y), %xmm8 movss %xmm8, (Y1) - jmp .L999 + jmp L(999) ALIGN_4 -.L200: +L(200): testq $2 * SIZE, LDA - jne .L300 + jne L(300) cmpq $4, N - jl .L210 + jl L(210) ALIGN_3 -.L201: +L(201): subq $4, N leaq 32 * SIZE(BUFFER), X1 @@ -3567,10 +3567,10 @@ xorps %xmm11, %xmm11 cmpq $3, M - jle .L207 + jle L(207) testq $SIZE, A1 - je .L20X + je L(20X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -3591,9 +3591,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L20X: +L(20X): testq $2 * SIZE, A1 - je .L20XX + je L(20XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -3627,7 +3627,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L20XX: +L(20XX): movaps -33 * SIZE(A1, LDA), %xmm12 movaps -34 * SIZE(A2), %xmm13 movaps -35 * SIZE(A2, LDA), %xmm14 @@ -3641,7 +3641,7 @@ movq MM, I sarq $4, I - jle .L205 + jle L(205) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-29 * SIZE, A1, LDA, 1, %xmm1) @@ -3649,10 +3649,10 @@ MOVUPS_A2 (-31 * SIZE, A2, LDA, 1, %xmm3) decq I - jle .L203 + jle L(203) ALIGN_4 -.L202: +L(202): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -3758,10 +3758,10 @@ addq $16 * SIZE, X1 decq I - jg .L202 + jg L(202) ALIGN_4 -.L203: +L(203): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-28 * SIZE, A1, %xmm0) @@ -3843,9 +3843,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L205: +L(205): testq $8, MM - jle .L206 + jle L(206) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-29 * SIZE, A1, LDA, 1, %xmm1) @@ -3892,9 +3892,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L206: +L(206): testq $4, MM - jle .L207 + jle L(207) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-29 * SIZE, A1, LDA, 1, %xmm1) @@ -3921,9 +3921,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L207: +L(207): testq $2, MM - jle .L208 + jle L(208) #ifdef movsd xorps %xmm0, %xmm0 @@ -3960,9 +3960,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L208: +L(208): testq $1, MM - jle .L209 + jle L(209) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -3979,7 +3979,7 @@ addss %xmm3, %xmm11 ALIGN_4 -.L209: +L(209): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm11, %xmm10 @@ -4038,12 +4038,12 @@ addq INCY, Y1 cmpq $4, N - jge .L201 + jge L(201) ALIGN_4 -.L210: +L(210): cmpq $3, N - jne .L220 + jne L(220) leaq 32 * SIZE(BUFFER), X1 @@ -4056,10 +4056,10 @@ xorps %xmm10, %xmm10 cmpq $3, M - jle .L217 + jle L(217) testq $SIZE, A1 - je .L21X + je L(21X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -4077,9 +4077,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L21X: +L(21X): testq $2 * SIZE, A1 - je .L21XX + je L(21XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -4107,7 +4107,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L21XX: +L(21XX): movaps -33 * SIZE(A1, LDA), %xmm12 movaps -34 * SIZE(A2), %xmm13 @@ -4120,17 +4120,17 @@ movq MM, I sarq $4, I - jle .L215 + jle L(215) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-29 * SIZE, A1, LDA, 1, %xmm1) MOVUPS_A1 (-30 * SIZE, A2, %xmm2) decq I - jle .L213 + jle L(213) ALIGN_4 -.L212: +L(212): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 3 - 128 + PREOFFSET(A1) #endif @@ -4210,10 +4210,10 @@ addq $16 * SIZE, X1 decq I - jg .L212 + jg L(212) ALIGN_4 -.L213: +L(213): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-28 * SIZE, A1, %xmm0) @@ -4295,9 +4295,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L215: +L(215): testq $8, MM - jle .L216 + jle L(216) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-29 * SIZE, A1, LDA, 1, %xmm1) @@ -4334,9 +4334,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L216: +L(216): testq $4, MM - jle .L217 + jle L(217) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-29 * SIZE, A1, LDA, 1, %xmm1) @@ -4358,9 +4358,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L217: +L(217): testq $2, MM - jle .L218 + jle L(218) #ifdef movsd xorps %xmm0, %xmm0 @@ -4390,9 +4390,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L218: +L(218): testq $1, MM - jle .L219 + jle L(219) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -4406,7 +4406,7 @@ addss %xmm2, %xmm10 ALIGN_4 -.L219: +L(219): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm11, %xmm10 @@ -4454,15 +4454,15 @@ movss %xmm9, (Y1) addq INCY, Y1 movss %xmm10, (Y1) - jmp .L999 + jmp L(999) ALIGN_4 -.L220: +L(220): testq N, N - jle .L999 + jle L(999) cmpq $2, N - jne .L230 + jne L(230) leaq 32 * SIZE(BUFFER), X1 @@ -4474,10 +4474,10 @@ xorps %xmm9, %xmm9 cmpq $3, M - jle .L227 + jle L(227) testq $SIZE, A1 - je .L22X + je L(22X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -4492,9 +4492,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L22X: +L(22X): testq $2 * SIZE, A1 - je .L22XX + je L(22XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -4516,7 +4516,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L22XX: +L(22XX): movaps -33 * SIZE(A2), %xmm12 MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) @@ -4524,16 +4524,16 @@ movq MM, I sarq $4, I - jle .L225 + jle L(225) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-29 * SIZE, A2, %xmm1) decq I - jle .L223 + jle L(223) ALIGN_4 -.L222: +L(222): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -4591,10 +4591,10 @@ addq $16 * SIZE, X1 decq I - jg .L222 + jg L(222) ALIGN_4 -.L223: +L(223): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-28 * SIZE, A1, %xmm2) @@ -4638,9 +4638,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L225: +L(225): testq $8, MM - jle .L226 + jle L(226) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-29 * SIZE, A2, %xmm1) @@ -4667,9 +4667,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L226: +L(226): testq $4, MM - jle .L227 + jle L(227) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-29 * SIZE, A2, %xmm1) @@ -4687,9 +4687,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L227: +L(227): testq $2, MM - jle .L228 + jle L(228) #ifdef movsd xorps %xmm0, %xmm0 @@ -4714,9 +4714,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L228: +L(228): testq $1, MM - jle .L229 + jle L(229) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -4727,7 +4727,7 @@ addss %xmm1, %xmm9 ALIGN_4 -.L229: +L(229): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm8, %xmm8 @@ -4755,12 +4755,12 @@ addq INCY, Y1 movss %xmm9, (Y1) addq INCY, Y1 - jmp .L999 + jmp L(999) ALIGN_4 -.L230: +L(230): cmpq $1, N - jne .L999 + jne L(999) leaq 32 * SIZE(BUFFER), X1 @@ -4770,10 +4770,10 @@ xorps %xmm9, %xmm9 cmpq $3, M - jle .L237 + jle L(237) testq $SIZE, A1 - je .L23X + je L(23X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -4784,9 +4784,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L23X: +L(23X): testq $2 * SIZE, A1 - je .L23XX + je L(23XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -4802,23 +4802,23 @@ addq $2 * SIZE, X1 ALIGN_3 -.L23XX: +L(23XX): MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) MOVUPS_XL1 (-28 * SIZE, X1, %xmm5) movq MM, I sarq $4, I - jle .L235 + jle L(235) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-28 * SIZE, A1, %xmm12) decq I - jle .L233 + jle L(233) ALIGN_4 -.L232: +L(232): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -4851,10 +4851,10 @@ addq $16 * SIZE, X1 decq I - jg .L232 + jg L(232) ALIGN_4 -.L233: +L(233): mulps %xmm4, %xmm0 MOVUPS_XL1 (-24 * SIZE, X1, %xmm4) addps %xmm0, %xmm8 @@ -4877,9 +4877,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L235: +L(235): testq $8, MM - jle .L236 + jle L(236) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -4894,9 +4894,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L236: +L(236): testq $4, MM - jle .L237 + jle L(237) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -4906,9 +4906,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L237: +L(237): testq $2, MM - jle .L238 + jle L(238) #ifdef movsd xorps %xmm0, %xmm0 @@ -4926,9 +4926,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L238: +L(238): testq $1, MM - jle .L239 + jle L(239) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -4936,7 +4936,7 @@ addss %xmm0, %xmm8 ALIGN_4 -.L239: +L(239): addps %xmm9, %xmm8 #ifdef HAVE_SSE3 @@ -4956,15 +4956,15 @@ addss (Y), %xmm8 movss %xmm8, (Y1) - jmp .L999 + jmp L(999) ALIGN_4 -.L300: +L(300): cmpq $4, N - jl .L310 + jl L(310) ALIGN_3 -.L301: +L(301): subq $4, N leaq 32 * SIZE(BUFFER), X1 @@ -4979,10 +4979,10 @@ xorps %xmm11, %xmm11 cmpq $3, M - jle .L307 + jle L(307) testq $SIZE, A1 - je .L30X + je L(30X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -5003,9 +5003,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L30X: +L(30X): testq $2 * SIZE, A1 - je .L30XX + je L(30XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -5039,7 +5039,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L30XX: +L(30XX): movaps -35 * SIZE(A1, LDA), %xmm12 movaps -34 * SIZE(A2), %xmm13 movaps -33 * SIZE(A2, LDA), %xmm14 @@ -5053,7 +5053,7 @@ movq MM, I sarq $4, I - jle .L305 + jle L(305) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-31 * SIZE, A1, LDA, 1, %xmm1) @@ -5061,10 +5061,10 @@ MOVUPS_A2 (-29 * SIZE, A2, LDA, 1, %xmm3) decq I - jle .L303 + jle L(303) ALIGN_4 -.L302: +L(302): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -5170,10 +5170,10 @@ addq $16 * SIZE, X1 decq I - jg .L302 + jg L(302) ALIGN_4 -.L303: +L(303): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-28 * SIZE, A1, %xmm0) @@ -5255,9 +5255,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L305: +L(305): testq $8, MM - jle .L306 + jle L(306) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-31 * SIZE, A1, LDA, 1, %xmm1) @@ -5304,9 +5304,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L306: +L(306): testq $4, MM - jle .L307 + jle L(307) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-31 * SIZE, A1, LDA, 1, %xmm1) @@ -5333,9 +5333,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L307: +L(307): testq $2, MM - jle .L308 + jle L(308) #ifdef movsd xorps %xmm0, %xmm0 @@ -5372,9 +5372,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L308: +L(308): testq $1, MM - jle .L309 + jle L(309) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -5391,7 +5391,7 @@ addss %xmm3, %xmm11 ALIGN_4 -.L309: +L(309): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm11, %xmm10 @@ -5450,15 +5450,15 @@ addq INCY, Y1 cmpq $4, N - jge .L301 + jge L(301) ALIGN_4 -.L310: +L(310): testq N, N - jle .L999 + jle L(999) cmpq $3, N - jne .L320 + jne L(320) leaq 32 * SIZE(BUFFER), X1 @@ -5471,10 +5471,10 @@ xorps %xmm10, %xmm10 cmpq $3, M - jle .L317 + jle L(317) testq $SIZE, A1 - je .L31X + je L(31X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -5492,9 +5492,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L31X: +L(31X): testq $2 * SIZE, A1 - je .L31XX + je L(31XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -5522,7 +5522,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L31XX: +L(31XX): movaps -35 * SIZE(A1, LDA), %xmm12 movaps -34 * SIZE(A2), %xmm13 @@ -5531,17 +5531,17 @@ movq MM, I sarq $4, I - jle .L315 + jle L(315) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-31 * SIZE, A1, LDA, 1, %xmm1) MOVUPS_A1 (-30 * SIZE, A2, %xmm2) decq I - jle .L313 + jle L(313) ALIGN_4 -.L312: +L(312): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 3 - 128 + PREOFFSET(A1) #endif @@ -5620,10 +5620,10 @@ addq $16 * SIZE, X1 decq I - jg .L312 + jg L(312) ALIGN_4 -.L313: +L(313): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-28 * SIZE, A1, %xmm0) @@ -5705,9 +5705,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L315: +L(315): testq $8, MM - jle .L316 + jle L(316) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-31 * SIZE, A1, LDA, 1, %xmm1) @@ -5744,9 +5744,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L316: +L(316): testq $4, MM - jle .L317 + jle L(317) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-31 * SIZE, A1, LDA, 1, %xmm1) @@ -5768,9 +5768,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L317: +L(317): testq $2, MM - jle .L318 + jle L(318) #ifdef movsd xorps %xmm0, %xmm0 @@ -5800,9 +5800,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L318: +L(318): testq $1, MM - jle .L319 + jle L(319) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -5816,7 +5816,7 @@ addss %xmm2, %xmm10 ALIGN_4 -.L319: +L(319): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm11, %xmm10 @@ -5864,12 +5864,12 @@ movss %xmm9, (Y1) addq INCY, Y1 movss %xmm10, (Y1) - jmp .L999 + jmp L(999) ALIGN_3 -.L320: +L(320): cmpq $2, N - jne .L330 + jne L(330) leaq 32 * SIZE(BUFFER), X1 @@ -5880,10 +5880,10 @@ xorps %xmm9, %xmm9 cmpq $3, M - jle .L327 + jle L(327) testq $SIZE, A1 - je .L32X + je L(32X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -5898,9 +5898,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L32X: +L(32X): testq $2 * SIZE, A1 - je .L32XX + je L(32XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -5922,7 +5922,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L32XX: +L(32XX): movaps -35 * SIZE(A2), %xmm12 MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) @@ -5930,16 +5930,16 @@ movq MM, I sarq $4, I - jle .L325 + jle L(325) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-31 * SIZE, A2, %xmm1) decq I - jle .L323 + jle L(323) ALIGN_4 -.L322: +L(322): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -5997,10 +5997,10 @@ addq $16 * SIZE, X1 decq I - jg .L322 + jg L(322) ALIGN_4 -.L323: +L(323): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-28 * SIZE, A1, %xmm0) @@ -6044,9 +6044,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L325: +L(325): testq $8, MM - jle .L326 + jle L(326) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-31 * SIZE, A2, %xmm1) @@ -6073,9 +6073,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L326: +L(326): testq $4, MM - jle .L327 + jle L(327) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-31 * SIZE, A2, %xmm1) @@ -6092,9 +6092,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L327: +L(327): testq $2, MM - jle .L328 + jle L(328) #ifdef movsd xorps %xmm0, %xmm0 @@ -6119,9 +6119,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L328: +L(328): testq $1, MM - jle .L329 + jle L(329) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -6132,7 +6132,7 @@ addss %xmm1, %xmm9 ALIGN_4 -.L329: +L(329): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm8, %xmm8 @@ -6160,12 +6160,12 @@ addq INCY, Y1 movss %xmm9, (Y1) addq INCY, Y1 - jmp .L999 + jmp L(999) ALIGN_4 -.L330: +L(330): cmpq $1, N - jne .L999 + jne L(999) leaq 32 * SIZE(BUFFER), X1 @@ -6175,10 +6175,10 @@ xorps %xmm9, %xmm9 cmpq $3, M - jle .L337 + jle L(337) testq $SIZE, A1 - je .L33X + je L(33X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -6189,9 +6189,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L33X: +L(33X): testq $2 * SIZE, A1 - je .L33XX + je L(33XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -6207,23 +6207,23 @@ addq $2 * SIZE, X1 ALIGN_3 -.L33XX: +L(33XX): MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) MOVUPS_XL1 (-28 * SIZE, X1, %xmm5) movq MM, I sarq $4, I - jle .L335 + jle L(335) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-28 * SIZE, A1, %xmm12) decq I - jle .L333 + jle L(333) ALIGN_4 -.L332: +L(332): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -6256,10 +6256,10 @@ addq $16 * SIZE, X1 decq I - jg .L332 + jg L(332) ALIGN_4 -.L333: +L(333): mulps %xmm4, %xmm0 MOVUPS_XL1 (-24 * SIZE, X1, %xmm4) addps %xmm0, %xmm8 @@ -6282,9 +6282,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L335: +L(335): testq $8, MM - jle .L336 + jle L(336) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -6299,9 +6299,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L336: +L(336): testq $4, MM - jle .L337 + jle L(337) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -6311,9 +6311,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L337: +L(337): testq $2, MM - jle .L338 + jle L(338) #ifdef movsd xorps %xmm0, %xmm0 @@ -6331,9 +6331,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L338: +L(338): testq $1, MM - jle .L339 + jle L(339) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -6341,7 +6341,7 @@ addss %xmm0, %xmm8 ALIGN_4 -.L339: +L(339): addps %xmm9, %xmm8 #ifdef HAVE_SSE3 @@ -6362,17 +6362,17 @@ addss (Y), %xmm8 movss %xmm8, (Y1) - jmp .L999 + jmp L(999) #endif ALIGN_4 -.L999: +L(999): leaq (,M,SIZE),%rax addq %rax,AA - jmp .L0t + jmp L(0t) ALIGN_4 -.L999x: +L(999x): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/sum.S b/kernel/x86_64/sum.S index 9f2cdc1ecd..792a38cf2e 100644 --- a/kernel/x86_64/sum.S +++ b/kernel/x86_64/sum.S @@ -56,9 +56,9 @@ fldz testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) salq $BASE_SHIFT, INCX @@ -66,14 +66,14 @@ fldz fldz cmpq $SIZE, INCX - jne .L40 + jne L(40) movq M, I sarq $3, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -101,30 +101,30 @@ faddp %st, %st(1) decq I - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): andq $7, M - jle .L998 + jle L(998) ALIGN_4 -.L21: +L(21): FLD (X) faddp %st,%st(1) addq $1 * SIZE, X decq M - jg .L21 - jmp .L998 + jg L(21) + jmp L(998) ALIGN_4 -.L40: +L(40): movq M, I sarq $3, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): FLD (X) addq INCX, X FLD (X) @@ -154,30 +154,30 @@ faddp %st, %st(1) decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): andq $7, M - jle .L998 + jle L(998) ALIGN_4 -.L61: +L(61): FLD (X) addq INCX, X faddp %st,%st(1) decq M - jg .L61 + jg L(61) ALIGN_4 -.L998: +L(998): faddp %st,%st(2) faddp %st,%st(1) faddp %st,%st(1) ALIGN_4 -.L999: +L(999): ret EPILOGUE diff --git a/kernel/x86_64/swap.S b/kernel/x86_64/swap.S index 9529724abe..a81d852c37 100644 --- a/kernel/x86_64/swap.S +++ b/kernel/x86_64/swap.S @@ -82,16 +82,16 @@ salq $BASE_SHIFT, INCY cmpq $SIZE, INCX - jne .L14 + jne L(14) cmpq $SIZE, INCY - jne .L14 + jne L(14) movq N, %rax sarq $3, %rax - jle .L15 + jle L(15) ALIGN_3 -.L16: +L(16): #ifdef XDOUBLE movq 0(X), %mm0 movq 8(X), %mm1 @@ -250,16 +250,16 @@ addq $8 * SIZE, X addq $8 * SIZE, Y decq %rax - jg .L16 + jg L(16) ALIGN_3 -.L15: +L(15): movq N, %rax andq $7, %rax - jle .L27 + jle L(27) ALIGN_3 -.L22: +L(22): #ifdef XDOUBLE movq 0(X), %mm0 @@ -281,21 +281,21 @@ addq $SIZE, X addq $SIZE, Y decq %rax - jg .L22 - jmp .L27 + jg L(22) + jmp L(27) ALIGN_3 /* INCX != 1 or INCY != 1 */ -.L14: +L(14): movq N, %rax movq X, XX movq Y, YY sarq $2, %rax - jle .L28 + jle L(28) ALIGN_2 -.L29: +L(29): #ifdef XDOUBLE movq 0(X), %mm0 movq 8(X), %mm1 @@ -391,16 +391,16 @@ #endif decq %rax - jg .L29 + jg L(29) ALIGN_3 -.L28: +L(28): movq N, %rax andq $3, %rax - jle .L27 + jle L(27) ALIGN_3 -.L35: +L(35): #ifdef XDOUBLE movq 0(X), %mm0 movq 8(X), %mm1 @@ -422,10 +422,10 @@ addq INCY, Y decq %rax - jg .L35 + jg L(35) ALIGN_3 -.L27: +L(27): EMMS xorq %rax,%rax diff --git a/kernel/x86_64/swap_sse.S b/kernel/x86_64/swap_sse.S index dc964dab22..1559772dea 100644 --- a/kernel/x86_64/swap_sse.S +++ b/kernel/x86_64/swap_sse.S @@ -75,18 +75,18 @@ leaq (, INCY, SIZE), INCY cmpq $SIZE, INCX - jne .L50 + jne L(50) cmpq $SIZE, INCY - jne .L50 + jne L(50) subq $-32 * SIZE, X subq $-32 * SIZE, Y cmpq $3, M - jle .L16 + jle L(16) testq $SIZE, Y - je .L05 + je L(05) movss -32 * SIZE(X), %xmm0 movss -32 * SIZE(Y), %xmm1 @@ -99,9 +99,9 @@ decq M ALIGN_3 -.L05: +L(05): testq $2 * SIZE, Y - je .L10 + je L(10) movsd -32 * SIZE(X), %xmm0 movsd -32 * SIZE(Y), %xmm1 @@ -112,25 +112,25 @@ addq $2 * SIZE, X addq $2 * SIZE, Y subq $2, M - jle .L19 + jle L(19) ALIGN_3 -.L10: +L(10): cmpq $3, M - jle .L16 + jle L(16) testq $2 * SIZE, X - jne .L30 + jne L(30) testq $1 * SIZE, X - jne .L20 + jne L(20) movq M, %rax sarq $5, %rax - jle .L13 + jle L(13) ALIGN_3 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -199,12 +199,12 @@ subq $-32 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L13: +L(13): testq $16, M - jle .L14 + jle L(14) movaps -32 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -234,9 +234,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $8, M - jle .L15 + jle L(15) movaps -32 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -254,9 +254,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $4, M - jle .L16 + jle L(16) movaps -32 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -268,9 +268,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $2, M - jle .L17 + jle L(17) movsd -32 * SIZE(X), %xmm0 movsd -32 * SIZE(Y), %xmm1 @@ -281,9 +281,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $1, M - jle .L19 + jle L(19) movss -32 * SIZE(X), %xmm0 movss -32 * SIZE(Y), %xmm1 @@ -292,7 +292,7 @@ movss %xmm0, -32 * SIZE(Y) ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -304,7 +304,7 @@ ret ALIGN_3 -.L20: +L(20): movaps -33 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -316,10 +316,10 @@ movq M, %rax sarq $5, %rax - jle .L23 + jle L(23) ALIGN_4 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -420,12 +420,12 @@ subq $-32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L23: +L(23): testq $16, M - jle .L24 + jle L(24) movaps -29 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -471,9 +471,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $8, M - jle .L25 + jle L(25) movaps -29 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -499,9 +499,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $4, M - jle .L26 + jle L(26) movaps -29 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -520,7 +520,7 @@ addq $4 * SIZE, Y ALIGN_3 -.L26: +L(26): pshufd $0x39, %xmm0, %xmm2 pshufd $0xff, %xmm0, %xmm0 @@ -528,7 +528,7 @@ movss %xmm0, -30 * SIZE(Y) testq $2, M - jle .L27 + jle L(27) movsd -29 * SIZE(X), %xmm0 movsd -29 * SIZE(Y), %xmm1 @@ -540,9 +540,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $1, M - jle .L29 + jle L(29) movss -29 * SIZE(X), %xmm0 movss -29 * SIZE(Y), %xmm1 @@ -551,7 +551,7 @@ movss %xmm1, -29 * SIZE(X) ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -563,9 +563,9 @@ ret ALIGN_3 -.L30: +L(30): testq $1 * SIZE, X - jne .L40 + jne L(40) movhps -32 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -575,10 +575,10 @@ movq M, %rax sarq $5, %rax - jle .L33 + jle L(33) ALIGN_4 -.L31: +L(31): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -663,12 +663,12 @@ subq $-32 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L33: +L(33): testq $16, M - jle .L34 + jle L(34) movaps -30 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -706,9 +706,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L34: +L(34): testq $8, M - jle .L35 + jle L(35) movaps -30 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -730,9 +730,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L35: +L(35): testq $4, M - jle .L36 + jle L(36) movaps -30 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -749,11 +749,11 @@ addq $4 * SIZE, Y ALIGN_3 -.L36: +L(36): movhps %xmm0, -32 * SIZE(Y) testq $2, M - jle .L37 + jle L(37) movsd -30 * SIZE(X), %xmm0 movsd -30 * SIZE(Y), %xmm1 @@ -765,9 +765,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L37: +L(37): testq $1, M - jle .L39 + jle L(39) movss -30 * SIZE(X), %xmm0 movss -30 * SIZE(Y), %xmm1 @@ -776,7 +776,7 @@ movss %xmm1, -30 * SIZE(X) ALIGN_3 -.L39: +L(39): xorq %rax,%rax RESTOREREGISTERS @@ -788,7 +788,7 @@ ret ALIGN_3 -.L40: +L(40): movaps -35 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -798,10 +798,10 @@ movq M, %rax sarq $5, %rax - jle .L43 + jle L(43) ALIGN_4 -.L41: +L(41): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -902,12 +902,12 @@ subq $-32 * SIZE, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L43: +L(43): testq $16, M - jle .L44 + jle L(44) movaps -31 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -953,9 +953,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L44: +L(44): testq $8, M - jle .L45 + jle L(45) movaps -31 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -981,9 +981,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L45: +L(45): testq $4, M - jle .L46 + jle L(46) movaps -31 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -1002,7 +1002,7 @@ addq $4 * SIZE, Y ALIGN_3 -.L46: +L(46): movsd -31 * SIZE(X), %xmm2 pshufd $0x39, %xmm1, %xmm1 @@ -1017,7 +1017,7 @@ addq $3 * SIZE, Y testq $2, M - jle .L47 + jle L(47) movsd -32 * SIZE(X), %xmm0 movsd -32 * SIZE(Y), %xmm1 @@ -1029,9 +1029,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L47: +L(47): testq $1, M - jle .L49 + jle L(49) movss -32 * SIZE(X), %xmm0 movss -32 * SIZE(Y), %xmm1 @@ -1040,7 +1040,7 @@ movss %xmm1, -32 * SIZE(X) ALIGN_3 -.L49: +L(49): xorq %rax,%rax RESTOREREGISTERS @@ -1052,13 +1052,13 @@ ret ALIGN_3 -.L50: +L(50): movq M, %rax sarq $3, %rax - jle .L55 + jle L(55) ALIGN_3 -.L51: +L(51): movss (X), %xmm0 movss (Y), %xmm1 @@ -1124,16 +1124,16 @@ addq INCY, Y decq %rax - jg .L51 + jg L(51) ALIGN_3 -.L55: +L(55): movq M, %rax andq $7, %rax - jle .L57 + jle L(57) ALIGN_3 -.L56: +L(56): movss (X), %xmm0 movss (Y), %xmm1 @@ -1143,10 +1143,10 @@ addq INCX, X addq INCY, Y decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L57: +L(57): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/swap_sse2.S b/kernel/x86_64/swap_sse2.S index e9260b979f..9fa9590103 100644 --- a/kernel/x86_64/swap_sse2.S +++ b/kernel/x86_64/swap_sse2.S @@ -75,12 +75,12 @@ leaq (, INCY, SIZE), INCY cmpq $SIZE, INCX - jne .L40 + jne L(40) cmpq $SIZE, INCY - jne .L40 + jne L(40) testq $SIZE, Y - je .L10 + je L(10) movsd 0 * SIZE(X), %xmm0 movsd 0 * SIZE(Y), %xmm8 @@ -91,22 +91,22 @@ addq $1 * SIZE, X addq $1 * SIZE, Y decq M - jle .L19 + jle L(19) ALIGN_4 -.L10: +L(10): subq $-16 * SIZE, X subq $-16 * SIZE, Y testq $SIZE, X - jne .L20 + jne L(20) movq M, %rax sarq $4, %rax - jle .L13 + jle L(13) ALIGN_3 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -175,12 +175,12 @@ subq $-16 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L13: +L(13): testq $8, M - jle .L14 + jle L(14) movaps -16 * SIZE(X), %xmm0 movaps -16 * SIZE(Y), %xmm1 @@ -210,9 +210,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $4, M - jle .L15 + jle L(15) movaps -16 * SIZE(X), %xmm0 movaps -16 * SIZE(Y), %xmm1 @@ -230,9 +230,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $2, M - jle .L16 + jle L(16) movaps -16 * SIZE(X), %xmm0 movaps -16 * SIZE(Y), %xmm1 @@ -244,9 +244,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $1, M - jle .L19 + jle L(19) movsd -16 * SIZE(X), %xmm0 movsd -16 * SIZE(Y), %xmm1 @@ -255,7 +255,7 @@ movlps %xmm0, -16 * SIZE(Y) ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -267,20 +267,20 @@ ret ALIGN_3 -.L20: +L(20): movhps -16 * SIZE(X), %xmm0 movaps -16 * SIZE(Y), %xmm1 movlps %xmm1, -16 * SIZE(X) decq M - jle .L29 + jle L(29) movq M, %rax sarq $4, %rax - jle .L23 + jle L(23) ALIGN_4 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -365,12 +365,12 @@ subq $-16 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L23: +L(23): testq $8, M - jle .L24 + jle L(24) movaps -15 * SIZE(X), %xmm2 movaps -14 * SIZE(Y), %xmm3 @@ -408,9 +408,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $4, M - jle .L25 + jle L(25) movaps -15 * SIZE(X), %xmm2 movaps -14 * SIZE(Y), %xmm3 @@ -432,9 +432,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $2, M - jle .L26 + jle L(26) movaps -15 * SIZE(X), %xmm2 movaps -14 * SIZE(Y), %xmm3 @@ -451,9 +451,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $1, M - jle .L29 + jle L(29) movhps %xmm0, -16 * SIZE(Y) movhps -15 * SIZE(X), %xmm0 @@ -463,7 +463,7 @@ addq $SIZE, Y ALIGN_3 -.L29: +L(29): movhps %xmm0, -16 * SIZE(Y) xorq %rax,%rax @@ -477,13 +477,13 @@ ret ALIGN_3 -.L40: +L(40): movq M, %rax sarq $3, %rax - jle .L45 + jle L(45) ALIGN_3 -.L41: +L(41): movsd (X), %xmm0 movsd (Y), %xmm1 @@ -549,16 +549,16 @@ addq INCY, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L45: +L(45): movq M, %rax andq $7, %rax - jle .L47 + jle L(47) ALIGN_3 -.L46: +L(46): movsd (X), %xmm0 movsd (Y), %xmm1 @@ -568,10 +568,10 @@ addq INCX, X addq INCY, Y decq %rax - jg .L46 + jg L(46) ALIGN_3 -.L47: +L(47): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/symv_L_sse.S b/kernel/x86_64/symv_L_sse.S index 55780734f2..7a76e73030 100644 --- a/kernel/x86_64/symv_L_sse.S +++ b/kernel/x86_64/symv_L_sse.S @@ -206,7 +206,7 @@ leaq (,LDA, SIZE), LDA testq M, M - jle .L999 + jle L(999) shufps $0, ALPHA, ALPHA @@ -214,10 +214,10 @@ movq M, %rax sarq $3, %rax - jle .L02 + jle L(02) ALIGN_3 -.L01: +L(01): movss 0 * SIZE(X), %xmm1 addq INCX, X movss 0 * SIZE(X), %xmm2 @@ -255,16 +255,16 @@ addq $8 * SIZE, XX decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L02: +L(02): movq M, %rax andq $7, %rax - jle .L05 + jle L(05) ALIGN_3 -.L03: +L(03): movss 0 * SIZE(X), %xmm1 addq INCX, X @@ -274,10 +274,10 @@ addq $1 * SIZE, XX decq %rax - jg .L03 + jg L(03) ALIGN_3 -.L05: +L(05): /* now we don't need original X */ movq Y, NEW_Y @@ -285,17 +285,17 @@ andq $-512, XX cmpq $SIZE, INCY - je .L10 + je L(10) movq Y, YY movq XX, NEW_Y movq M, %rax sarq $3, %rax - jle .L07 + jle L(07) ALIGN_3 -.L06: +L(06): movss 0 * SIZE(YY), %xmm0 addq INCY, YY movss 0 * SIZE(YY), %xmm1 @@ -324,16 +324,16 @@ addq $8 * SIZE, XX decq %rax - jg .L06 + jg L(06) ALIGN_3 -.L07: +L(07): movq M, %rax andq $7, %rax - jle .L10 + jle L(10) ALIGN_3 -.L08: +L(08): movss 0 * SIZE(YY), %xmm0 addq INCY, YY @@ -341,17 +341,17 @@ addq $1 * SIZE, XX decq %rax - jg .L08 + jg L(08) ALIGN_3 -.L10: +L(10): xorq IS, IS # is = 0 cmpq $4, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): movq A, A1 leaq (A, LDA, 2), A2 leaq 4 * SIZE(A, LDA, 4), A @@ -420,10 +420,10 @@ subq IS, I subq $4, I sarq $4, I - jle .L14 + jle L(14) ALIGN_3 -.L12: +L(12): movaps xtemp1, xt1 mulps a1, xt1 mulps atemp1, a1 @@ -599,15 +599,15 @@ addq $16 * SIZE, A2 decq I - jg .L12 + jg L(12) ALIGN_3 -.L14: +L(14): movq M, I subq IS, I subq $4, I test $8, I - jle .L15 + jle L(15) movaps xtemp1, xt1 mulps a1, xt1 @@ -691,9 +691,9 @@ addq $8 * SIZE, A2 ALIGN_3 -.L15: +L(15): test $4, I - jle .L17 + jle L(17) movaps xtemp1, xt1 mulps a1, xt1 @@ -734,9 +734,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L17: +L(17): testq $2, M - jle .L18 + jle L(18) pxor xtemp2, xtemp2 @@ -782,9 +782,9 @@ addq $2 * SIZE, A2 ALIGN_3 -.L18: +L(18): testq $1, M - jle .L19 + jle L(19) movss 0 * SIZE(XX), xtemp1 @@ -822,7 +822,7 @@ movss yy1, 0 * SIZE(YY) ALIGN_3 -.L19: +L(19): #ifndef HAVE_SSE3 movaps xsum1, xtemp1 unpcklps xsum3, xsum1 @@ -863,12 +863,12 @@ movq IS, I addq $4, I cmpq N, I - jle .L11 + jle L(11) ALIGN_3 -.L20: +L(20): testq $2, N - jle .L30 + jle L(30) movq A, A1 leaq 2 * SIZE(A, LDA, 2), A @@ -890,7 +890,7 @@ pshufd $0x55, atemp4, atemp2 testq $1, M - jle .L29 + jle L(29) movss 2 * SIZE(A1), a1 movss 2 * SIZE(A1, LDA, 1), a2 @@ -912,7 +912,7 @@ movss yy1, 2 * SIZE(NEW_Y, IS, SIZE) ALIGN_3 -.L29: +L(29): #ifndef HAVE_SSE3 unpcklps xsum2, xsum1 @@ -932,9 +932,9 @@ addq $2, IS ALIGN_3 -.L30: +L(30): testq $1, N - jle .L990 + jle L(990) movss 0 * SIZE(NEW_X, IS, SIZE), xsum1 mulss 0 * SIZE(A), xsum1 @@ -942,16 +942,16 @@ movss xsum1, 0 * SIZE(NEW_Y, IS, SIZE) ALIGN_3 -.L990: +L(990): cmpq $SIZE, INCY - je .L999 + je L(999) movq M, %rax sarq $3, %rax - jle .L997 + jle L(997) ALIGN_3 -.L996: +L(996): movss 0 * SIZE(NEW_Y), %xmm0 movss 1 * SIZE(NEW_Y), %xmm1 movss 2 * SIZE(NEW_Y), %xmm2 @@ -980,16 +980,16 @@ addq $8 * SIZE, NEW_Y decq %rax - jg .L996 + jg L(996) ALIGN_3 -.L997: +L(997): movq M, %rax andq $7, %rax - jle .L999 + jle L(999) ALIGN_3 -.L998: +L(998): movss 0 * SIZE(NEW_Y), %xmm0 movss %xmm0, 0 * SIZE(Y) @@ -998,10 +998,10 @@ addq $1 * SIZE, NEW_Y decq %rax - jg .L998 + jg L(998) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/symv_L_sse2.S b/kernel/x86_64/symv_L_sse2.S index 77331d95f2..12c20a5550 100644 --- a/kernel/x86_64/symv_L_sse2.S +++ b/kernel/x86_64/symv_L_sse2.S @@ -206,7 +206,7 @@ leaq (,LDA, SIZE), LDA testq M, M - jle .L999 + jle L(999) unpcklpd ALPHA, ALPHA @@ -214,10 +214,10 @@ movq M, %rax sarq $3, %rax - jle .L02 + jle L(02) ALIGN_3 -.L01: +L(01): movsd 0 * SIZE(X), %xmm1 addq INCX, X movhpd 0 * SIZE(X), %xmm1 @@ -247,16 +247,16 @@ addq $8 * SIZE, XX decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L02: +L(02): movq M, %rax andq $7, %rax - jle .L05 + jle L(05) ALIGN_3 -.L03: +L(03): movsd 0 * SIZE(X), %xmm1 addq INCX, X @@ -266,10 +266,10 @@ addq $1 * SIZE, XX decq %rax - jg .L03 + jg L(03) ALIGN_3 -.L05: +L(05): /* now we don't need original X */ movq Y, NEW_Y @@ -277,17 +277,17 @@ andq $-512, XX cmpq $SIZE, INCY - je .L10 + je L(10) movq Y, YY movq XX, NEW_Y movq M, %rax sarq $3, %rax - jle .L07 + jle L(07) ALIGN_3 -.L06: +L(06): movsd 0 * SIZE(YY), %xmm0 addq INCY, YY movhpd 0 * SIZE(YY), %xmm0 @@ -312,16 +312,16 @@ addq $8 * SIZE, XX decq %rax - jg .L06 + jg L(06) ALIGN_3 -.L07: +L(07): movq M, %rax andq $7, %rax - jle .L10 + jle L(10) ALIGN_3 -.L08: +L(08): movsd 0 * SIZE(YY), %xmm0 addq INCY, YY @@ -329,17 +329,17 @@ addq $1 * SIZE, XX decq %rax - jg .L08 + jg L(08) ALIGN_3 -.L10: +L(10): xorq IS, IS # is = 0 cmpq $4, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): movq A, A1 leaq (A, LDA, 2), A2 leaq 4 * SIZE(A, LDA, 4), A @@ -423,10 +423,10 @@ subq IS, I subq $4, I sarq $3, I - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): movapd xtemp1, xt1 mulpd a1, xt1 mulpd atemp1, a1 @@ -601,15 +601,15 @@ addq $8 * SIZE, A2 decq I - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): movq M, I subq IS, I subq $4, I test $4, I - jle .L17 + jle L(17) movapd xtemp1, xt1 mulpd a1, xt1 @@ -693,9 +693,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L17: +L(17): testq $2, M - jle .L18 + jle L(18) movapd xtemp1, xt1 mulpd a1, xt1 @@ -739,9 +739,9 @@ addq $2 * SIZE, A2 ALIGN_3 -.L18: +L(18): testq $1, M - jle .L19 + jle L(19) movapd xtemp1, xt1 mulsd a1, xt1 @@ -773,7 +773,7 @@ movsd yy1, 0 * SIZE(YY) ALIGN_3 -.L19: +L(19): #ifndef HAVE_SSE3 movapd xsum1, atemp1 movapd xsum3, atemp3 @@ -809,12 +809,12 @@ movq IS, I addq $4, I cmpq N, I - jle .L11 + jle L(11) ALIGN_3 -.L20: +L(20): testq $2, N - jle .L30 + jle L(30) movq A, A1 leaq 2 * SIZE(A, LDA, 2), A @@ -838,7 +838,7 @@ unpckhpd atemp2, atemp2 testq $1, M - jle .L29 + jle L(29) movsd 2 * SIZE(A1), a1 movsd 2 * SIZE(A1, LDA, 1), a2 @@ -860,7 +860,7 @@ movsd yy1, 2 * SIZE(NEW_Y, IS, SIZE) ALIGN_3 -.L29: +L(29): #ifndef HAVE_SSE3 movapd xsum1, atemp1 unpcklpd xsum2, xsum1 @@ -881,9 +881,9 @@ addq $2, IS ALIGN_3 -.L30: +L(30): testq $1, N - jle .L990 + jle L(990) movsd 0 * SIZE(A), xsum1 movsd 0 * SIZE(NEW_X, IS, SIZE), atemp1 @@ -894,16 +894,16 @@ movsd yy1, 0 * SIZE(NEW_Y, IS, SIZE) ALIGN_3 -.L990: +L(990): cmpq $SIZE, INCY - je .L999 + je L(999) movq M, %rax sarq $3, %rax - jle .L997 + jle L(997) ALIGN_3 -.L996: +L(996): movapd 0 * SIZE(NEW_Y), %xmm0 movapd 2 * SIZE(NEW_Y), %xmm1 movapd 4 * SIZE(NEW_Y), %xmm2 @@ -928,16 +928,16 @@ addq $8 * SIZE, NEW_Y decq %rax - jg .L996 + jg L(996) ALIGN_3 -.L997: +L(997): movq M, %rax andq $7, %rax - jle .L999 + jle L(999) ALIGN_3 -.L998: +L(998): movsd 0 * SIZE(NEW_Y), %xmm0 movsd %xmm0, 0 * SIZE(Y) @@ -946,11 +946,11 @@ addq $1 * SIZE, NEW_Y decq %rax - jg .L998 + jg L(998) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/symv_U_sse.S b/kernel/x86_64/symv_U_sse.S index b611823039..d11a2a7ede 100644 --- a/kernel/x86_64/symv_U_sse.S +++ b/kernel/x86_64/symv_U_sse.S @@ -206,7 +206,7 @@ leaq (,LDA, SIZE), LDA testq M, M - jle .L999 + jle L(999) negq IS addq M, IS @@ -221,10 +221,10 @@ movq M, %rax sarq $3, %rax - jle .L02 + jle L(02) ALIGN_3 -.L01: +L(01): movss 0 * SIZE(X), %xmm1 addq INCX, X movss 0 * SIZE(X), %xmm2 @@ -262,16 +262,16 @@ addq $8 * SIZE, XX decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L02: +L(02): movq M, %rax andq $7, %rax - jle .L05 + jle L(05) ALIGN_3 -.L03: +L(03): movss 0 * SIZE(X), %xmm1 addq INCX, X @@ -281,10 +281,10 @@ addq $1 * SIZE, XX decq %rax - jg .L03 + jg L(03) ALIGN_3 -.L05: +L(05): /* now we don't need original X */ movq Y, NEW_Y @@ -292,17 +292,17 @@ andq $-512, XX cmpq $SIZE, INCY - je .L10 + je L(10) movq Y, YY movq XX, NEW_Y movq M, %rax sarq $3, %rax - jle .L07 + jle L(07) ALIGN_3 -.L06: +L(06): movss 0 * SIZE(YY), %xmm0 addq INCY, YY movss 0 * SIZE(YY), %xmm1 @@ -331,16 +331,16 @@ addq $8 * SIZE, XX decq %rax - jg .L06 + jg L(06) ALIGN_3 -.L07: +L(07): movq M, %rax andq $7, %rax - jle .L10 + jle L(10) ALIGN_3 -.L08: +L(08): movss 0 * SIZE(YY), %xmm0 addq INCY, YY @@ -348,17 +348,17 @@ addq $1 * SIZE, XX decq %rax - jg .L08 + jg L(08) ALIGN_3 -.L10: +L(10): movq IS, I addq $4, I cmpq M, I - jg .L20 + jg L(20) ALIGN_3 -.L11: +L(11): movq A, A1 leaq (A, LDA, 2), A2 leaq (A, LDA, 4), A @@ -395,10 +395,10 @@ movq IS, I sarq $4, I - jle .L14 + jle L(14) ALIGN_3 -.L12: +L(12): movaps xtemp1, xt1 mulps a1, xt1 mulps atemp1, a1 @@ -574,12 +574,12 @@ addq $16 * SIZE, A2 decq I - jg .L12 + jg L(12) ALIGN_3 -.L14: +L(14): testq $8, IS - jle .L15 + jle L(15) movaps xtemp1, xt1 mulps a1, xt1 @@ -663,9 +663,9 @@ addq $8 * SIZE, A2 ALIGN_3 -.L15: +L(15): testq $4, IS - jle .L18 + jle L(18) movaps xtemp1, xt1 mulps a1, xt1 @@ -702,7 +702,7 @@ addq $4 * SIZE, A2 ALIGN_3 -.L18: +L(18): movaps 0 * SIZE(NEW_X, IS, SIZE), atemp1 movss 0 * SIZE(A1), a1 @@ -779,12 +779,12 @@ movq IS, I addq $4, I cmpq M, I - jle .L11 + jle L(11) ALIGN_3 -.L20: +L(20): testq $2, M - jle .L30 + jle L(30) movq A, A1 leaq (A, LDA, 2), A @@ -812,10 +812,10 @@ movq IS, I sarq $2, I - jle .L28 + jle L(28) ALIGN_3 -.L22: +L(22): movaps xtemp1, xt1 mulps a1, xt1 mulps atemp1, a1 @@ -843,10 +843,10 @@ addq $4 * SIZE, A1 decq I - jg .L22 + jg L(22) ALIGN_3 -.L28: +L(28): movsd 0 * SIZE(NEW_X, IS, SIZE), atemp1 movss 0 * SIZE(A1), a1 @@ -883,9 +883,9 @@ addq $2, IS ALIGN_3 -.L30: +L(30): testq $1, M - jle .L990 + jle L(990) movq A, A1 @@ -909,10 +909,10 @@ movq IS, I sarq $1, I - jle .L38 + jle L(38) ALIGN_3 -.L32: +L(32): movaps xtemp1, xt1 movss 2 * SIZE(XX), xtemp1 mulps a1, xt1 @@ -940,10 +940,10 @@ addq $2 * SIZE, A1 decq I - jg .L32 + jg L(32) ALIGN_3 -.L38: +L(38): movsd 0 * SIZE(NEW_X, IS, SIZE), atemp1 movss 0 * SIZE(A1), a1 @@ -971,16 +971,16 @@ addq $2, IS ALIGN_3 -.L990: +L(990): cmpq $SIZE, INCY - je .L999 + je L(999) movq M, %rax sarq $3, %rax - jle .L997 + jle L(997) ALIGN_3 -.L996: +L(996): movss 0 * SIZE(NEW_Y), %xmm0 movss 1 * SIZE(NEW_Y), %xmm1 movss 2 * SIZE(NEW_Y), %xmm2 @@ -1009,16 +1009,16 @@ addq $8 * SIZE, NEW_Y decq %rax - jg .L996 + jg L(996) ALIGN_3 -.L997: +L(997): movq M, %rax andq $7, %rax - jle .L999 + jle L(999) ALIGN_3 -.L998: +L(998): movss 0 * SIZE(NEW_Y), %xmm0 movss %xmm0, 0 * SIZE(Y) @@ -1027,11 +1027,11 @@ addq $1 * SIZE, NEW_Y decq %rax - jg .L998 + jg L(998) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/symv_U_sse2.S b/kernel/x86_64/symv_U_sse2.S index 99bc07d502..d531febd99 100644 --- a/kernel/x86_64/symv_U_sse2.S +++ b/kernel/x86_64/symv_U_sse2.S @@ -205,7 +205,7 @@ leaq (,LDA, SIZE), LDA testq M, M - jle .L999 + jle L(999) negq IS addq M, IS @@ -220,10 +220,10 @@ movq M, %rax sarq $3, %rax - jle .L02 + jle L(02) ALIGN_3 -.L01: +L(01): movsd 0 * SIZE(X), %xmm1 addq INCX, X movhpd 0 * SIZE(X), %xmm1 @@ -253,16 +253,16 @@ addq $8 * SIZE, XX decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L02: +L(02): movq M, %rax andq $7, %rax - jle .L05 + jle L(05) ALIGN_3 -.L03: +L(03): movsd 0 * SIZE(X), %xmm1 addq INCX, X @@ -272,10 +272,10 @@ addq $1 * SIZE, XX decq %rax - jg .L03 + jg L(03) ALIGN_3 -.L05: +L(05): /* now we don't need original X */ movq Y, NEW_Y @@ -283,17 +283,17 @@ andq $-512, XX cmpq $SIZE, INCY - je .L10 + je L(10) movq Y, YY movq XX, NEW_Y movq M, %rax sarq $3, %rax - jle .L07 + jle L(07) ALIGN_3 -.L06: +L(06): movsd 0 * SIZE(YY), %xmm0 addq INCY, YY movhpd 0 * SIZE(YY), %xmm0 @@ -318,16 +318,16 @@ addq $8 * SIZE, XX decq %rax - jg .L06 + jg L(06) ALIGN_3 -.L07: +L(07): movq M, %rax andq $7, %rax - jle .L10 + jle L(10) ALIGN_3 -.L08: +L(08): movsd 0 * SIZE(YY), %xmm0 addq INCY, YY @@ -335,17 +335,17 @@ addq $1 * SIZE, XX decq %rax - jg .L08 + jg L(08) ALIGN_3 -.L10: +L(10): movq IS, I addq $4, I cmpq M, I - jg .L20 + jg L(20) ALIGN_3 -.L11: +L(11): movq A, A1 leaq (A, LDA, 2), A2 leaq (A, LDA, 4), A @@ -391,10 +391,10 @@ movq IS, I sarq $3, I - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): movapd xtemp1, xt1 mulpd a1, xt1 mulpd atemp1, a1 @@ -569,12 +569,12 @@ addq $8 * SIZE, A2 decq I - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): testq $4, IS - jle .L18 + jle L(18) movapd xtemp1, xt1 mulpd a1, xt1 @@ -652,7 +652,7 @@ addq $4 * SIZE, A2 ALIGN_3 -.L18: +L(18): unpckhpd atemp2, atemp1 unpckhpd atemp4, atemp3 @@ -726,15 +726,15 @@ movq IS, I addq $4, I cmpq M, I - jle .L11 + jle L(11) ALIGN_3 -.L20: +L(20): testq $2, M - je .L30 + je L(30) ALIGN_3 -.L21: +L(21): movq A, A1 leaq (A, LDA, 2), A @@ -766,10 +766,10 @@ movq IS, I sarq $1, I - jle .L28 + jle L(28) ALIGN_3 -.L22: +L(22): movapd xtemp1, xt1 mulpd a1, xt1 mulpd atemp1, a1 @@ -797,10 +797,10 @@ addq $2 * SIZE, A1 decq I - jg .L22 + jg L(22) ALIGN_3 -.L28: +L(28): unpckhpd atemp2, atemp1 movsd 0 * SIZE(A1), a1 @@ -832,12 +832,12 @@ addq $2, IS ALIGN_3 -.L30: +L(30): testq $1, M - je .L990 + je L(990) ALIGN_3 -.L31: +L(31): movq A, A1 #ifdef HAVE_SSE3 @@ -858,10 +858,10 @@ movq IS, I testq I, I - jle .L38 + jle L(38) ALIGN_3 -.L32: +L(32): movapd xtemp1, xt1 mulpd a1, xt1 mulpd atemp1, a1 @@ -879,10 +879,10 @@ addq $1 * SIZE, A1 decq I - jg .L32 + jg L(32) ALIGN_3 -.L38: +L(38): movsd 0 * SIZE(A1), a1 mulsd atemp1, a1 addsd a1, xsum1 @@ -892,16 +892,16 @@ movsd yy1, 0 * SIZE(YY) ALIGN_3 -.L990: +L(990): cmpq $SIZE, INCY - je .L999 + je L(999) movq M, %rax sarq $3, %rax - jle .L997 + jle L(997) ALIGN_3 -.L996: +L(996): movapd 0 * SIZE(NEW_Y), %xmm0 movapd 2 * SIZE(NEW_Y), %xmm1 movapd 4 * SIZE(NEW_Y), %xmm2 @@ -926,16 +926,16 @@ addq $8 * SIZE, NEW_Y decq %rax - jg .L996 + jg L(996) ALIGN_3 -.L997: +L(997): movq M, %rax andq $7, %rax - jle .L999 + jle L(999) ALIGN_3 -.L998: +L(998): movsd 0 * SIZE(NEW_Y), %xmm0 movsd %xmm0, 0 * SIZE(Y) @@ -944,11 +944,11 @@ addq $1 * SIZE, NEW_Y decq %rax - jg .L998 + jg L(998) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LN_2x8_nehalem.S b/kernel/x86_64/trsm_kernel_LN_2x8_nehalem.S index 8deff20aa9..30cc844359 100644 --- a/kernel/x86_64/trsm_kernel_LN_2x8_nehalem.S +++ b/kernel/x86_64/trsm_kernel_LN_2x8_nehalem.S @@ -167,10 +167,10 @@ movq N, J sarq $3, J NOBRANCH - jle .L30 + jle L(30) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -209,7 +209,7 @@ testq $1, M BRANCH - jle .L20 + jle L(20) ALIGN_4 #ifdef LN @@ -243,10 +243,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -308,10 +308,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -320,10 +320,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -343,10 +343,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -683,14 +683,14 @@ #endif ALIGN_4 -.L20: +L(20): movq M, I sarq $1, I NOBRANCH - jle .L29 + jle L(29) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -744,10 +744,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm12 @@ -870,10 +870,10 @@ subq $-8 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -882,10 +882,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addpd %xmm1, %xmm12 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm13 @@ -921,10 +921,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1372,10 +1372,10 @@ decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L29: +L(29): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 8), B @@ -1394,12 +1394,12 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L30: +L(30): testq $4, N - jle .L50 + jle L(50) ALIGN_4 #if defined(LT) || defined(RN) @@ -1436,7 +1436,7 @@ testq $1, M BRANCH - jle .L40 + jle L(40) ALIGN_4 #ifdef LN @@ -1470,10 +1470,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1511,10 +1511,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1523,10 +1523,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1540,10 +1540,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1701,14 +1701,14 @@ #endif ALIGN_4 -.L40: +L(40): movq M, I sarq $1, I NOBRANCH - jle .L49 + jle L(49) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -1748,10 +1748,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -1823,10 +1823,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1835,10 +1835,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): addpd %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm9 @@ -1860,10 +1860,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2091,10 +2091,10 @@ decq I BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L49: +L(49): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -2112,9 +2112,9 @@ #endif ALIGN_4 -.L50: +L(50): testq $2, N - jle .L70 + jle L(70) ALIGN_4 #if defined(LT) || defined(RN) @@ -2151,7 +2151,7 @@ testq $1, M BRANCH - jle .L60 + jle L(60) ALIGN_4 #ifdef LN @@ -2183,10 +2183,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): mulpd %xmm0, %xmm1 movddup -15 * SIZE(AO), %xmm0 addpd %xmm1, %xmm8 @@ -2212,10 +2212,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2224,10 +2224,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): mulpd %xmm0, %xmm1 movddup -15 * SIZE(AO), %xmm0 addpd %xmm1, %xmm8 @@ -2238,10 +2238,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2344,14 +2344,14 @@ #endif ALIGN_4 -.L60: +L(60): movq M, I sarq $1, I NOBRANCH - jle .L69 + jle L(69) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -2387,10 +2387,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -2430,13 +2430,13 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 ALIGN_3 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2445,10 +2445,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addpd %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm9 @@ -2462,10 +2462,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2600,10 +2600,10 @@ decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L69: +L(69): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2621,9 +2621,9 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, N - jle .L999 + jle L(999) ALIGN_4 #if defined(LT) || defined(RN) @@ -2658,7 +2658,7 @@ testq $1, M BRANCH - jle .L80 + jle L(80) ALIGN_4 #ifdef LN @@ -2692,10 +2692,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): mulpd %xmm0, %xmm1 movsd -14 * SIZE(AO), %xmm0 movhps -13 * SIZE(AO), %xmm0 @@ -2715,12 +2715,12 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) addpd %xmm9, %xmm8 ALIGN_3 -.L85: +L(85): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2729,10 +2729,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): mulsd %xmm0, %xmm1 movsd -15 * SIZE(AO), %xmm0 addsd %xmm1, %xmm8 @@ -2743,10 +2743,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -2819,14 +2819,14 @@ ALIGN_4 -.L80: +L(80): movq M, I sarq $1, I NOBRANCH - jle .L89 + jle L(89) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -2861,10 +2861,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -2892,12 +2892,12 @@ subq $1, %rax BRANCH - jg .L72 + jg L(72) addpd %xmm9, %xmm8 ALIGN_3 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2906,10 +2906,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addpd %xmm1, %xmm8 movddup -16 * SIZE(BO), %xmm1 mulpd %xmm0, %xmm1 @@ -2920,10 +2920,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3024,10 +3024,10 @@ decq I BRANCH - jg .L71 + jg L(71) ALIGN_4 -.L89: +L(89): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -3046,7 +3046,7 @@ ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LN_4x2_atom.S b/kernel/x86_64/trsm_kernel_LN_4x2_atom.S index a192674999..60bec8cc0a 100644 --- a/kernel/x86_64/trsm_kernel_LN_4x2_atom.S +++ b/kernel/x86_64/trsm_kernel_LN_4x2_atom.S @@ -156,10 +156,10 @@ movq N, J sarq $1, J - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -197,7 +197,7 @@ #endif testq $1, M - je .L20 + je L(20) #ifdef LN movq K, %rax @@ -233,10 +233,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): addsd %xmm5, %xmm8 movsd 2 * SIZE(BO), %xmm5 mulsd %xmm0, %xmm1 @@ -278,10 +278,10 @@ addq $8 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -294,10 +294,10 @@ andq $3, %rax BRANCH BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulsd %xmm0, %xmm1 addq $2 * SIZE, BO mulsd %xmm0, %xmm3 @@ -311,10 +311,10 @@ addq $1 * SIZE, AO decq %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -413,10 +413,10 @@ #endif ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - je .L30 + je L(30) #ifdef LN movq K, %rax @@ -456,10 +456,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addsd %xmm2, %xmm9 movaps %xmm0, %xmm2 @@ -537,10 +537,10 @@ addq $8 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -549,10 +549,10 @@ #endif andq $3, %rax BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): addsd %xmm2, %xmm9 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -575,10 +575,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): addsd %xmm2, %xmm9 addsd %xmm6, %xmm11 @@ -740,13 +740,13 @@ #endif ALIGN_4 -.L30: +L(30): movq M, I sarq $2, I - jle .L39 + jle L(39) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -795,10 +795,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): addsd %xmm2, %xmm13 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps %xmm0, %xmm2 @@ -941,10 +941,10 @@ mulsd %xmm3, %xmm6 movsd 1 * SIZE(BO), %xmm3 - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -953,10 +953,10 @@ #endif andq $3, %rax BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): addsd %xmm2, %xmm13 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -995,10 +995,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): addsd %xmm2, %xmm13 addsd %xmm7, %xmm14 addsd %xmm6, %xmm15 @@ -1298,10 +1298,10 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -1319,12 +1319,12 @@ #endif decq J # j -- - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $1, N - je .L999 + je L(999) ALIGN_4 #if defined(LT) || defined(RN) @@ -1358,7 +1358,7 @@ #endif testq $1, M - je .L50 + je L(50) #ifdef LN movq K, %rax @@ -1395,10 +1395,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): addsd %xmm5, %xmm8 movsd 2 * SIZE(BO), %xmm5 mulsd %xmm0, %xmm1 @@ -1423,13 +1423,13 @@ addq $4 * SIZE, BO decq %rax - jne .L62 + jne L(62) addsd %xmm5, %xmm8 addsd %xmm7, %xmm9 ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1438,10 +1438,10 @@ #endif andq $3, %rax BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): movsd 0 * SIZE(AO), %xmm0 movsd 0 * SIZE(BO), %xmm1 @@ -1453,10 +1453,10 @@ decq %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addsd %xmm9, %xmm8 #if defined(LN) || defined(RT) @@ -1530,9 +1530,9 @@ #endif ALIGN_4 -.L50: +L(50): testq $2, M - je .L60 + je L(60) #ifdef LN movq K, %rax @@ -1567,10 +1567,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): addsd %xmm2, %xmm8 movsd 2 * SIZE(AO), %xmm2 mulsd %xmm4, %xmm0 @@ -1611,10 +1611,10 @@ mulsd %xmm5, %xmm3 movsd 1 * SIZE(BO), %xmm5 - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1626,10 +1626,10 @@ andq $3, %rax BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): mulsd %xmm4, %xmm0 mulsd %xmm4, %xmm1 movsd 1 * SIZE(BO), %xmm4 @@ -1643,10 +1643,10 @@ addq $1 * SIZE, BO decq %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1743,13 +1743,13 @@ #endif ALIGN_4 -.L60: +L(60): movq M, I sarq $2, I - jle .L69 + jle L(69) ALIGN_4 -.L41: +L(41): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1790,10 +1790,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): addsd %xmm9, %xmm8 movsd 4 * SIZE(AO), %xmm9 mulsd %xmm4, %xmm0 @@ -1867,10 +1867,10 @@ mulsd %xmm5, %xmm15 movsd 1 * SIZE(BO), %xmm5 - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1886,10 +1886,10 @@ andq $3, %rax BRANCH BRANCH - je .L49 + je L(49) ALIGN_4 -.L46: +L(46): mulsd %xmm4, %xmm0 mulsd %xmm4, %xmm1 mulsd %xmm4, %xmm2 @@ -1909,10 +1909,10 @@ addq $1 * SIZE, BO decq %rax BRANCH - jg .L46 + jg L(46) ALIGN_4 -.L49: +L(49): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2065,10 +2065,10 @@ #endif decq I # i -- - jg .L41 + jg L(41) ALIGN_4 -.L69: +L(69): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -2087,7 +2087,7 @@ #endif ALIGN_2 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LN_4x4_barcelona.S b/kernel/x86_64/trsm_kernel_LN_4x4_barcelona.S index 69278bb5e8..e5e69c5451 100644 --- a/kernel/x86_64/trsm_kernel_LN_4x4_barcelona.S +++ b/kernel/x86_64/trsm_kernel_LN_4x4_barcelona.S @@ -480,9 +480,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -520,7 +520,7 @@ #endif testq $1, M - je .L20 + je L(20) #ifdef LN movq K, %rax @@ -565,10 +565,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L36 + je L(36) ALIGN_4 -.L32: +L(32): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO, %rax, 4), %xmm0 addpd %xmm1, %xmm8 @@ -596,10 +596,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L32 + jl L(32) ALIGN_4 -.L36: +L(36): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -607,7 +607,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L38 + je L(38) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -615,7 +615,7 @@ negq %rax ALIGN_4 -.L37: +L(37): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO, %rax, 4), %xmm0 addpd %xmm1, %xmm8 @@ -624,10 +624,10 @@ movddup -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L37 + jl L(37) ALIGN_4 -.L38: +L(38): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -808,12 +808,12 @@ #endif ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -857,10 +857,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L26 + je L(26) ALIGN_4 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -916,10 +916,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L22 + jl L(22) ALIGN_4 -.L26: +L(26): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -927,7 +927,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L29 + je L(29) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -935,7 +935,7 @@ negq %rax ALIGN_4 -.L27: +L(27): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -951,10 +951,10 @@ movddup -11 * SIZE(BO, %rax, 4), %xmm5 addq $SIZE, %rax - jl .L27 + jl L(27) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1172,13 +1172,13 @@ #endif ALIGN_4 -.L30: +L(30): movq M, I sarq $2, I # i = (m >> 2) - jle .L39 + jle L(39) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1247,10 +1247,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -1260,10 +1260,10 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) BRANCH - jl .L12 + jl L(12) ALIGN_4 -.L15: +L(15): prefetch 14 * SIZE(BB) subq $-16 * SIZE, BB @@ -1274,7 +1274,7 @@ subq KK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_4 @@ -1287,7 +1287,7 @@ subq $-16 * SIZE, AO ALIGN_4 -.L16: +L(16): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1295,7 +1295,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L19 + je L(19) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -1303,7 +1303,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -1330,10 +1330,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L17 + jl L(17) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1740,10 +1740,10 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1762,12 +1762,12 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $2, N - je .L80 + je L(80) #if defined(LT) || defined(RN) movq A, AO @@ -1802,10 +1802,10 @@ #endif testq $1, M - je .L60 + je L(60) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1848,10 +1848,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L76 + je L(76) ALIGN_4 -.L72: +L(72): mulpd -16 * SIZE(BO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -12 * SIZE(AO, %rax, 1), %xmm0 @@ -1870,10 +1870,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L72 + jl L(72) ALIGN_4 -.L76: +L(76): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1881,7 +1881,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L78 + je L(78) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -1889,16 +1889,16 @@ negq %rax ALIGN_4 -.L77: +L(77): mulpd -16 * SIZE(BO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L77 + jl L(77) ALIGN_4 -.L78: +L(78): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -2001,9 +2001,9 @@ #endif ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) #ifdef LN movq K, %rax @@ -2047,10 +2047,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L66 + je L(66) ALIGN_4 -.L62: +L(62): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -2082,10 +2082,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L62 + jl L(62) ALIGN_4 -.L66: +L(66): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2093,7 +2093,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L69 + je L(69) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -2101,7 +2101,7 @@ negq %rax ALIGN_4 -.L67: +L(67): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -2111,10 +2111,10 @@ movddup -13 * SIZE(BO, %rax, 2), %xmm3 addq $SIZE, %rax - jl .L67 + jl L(67) ALIGN_4 -.L69: +L(69): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -2255,13 +2255,13 @@ #endif ALIGN_4 -.L70: +L(70): movq M, I sarq $2, I # i = (m >> 2) - jle .L79 + jle L(79) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2316,10 +2316,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L56 + je L(56) ALIGN_4 -.L52: +L(52): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -2371,10 +2371,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L52 + jl L(52) ALIGN_4 -.L56: +L(56): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2382,7 +2382,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L59 + je L(59) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -2390,7 +2390,7 @@ negq %rax ALIGN_4 -.L57: +L(57): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -2405,10 +2405,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L57 + jl L(57) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2621,10 +2621,10 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2643,9 +2643,9 @@ #endif ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) #if defined(LT) || defined(RN) movq A, AO @@ -2678,7 +2678,7 @@ #endif testq $1, M - je .L100 + je L(100) #ifdef LN movq K, %rax @@ -2717,10 +2717,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L116 + je L(116) ALIGN_4 -.L112: +L(112): mulpd -16 * SIZE(BO, %rax, 1), %xmm0 addpd %xmm0, %xmm8 movapd -12 * SIZE(AO, %rax, 1), %xmm0 @@ -2731,10 +2731,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L112 + jl L(112) ALIGN_4 -.L116: +L(116): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2742,7 +2742,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L118 + je L(118) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -2750,16 +2750,16 @@ negq %rax ALIGN_4 -.L117: +L(117): mulsd -16 * SIZE(BO, %rax, 1), %xmm0 addsd %xmm0, %xmm8 movsd -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L117 + jl L(117) ALIGN_4 -.L118: +L(118): addpd %xmm9, %xmm8 haddpd %xmm8, %xmm8 @@ -2835,9 +2835,9 @@ #endif ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) #ifdef LN movq K, %rax @@ -2880,10 +2880,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L106 + je L(106) ALIGN_4 -.L102: +L(102): mulpd -16 * SIZE(AO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -12 * SIZE(BO, %rax, 1), %xmm0 @@ -2902,10 +2902,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L102 + jl L(102) ALIGN_4 -.L106: +L(106): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2913,7 +2913,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L109 + je L(109) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -2921,16 +2921,16 @@ negq %rax ALIGN_4 -.L107: +L(107): movddup -16 * SIZE(BO, %rax, 1), %xmm0 mulpd -16 * SIZE(AO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 addq $SIZE, %rax - jl .L107 + jl L(107) ALIGN_4 -.L109: +L(109): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -3047,13 +3047,13 @@ #endif ALIGN_4 -.L110: +L(110): movq M, I sarq $2, I # i = (m >> 2) - jle .L119 + jle L(119) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -3102,10 +3102,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L96 + je L(96) ALIGN_4 -.L92: +L(92): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -3133,10 +3133,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L92 + jl L(92) ALIGN_4 -.L96: +L(96): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3144,7 +3144,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L99 + je L(99) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -3152,7 +3152,7 @@ negq %rax ALIGN_4 -.L97: +L(97): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -3161,9 +3161,9 @@ movddup -15 * SIZE(BO, %rax, 1), %xmm1 addq $SIZE, %rax - jl .L97 + jl L(97) ALIGN_4 -.L99: +L(99): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -3340,10 +3340,10 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L119: +L(119): #ifdef LN leaq (B, K, SIZE), B #endif @@ -3361,7 +3361,7 @@ #endif ALIGN_4 -.L999: +L(999): movq (%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LN_4x4_core2.S b/kernel/x86_64/trsm_kernel_LN_4x4_core2.S index 3a16bfc3f3..8ce9402c31 100644 --- a/kernel/x86_64/trsm_kernel_LN_4x4_core2.S +++ b/kernel/x86_64/trsm_kernel_LN_4x4_core2.S @@ -171,9 +171,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) -.L01: +L(01): /* Copying to Sub Buffer */ #ifdef LN @@ -210,10 +210,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): prefetcht0 (PREFETCH_R + 0) * SIZE(B) movapd -16 * SIZE(B), %xmm0 movapd -14 * SIZE(B), %xmm1 @@ -269,10 +269,10 @@ subq $-16 * SIZE, B subq $-32 * SIZE, BO subq $1, %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -281,10 +281,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movapd -16 * SIZE(B), %xmm0 movapd -14 * SIZE(B), %xmm1 @@ -301,10 +301,10 @@ addq $4 * SIZE, B addq $8 * SIZE, BO subq $1, %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): leaq (PREFETCH_R + 0) * SIZE(B), BB #if defined(LT) || defined(RN) @@ -325,10 +325,10 @@ #endif testq $1, M - je .L20 + je L(20) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -362,10 +362,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -16 * SIZE(AO), %xmm0 @@ -435,10 +435,10 @@ subq $ -4 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -447,10 +447,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): movsd -16 * SIZE(AO), %xmm0 movsd -16 * SIZE(BO), %xmm2 movsd -14 * SIZE(BO), %xmm3 @@ -470,10 +470,10 @@ addq $1 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -649,12 +649,12 @@ #endif ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -688,10 +688,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -761,10 +761,10 @@ subq $ -8 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -773,10 +773,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -796,10 +796,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jne .L26 + jne L(26) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1042,13 +1042,13 @@ #endif ALIGN_4 -.L30: +L(30): movq M, I sarq $2, I # i = (m >> 2) - jle .L39 + jle L(39) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1111,10 +1111,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm2, %xmm10 @@ -1234,10 +1234,10 @@ mulpd %xmm1, %xmm5 BRANCH - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1246,10 +1246,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): movapd -16 * SIZE(AO), %xmm0 addpd %xmm2, %xmm10 movapd -16 * SIZE(BO), %xmm2 @@ -1282,10 +1282,10 @@ addq $8 * SIZE, BO # boffset1 += 8 subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): addpd %xmm2, %xmm10 addpd %xmm3, %xmm14 addpd %xmm4, %xmm11 @@ -1738,11 +1738,11 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1764,18 +1764,18 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $3, N - je .L999 + je L(999) testq $2, N - je .L80 + je L(80) ALIGN_4 -.L41: +L(41): /* Copying to Sub Buffer */ #ifdef LN @@ -1812,10 +1812,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): movddup -16 * SIZE(B), %xmm0 movddup -15 * SIZE(B), %xmm1 movddup -14 * SIZE(B), %xmm2 @@ -1837,10 +1837,10 @@ addq $8 * SIZE, B addq $16 * SIZE, BO subq $1, %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1849,10 +1849,10 @@ #endif andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movddup -16 * SIZE(B), %xmm0 movddup -15 * SIZE(B), %xmm1 @@ -1862,10 +1862,10 @@ addq $2 * SIZE, B addq $4 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): #if defined(LT) || defined(RN) movq A, AO #else @@ -1884,10 +1884,10 @@ #endif testq $1, M - je .L60 + je L(60) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1921,10 +1921,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -16 * SIZE(AO), %xmm0 @@ -1964,10 +1964,10 @@ subq $ -4 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1976,10 +1976,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): movsd -16 * SIZE(AO), %xmm0 movsd -16 * SIZE(BO), %xmm2 movsd -14 * SIZE(BO), %xmm3 @@ -1993,10 +1993,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addsd %xmm10, %xmm8 addsd %xmm11, %xmm9 @@ -2115,12 +2115,12 @@ #endif ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -2154,10 +2154,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -2197,10 +2197,10 @@ subq $ -8 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2209,10 +2209,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_4 -.L66: +L(66): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -2226,10 +2226,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L66 + jg L(66) ALIGN_4 -.L69: +L(69): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -2387,13 +2387,13 @@ #endif ALIGN_4 -.L70: +L(70): movq M, I sarq $2, I # i = (m >> 2) - jle .L79 + jle L(79) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2437,10 +2437,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -2518,10 +2518,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2530,10 +2530,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): movapd -16 * SIZE(AO), %xmm0 movapd -14 * SIZE(AO), %xmm1 @@ -2555,10 +2555,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2796,10 +2796,10 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2821,12 +2821,12 @@ #endif ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L81: +L(81): /* Copying to Sub Buffer */ #ifdef LN @@ -2863,10 +2863,10 @@ subq KK, %rax #endif sarq $3, %rax - jle .L83 + jle L(83) ALIGN_4 -.L82: +L(82): movddup -16 * SIZE(B), %xmm0 movddup -15 * SIZE(B), %xmm1 movddup -14 * SIZE(B), %xmm2 @@ -2888,10 +2888,10 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO subq $1, %rax - jne .L82 + jne L(82) ALIGN_4 -.L83: +L(83): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2900,10 +2900,10 @@ #endif andq $7, %rax BRANCH - jle .L90 + jle L(90) ALIGN_4 -.L84: +L(84): movddup -16 * SIZE(B), %xmm0 movapd %xmm0, 0 * SIZE(BO) @@ -2911,10 +2911,10 @@ addq $1 * SIZE, B addq $2 * SIZE, BO subq $1, %rax - jne .L84 + jne L(84) ALIGN_4 -.L90: +L(90): #if defined(LT) || defined(RN) movq A, AO #else @@ -2931,10 +2931,10 @@ #endif testq $1, M - je .L100 + je L(100) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -2968,10 +2968,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -16 * SIZE(AO), %xmm0 @@ -2997,10 +2997,10 @@ subq $-4 * SIZE, AO subq $-8 * SIZE, BO subq $1, %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3009,10 +3009,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): movsd -16 * SIZE(AO), %xmm0 movsd -16 * SIZE(BO), %xmm2 @@ -3022,10 +3022,10 @@ addq $1 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addsd %xmm10, %xmm8 addsd %xmm11, %xmm9 addsd %xmm9, %xmm8 @@ -3126,12 +3126,12 @@ #endif ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -3165,10 +3165,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L105 + je L(105) ALIGN_4 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -3196,10 +3196,10 @@ subq $-8 * SIZE, AO subq $-8 * SIZE, BO subq $1, %rax - jne .L102 + jne L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3208,10 +3208,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_4 -.L106: +L(106): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 @@ -3221,10 +3221,10 @@ addq $2 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L106 + jg L(106) ALIGN_4 -.L109: +L(109): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -3354,13 +3354,13 @@ #endif ALIGN_4 -.L110: +L(110): movq M, I sarq $2, I # i = (m >> 2) - jle .L119 + jle L(119) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -3399,10 +3399,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -3448,10 +3448,10 @@ subq $-16 * SIZE, AO subq $ -8 * SIZE, BO subq $1, %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3460,10 +3460,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_4 -.L96: +L(96): movapd -16 * SIZE(AO), %xmm0 movapd -14 * SIZE(AO), %xmm1 @@ -3478,10 +3478,10 @@ addq $4 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L96 + jg L(96) ALIGN_4 -.L99: +L(99): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -3682,10 +3682,10 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L119: +L(119): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -3708,7 +3708,7 @@ ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/trsm_kernel_LN_4x4_penryn.S b/kernel/x86_64/trsm_kernel_LN_4x4_penryn.S index 3bc7ae1f85..e222635b78 100644 --- a/kernel/x86_64/trsm_kernel_LN_4x4_penryn.S +++ b/kernel/x86_64/trsm_kernel_LN_4x4_penryn.S @@ -166,10 +166,10 @@ movq N, J sarq $2, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -208,7 +208,7 @@ testq $1, M BRANCH - jle .L20 + jle L(20) ALIGN_4 #ifdef LN @@ -244,10 +244,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) shufps $0x44, %xmm0, %xmm0 @@ -294,10 +294,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -306,10 +306,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): shufps $0x44, %xmm0, %xmm0 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm3 @@ -325,10 +325,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -483,10 +483,10 @@ #endif ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) ALIGN_4 #ifdef LN @@ -523,10 +523,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_4 -.L22: +L(22): addpd %xmm3, %xmm11 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -592,10 +592,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -604,10 +604,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): addpd %xmm3, %xmm11 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -628,10 +628,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -859,14 +859,14 @@ #endif ALIGN_4 -.L30: +L(30): movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L39 + jle L(39) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -933,10 +933,10 @@ #endif sarq $3, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): addpd %xmm3, %xmm11 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -14 * SIZE(BO), %xmm3 @@ -1186,10 +1186,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1198,10 +1198,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addpd %xmm3, %xmm11 movaps -14 * SIZE(BO), %xmm3 addpd %xmm4, %xmm15 @@ -1237,10 +1237,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax subq $4, %rax @@ -1665,10 +1665,10 @@ decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1687,13 +1687,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $2, N BRANCH - jle .L80 + jle L(80) #if defined(LT) || defined(RN) movq A, AO @@ -1733,7 +1733,7 @@ testq $1, M BRANCH - jle .L60 + jle L(60) ALIGN_4 #ifdef LN @@ -1766,10 +1766,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_4 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) shufps $0x44, %xmm0, %xmm0 @@ -1800,10 +1800,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1812,10 +1812,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): shufps $0x44, %xmm0, %xmm0 mulpd %xmm0, %xmm2 movsd -15 * SIZE(AO), %xmm0 @@ -1827,10 +1827,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1937,10 +1937,10 @@ #endif ALIGN_4 -.L60: +L(60): testq $2, M BRANCH - jle .L70 + jle L(70) ALIGN_4 #ifdef LN @@ -1974,10 +1974,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x4e, %xmm2, %xmm7 @@ -2020,10 +2020,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2032,10 +2032,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm7 @@ -2050,10 +2050,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2198,14 +2198,14 @@ #endif ALIGN_4 -.L70: +L(70): movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L79 + jle L(79) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2254,10 +2254,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_4 -.L52: +L(52): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -2332,10 +2332,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2344,10 +2344,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -2370,10 +2370,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2595,10 +2595,10 @@ decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2616,10 +2616,10 @@ #endif ALIGN_4 -.L80: +L(80): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(LT) || defined(RN) movq A, AO @@ -2653,7 +2653,7 @@ testq $1, M BRANCH - jle .L90 + jle L(90) #ifdef LN movq K, %rax @@ -2685,10 +2685,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_4 -.L112: +L(112): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulsd %xmm0, %xmm2 @@ -2715,10 +2715,10 @@ subq $-4 * SIZE, BO subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2727,10 +2727,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd %xmm0, %xmm2 addsd %xmm2, %xmm8 movsd -15 * SIZE(AO), %xmm0 @@ -2741,10 +2741,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -2830,10 +2830,10 @@ #endif ALIGN_4 -.L90: +L(90): testq $2, M BRANCH - jle .L110 + jle L(110) ALIGN_4 #ifdef LN @@ -2866,10 +2866,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_4 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm2, %xmm3 @@ -2904,10 +2904,10 @@ subq $-4 * SIZE, BO subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2916,10 +2916,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L108 + je L(108) ALIGN_4 -.L106: +L(106): pshufd $0x44, %xmm2, %xmm3 movsd -15 * SIZE(BO), %xmm2 @@ -2932,10 +2932,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_4 -.L108: +L(108): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3051,14 +3051,14 @@ #endif ALIGN_4 -.L110: +L(110): movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L119 + jle L(119) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -3097,10 +3097,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_4 -.L92: +L(92): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm2, %xmm3 @@ -3157,10 +3157,10 @@ subq $ -4 * SIZE, BO subq $1, %rax BRANCH - jg .L92 + jg L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3169,10 +3169,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): pshufd $0x44, %xmm2, %xmm3 pshufd $0x44, %xmm2, %xmm4 movsd -15 * SIZE(BO), %xmm2 @@ -3190,10 +3190,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3376,10 +3376,10 @@ decq I BRANCH - jg .L91 + jg L(91) ALIGN_4 -.L119: +L(119): #ifdef LN leaq (B, K, SIZE), B #endif @@ -3396,7 +3396,7 @@ #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LN_4x4_sse2.S b/kernel/x86_64/trsm_kernel_LN_4x4_sse2.S index c846080a40..17fa5bf912 100644 --- a/kernel/x86_64/trsm_kernel_LN_4x4_sse2.S +++ b/kernel/x86_64/trsm_kernel_LN_4x4_sse2.S @@ -294,9 +294,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) -.L01: +L(01): /* Copying to Sub Buffer */ #ifdef LN @@ -333,12 +333,12 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): PREFETCHNTA 40 * SIZE(B) movsd 0 * SIZE(B), %xmm0 @@ -371,10 +371,10 @@ movsd %xmm7, -1 * SIZE(BO) decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -383,10 +383,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movsd 0 * SIZE(B), %xmm0 movsd 1 * SIZE(B), %xmm1 movsd 2 * SIZE(B), %xmm2 @@ -404,10 +404,10 @@ addq $4 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -426,10 +426,10 @@ #endif testq $1, M - je .L20 + je L(20) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -470,10 +470,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulsd %xmm8, %xmm9 addsd %xmm9, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -582,10 +582,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -594,10 +594,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulsd %xmm8, %xmm9 addsd %xmm9, %xmm0 movsd 2 * SIZE(BO), %xmm9 @@ -614,10 +614,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -793,12 +793,12 @@ #endif ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -839,10 +839,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -952,10 +952,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -964,10 +964,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -984,10 +984,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1258,13 +1258,13 @@ ALIGN_4 -.L30: +L(30): movq M, I sarq $2, I # i = (m >> 2) - jle .L39 + jle L(39) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1328,8 +1328,8 @@ #endif andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -1347,7 +1347,7 @@ KERNEL7(16 * 1) KERNEL8(16 * 1) cmpq $64 * 2, %rax - jle .L12 + jle L(12) KERNEL1(16 * 2) KERNEL2(16 * 2) KERNEL3(16 * 2) @@ -1365,7 +1365,7 @@ KERNEL7(16 * 3) KERNEL8(16 * 3) cmpq $64 * 4, %rax - jle .L12 + jle L(12) KERNEL1(16 * 4) KERNEL2(16 * 4) KERNEL3(16 * 4) @@ -1383,7 +1383,7 @@ KERNEL7(16 * 5) KERNEL8(16 * 5) cmpq $64 * 6, %rax - jle .L12 + jle L(12) KERNEL1(16 * 6) KERNEL2(16 * 6) KERNEL3(16 * 6) @@ -1404,14 +1404,14 @@ addq $16 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $64 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 4), BO # * 64 ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1420,10 +1420,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -1452,10 +1452,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1967,10 +1967,10 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1992,18 +1992,18 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $3, N - je .L999 + je L(999) testq $2, N - je .L80 + je L(80) ALIGN_4 -.L41: +L(41): /* Copying to Sub Buffer */ #ifdef LN @@ -2040,10 +2040,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): PREFETCH 56 * SIZE(B) movsd 0 * SIZE(B), %xmm0 @@ -2076,10 +2076,10 @@ movsd %xmm7, -1 * SIZE(BO) decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2088,10 +2088,10 @@ #endif andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movsd 0 * SIZE(B), %xmm0 movsd 1 * SIZE(B), %xmm1 @@ -2103,10 +2103,10 @@ addq $2 * SIZE, B addq $4 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): #if defined(LT) || defined(RN) movq A, AO #else @@ -2125,10 +2125,10 @@ #endif testq $1, M - je .L60 + je L(60) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -2169,10 +2169,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulsd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulsd 2 * SIZE(BO), %xmm8 @@ -2233,10 +2233,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2245,10 +2245,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulsd %xmm8, %xmm9 mulsd 2 * SIZE(BO), %xmm8 addsd %xmm9, %xmm0 @@ -2259,10 +2259,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addsd %xmm2, %xmm0 addsd %xmm3, %xmm1 @@ -2381,12 +2381,12 @@ #endif ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -2427,10 +2427,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(BO), %xmm8 @@ -2492,10 +2492,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2504,10 +2504,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_4 -.L66: +L(66): mulpd %xmm8, %xmm9 mulpd 2 * SIZE(BO), %xmm8 addpd %xmm9, %xmm0 @@ -2518,10 +2518,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L69: +L(69): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -2690,13 +2690,13 @@ #endif ALIGN_4 -.L70: +L(70): movq M, I sarq $2, I # i = (m >> 2) - jle .L79 + jle L(79) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2747,10 +2747,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(BO), %xmm8 @@ -2862,10 +2862,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2874,10 +2874,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 @@ -2894,10 +2894,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3162,10 +3162,10 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -3187,12 +3187,12 @@ #endif ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L81: +L(81): /* Copying to Sub Buffer */ #ifdef LN @@ -3229,10 +3229,10 @@ subq KK, %rax #endif sarq $3, %rax - jle .L83 + jle L(83) ALIGN_4 -.L82: +L(82): PREFETCH 56 * SIZE(B) movsd 0 * SIZE(B), %xmm0 @@ -3265,10 +3265,10 @@ movsd %xmm7, -1 * SIZE(BO) decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L83: +L(83): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3277,10 +3277,10 @@ #endif andq $7, %rax BRANCH - jle .L90 + jle L(90) ALIGN_4 -.L84: +L(84): movsd 0 * SIZE(B), %xmm0 movsd %xmm0, 0 * SIZE(BO) @@ -3289,10 +3289,10 @@ addq $1 * SIZE, B addq $2 * SIZE, BO decq %rax - jne .L84 + jne L(84) ALIGN_4 -.L90: +L(90): #if defined(LT) || defined(RN) movq A, AO #else @@ -3309,10 +3309,10 @@ #endif testq $1, M - je .L100 + je L(100) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -3350,10 +3350,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulsd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd 1 * SIZE(AO), %xmm8 @@ -3385,10 +3385,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3397,10 +3397,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd %xmm8, %xmm9 movsd 1 * SIZE(AO), %xmm8 addsd %xmm9, %xmm0 @@ -3409,10 +3409,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addsd %xmm2, %xmm0 addsd %xmm3, %xmm1 addsd %xmm1, %xmm0 @@ -3513,12 +3513,12 @@ #endif ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -3556,10 +3556,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L105 + je L(105) ALIGN_4 -.L102: +L(102): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd 2 * SIZE(AO), %xmm8 @@ -3592,10 +3592,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3604,10 +3604,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_4 -.L106: +L(106): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(AO), %xmm8 @@ -3616,10 +3616,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L106 + jg L(106) ALIGN_4 -.L109: +L(109): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -3750,13 +3750,13 @@ #endif ALIGN_4 -.L110: +L(110): movq M, I sarq $2, I # i = (m >> 2) - jle .L119 + jle L(119) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -3803,10 +3803,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(AO), %xmm9 @@ -3863,10 +3863,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3875,10 +3875,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_4 -.L96: +L(96): mulpd %xmm9, %xmm8 mulpd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -3889,10 +3889,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L99: +L(99): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -4094,10 +4094,10 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L119: +L(119): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -4119,7 +4119,7 @@ #endif ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/trsm_kernel_LN_4x4_sse3.S b/kernel/x86_64/trsm_kernel_LN_4x4_sse3.S index fedeb5a580..aeb2b482c5 100644 --- a/kernel/x86_64/trsm_kernel_LN_4x4_sse3.S +++ b/kernel/x86_64/trsm_kernel_LN_4x4_sse3.S @@ -400,10 +400,10 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -441,10 +441,10 @@ #endif testq $1, M - je .L30 + je L(30) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -478,10 +478,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -543,10 +543,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -555,10 +555,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -570,10 +570,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax @@ -758,13 +758,13 @@ #endif ALIGN_4 -.L30: +L(30): testq $2, M BRANCH - je .L20 + je L(20) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax @@ -798,10 +798,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -911,10 +911,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -923,10 +923,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -944,10 +944,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1161,13 +1161,13 @@ #endif ALIGN_4 -.L20: +L(20): movq M, I sarq $2, I # i = (m >> 2) - jle .L39 + jle L(39) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax @@ -1232,8 +1232,8 @@ #if 1 andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1 (16 * 0) KERNEL2 (16 * 0) KERNEL3 (16 * 0) @@ -1252,7 +1252,7 @@ KERNEL16(16 * 0) cmpq $128 * 1, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 1) KERNEL2 (16 * 1) KERNEL3 (16 * 1) @@ -1271,7 +1271,7 @@ KERNEL16(16 * 1) cmpq $128 * 2, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 2) KERNEL2 (16 * 2) KERNEL3 (16 * 2) @@ -1290,7 +1290,7 @@ KERNEL16(16 * 2) cmpq $128 * 3, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 3) KERNEL2 (16 * 3) KERNEL3 (16 * 3) @@ -1309,7 +1309,7 @@ KERNEL16(16 * 3) cmpq $128 * 4, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 4) KERNEL2 (16 * 4) KERNEL3 (16 * 4) @@ -1328,7 +1328,7 @@ KERNEL16(16 * 4) cmpq $128 * 5, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 5) KERNEL2 (16 * 5) KERNEL3 (16 * 5) @@ -1347,7 +1347,7 @@ KERNEL16(16 * 5) cmpq $128 * 6, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 6) KERNEL2 (16 * 6) KERNEL3 (16 * 6) @@ -1366,7 +1366,7 @@ KERNEL16(16 * 6) cmpq $128 * 7, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 7) KERNEL2 (16 * 7) KERNEL3 (16 * 7) @@ -1387,18 +1387,18 @@ addq $32 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $128 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 2), BO # * 64 #else sarq $3, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1625,11 +1625,11 @@ addq $32 * SIZE, AO decq %rax BRANCH - jne .L12 + jne L(12) #endif ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1638,10 +1638,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 addpd %xmm9, %xmm0 @@ -1672,10 +1672,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax @@ -2081,11 +2081,11 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -2103,12 +2103,12 @@ #endif decq J # j -- - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $2, N - je .L80 + je L(80) ALIGN_4 #if defined(LT) || defined(RN) @@ -2149,10 +2149,10 @@ testq $1, M - je .L70 + je L(70) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -2185,10 +2185,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movddup 1 * SIZE(AO), %xmm8 @@ -2220,10 +2220,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2232,10 +2232,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulpd %xmm8, %xmm9 movddup 1 * SIZE(AO), %xmm8 addpd %xmm9, %xmm0 @@ -2244,10 +2244,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -2364,12 +2364,12 @@ #endif ALIGN_4 -.L70: +L(70): testq $2, M - je .L60 + je L(60) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -2402,10 +2402,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -2467,10 +2467,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2479,10 +2479,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_4 -.L66: +L(66): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -2494,10 +2494,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L69: +L(69): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -2633,13 +2633,13 @@ #endif ALIGN_4 -.L60: +L(60): movq M, I sarq $2, I # i = (m >> 2) - jle .L79 + jle L(79) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2683,10 +2683,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -2805,10 +2805,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2817,10 +2817,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 addpd %xmm9, %xmm0 @@ -2839,10 +2839,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3059,11 +3059,11 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -3082,9 +3082,9 @@ #endif ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 #if defined(LT) || defined(RN) @@ -3118,10 +3118,10 @@ #endif testq $1, M - je .L110 + je L(110) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -3154,10 +3154,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulpd %xmm9, %xmm8 movapd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -3176,10 +3176,10 @@ addq $8 * SIZE, AO addq $8 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3188,10 +3188,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd 0 * SIZE(BO), %xmm9 addsd %xmm9, %xmm0 movsd 1 * SIZE(AO), %xmm9 @@ -3199,10 +3199,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addpd %xmm1, %xmm0 haddpd %xmm0, %xmm0 @@ -3291,12 +3291,12 @@ #endif ALIGN_4 -.L110: +L(110): testq $2, M - je .L100 + je L(100) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -3329,10 +3329,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L105 + je L(105) ALIGN_4 -.L102: +L(102): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movddup 1 * SIZE(BO), %xmm9 @@ -3364,10 +3364,10 @@ addq $16 * SIZE, AO addq $ 8 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3376,10 +3376,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_4 -.L106: +L(106): mulpd %xmm9, %xmm8 movddup 1 * SIZE(BO), %xmm9 addpd %xmm8, %xmm0 @@ -3388,10 +3388,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L106 + jg L(106) ALIGN_4 -.L109: +L(109): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -3507,13 +3507,13 @@ #endif ALIGN_4 -.L100: +L(100): movq M, I sarq $2, I # i = (m >> 2) - jle .L119 + jle L(119) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -3552,10 +3552,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(AO), %xmm9 @@ -3610,10 +3610,10 @@ addq $32 * SIZE, AO addq $8 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3622,10 +3622,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_4 -.L96: +L(96): mulpd %xmm9, %xmm8 mulpd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -3636,10 +3636,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L99: +L(99): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -3822,10 +3822,10 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L119: +L(119): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -3844,7 +3844,7 @@ #endif ALIGN_2 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LN_4x8_nehalem.S b/kernel/x86_64/trsm_kernel_LN_4x8_nehalem.S index 8717fc3368..4f55ba0165 100644 --- a/kernel/x86_64/trsm_kernel_LN_4x8_nehalem.S +++ b/kernel/x86_64/trsm_kernel_LN_4x8_nehalem.S @@ -166,10 +166,10 @@ movq N, J sarq $3, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -208,7 +208,7 @@ testq $1, M BRANCH - jle .L20 + jle L(20) #ifdef LN movq K, %rax @@ -240,10 +240,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movaps -32 * SIZE(BO), %xmm2 @@ -283,10 +283,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -295,10 +295,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -313,10 +313,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -715,10 +715,10 @@ #endif ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) #ifdef LN movq K, %rax @@ -756,10 +756,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -837,10 +837,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -849,10 +849,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -876,10 +876,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1329,14 +1329,14 @@ #endif ALIGN_4 -.L30: +L(30): movq M, I sarq $2, I NOBRANCH - jle .L39 + jle L(39) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1390,10 +1390,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm12 @@ -1520,10 +1520,10 @@ subq $-16 * SIZE, AO subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1532,10 +1532,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addps %xmm1, %xmm12 movaps -32 * SIZE(BO), %xmm1 addps %xmm2, %xmm13 @@ -1571,10 +1571,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2194,10 +2194,10 @@ decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 8), B @@ -2216,12 +2216,12 @@ subq $1, J BRANCH - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $4, N - jle .L70 + jle L(70) #if defined(LT) || defined(RN) movq A, AO @@ -2257,7 +2257,7 @@ testq $1, M BRANCH - jle .L50 + jle L(50) #ifdef LN movq K, %rax @@ -2288,10 +2288,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movaps -32 * SIZE(BO), %xmm2 @@ -2319,11 +2319,11 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) addps %xmm9, %xmm8 ALIGN_3 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2332,10 +2332,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -2347,10 +2347,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2524,10 +2524,10 @@ #endif ALIGN_4 -.L50: +L(50): testq $2, M BRANCH - jle .L60 + jle L(60) #ifdef LN movq K, %rax @@ -2560,10 +2560,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2605,10 +2605,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2617,10 +2617,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2635,10 +2635,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2842,14 +2842,14 @@ #endif ALIGN_4 -.L60: +L(60): movq M, I sarq $2, I NOBRANCH - jle .L69 + jle L(69) ALIGN_4 -.L41: +L(41): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2889,10 +2889,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -2959,10 +2959,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2971,10 +2971,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): addps %xmm1, %xmm8 movaps -32 * SIZE(BO), %xmm1 addps %xmm2, %xmm9 @@ -2995,10 +2995,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3300,10 +3300,10 @@ decq I BRANCH - jg .L41 + jg L(41) ALIGN_4 -.L69: +L(69): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -3321,9 +3321,9 @@ #endif ALIGN_4 -.L70: +L(70): testq $2, N - jle .L100 + jle L(100) #if defined(LT) || defined(RN) movq A, AO @@ -3359,7 +3359,7 @@ testq $1, M BRANCH - jle .L80 + jle L(80) #ifdef LN movq K, %rax @@ -3390,10 +3390,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_3 -.L92: +L(92): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movsd -32 * SIZE(BO), %xmm2 @@ -3421,11 +3421,11 @@ subq $1, %rax BRANCH - jg .L92 + jg L(92) addps %xmm9, %xmm8 ALIGN_3 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3434,10 +3434,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_3 -.L96: +L(96): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -3449,10 +3449,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_3 -.L98: +L(98): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3558,10 +3558,10 @@ #endif ALIGN_4 -.L80: +L(80): testq $2, M BRANCH - jle .L90 + jle L(90) #ifdef LN movq K, %rax @@ -3594,10 +3594,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): addps %xmm1, %xmm8 movsd -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -3627,10 +3627,10 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) ALIGN_3 -.L85: +L(85): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3639,10 +3639,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): addps %xmm1, %xmm8 movsd -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -3654,10 +3654,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_3 -.L88: +L(88): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3784,14 +3784,14 @@ #endif ALIGN_4 -.L90: +L(90): movq M, I sarq $2, I NOBRANCH - jle .L99 + jle L(99) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -3826,10 +3826,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -3872,10 +3872,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_3 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3884,10 +3884,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addps %xmm1, %xmm8 pshufd $0x00, %xmm3, %xmm1 mulps %xmm0, %xmm1 @@ -3902,10 +3902,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_3 -.L78: +L(78): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -4109,10 +4109,10 @@ decq I BRANCH - jg .L71 + jg L(71) ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -4130,9 +4130,9 @@ #endif ALIGN_4 -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) #if defined(LT) || defined(RN) movq A, AO @@ -4166,7 +4166,7 @@ testq $1, M BRANCH - jle .L110 + jle L(110) #ifdef LN movq K, %rax @@ -4196,10 +4196,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L125 + jle L(125) ALIGN_3 -.L122: +L(122): addss %xmm2, %xmm8 movss -32 * SIZE(BO), %xmm2 mulss %xmm0, %xmm2 @@ -4225,10 +4225,10 @@ subq $1, %rax BRANCH - jg .L122 + jg L(122) ALIGN_3 -.L125: +L(125): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4237,10 +4237,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_3 -.L126: +L(126): addss %xmm2, %xmm8 movss -32 * SIZE(BO), %xmm2 mulss %xmm0, %xmm2 @@ -4251,10 +4251,10 @@ subq $1, %rax BRANCH - jg .L126 + jg L(126) ALIGN_3 -.L128: +L(128): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -4327,10 +4327,10 @@ #endif ALIGN_4 -.L110: +L(110): testq $2, M BRANCH - jle .L120 + jle L(120) #ifdef LN movq K, %rax @@ -4360,10 +4360,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_3 -.L112: +L(112): addps %xmm1, %xmm8 movss -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -4393,10 +4393,10 @@ subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_3 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4405,10 +4405,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_3 -.L116: +L(116): addps %xmm1, %xmm8 movss -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -4420,10 +4420,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_3 -.L118: +L(118): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -4527,14 +4527,14 @@ #endif ALIGN_4 -.L120: +L(120): movq M, I sarq $2, I NOBRANCH - jle .L129 + jle L(129) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -4565,10 +4565,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_3 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -4599,10 +4599,10 @@ subq $ -4 * SIZE, BO subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_3 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4611,10 +4611,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L108 + je L(108) ALIGN_3 -.L106: +L(106): addps %xmm1, %xmm8 pshufd $0x00, %xmm3, %xmm1 movss -31 * SIZE(BO), %xmm3 @@ -4626,10 +4626,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_3 -.L108: +L(108): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -4797,10 +4797,10 @@ decq I BRANCH - jg .L101 + jg L(101) ALIGN_4 -.L129: +L(129): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -4818,7 +4818,7 @@ #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LN_8x4_sse.S b/kernel/x86_64/trsm_kernel_LN_8x4_sse.S index a318747632..aeae331398 100644 --- a/kernel/x86_64/trsm_kernel_LN_8x4_sse.S +++ b/kernel/x86_64/trsm_kernel_LN_8x4_sse.S @@ -189,9 +189,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L50 + jle L(50) -.L01: +L(01): /* Copying to Sub Buffer */ #ifdef LN @@ -228,10 +228,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 movaps 8 * SIZE(B), %xmm11 @@ -279,10 +279,10 @@ addq $64 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -291,10 +291,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movaps 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -310,10 +310,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -332,7 +332,7 @@ #endif testq $1, M - je .L20 + je L(20) #ifdef LN movq K, %rax @@ -374,10 +374,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): mulss %xmm8, %xmm9 addss %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -496,10 +496,10 @@ addq $ 8 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -508,10 +508,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -529,10 +529,10 @@ addq $ 1 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -733,9 +733,9 @@ #endif ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) #ifdef LN movq K, %rax @@ -778,10 +778,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -900,10 +900,10 @@ addq $ 16 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -912,10 +912,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -933,10 +933,10 @@ addq $ 2 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1181,9 +1181,9 @@ #endif ALIGN_4 -.L30: +L(30): testq $4, M - je .L40 + je L(40) #ifdef LN movq K, %rax @@ -1226,10 +1226,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -1343,10 +1343,10 @@ addq $ 32 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1355,10 +1355,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1375,10 +1375,10 @@ addq $ 4 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1702,13 +1702,13 @@ #endif ALIGN_4 -.L40: +L(40): movq M, I sarq $3, I # i = (m >> 3) - jle .L49 + jle L(49) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $3 + BASE_SHIFT, %rax @@ -1760,10 +1760,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 0 * SIZE(BO), %xmm9 @@ -1864,10 +1864,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1876,9 +1876,9 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1907,10 +1907,10 @@ addq $8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2573,10 +2573,10 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L49: +L(49): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -2598,11 +2598,11 @@ #endif decq J # j -- - jg .L01 + jg L(01) -.L50: +L(50): testq $2, N - je .L100 + je L(100) #ifdef LN movq OFFSET, %rax @@ -2638,10 +2638,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 @@ -2668,10 +2668,10 @@ addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2680,10 +2680,10 @@ #endif andq $3, %rax BRANCH - jle .L60 + jle L(60) ALIGN_4 -.L54: +L(54): movsd 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -2695,10 +2695,10 @@ addq $2 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L54 + jne L(54) ALIGN_4 -.L60: +L(60): #if defined(LT) || defined(RN) movq A, AO #else @@ -2716,7 +2716,7 @@ #endif testq $1, M - je .L70 + je L(70) #ifdef LN movq K, %rax @@ -2758,10 +2758,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -2832,10 +2832,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2844,10 +2844,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -2859,10 +2859,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): addss %xmm2, %xmm0 addss %xmm3, %xmm1 @@ -2989,9 +2989,9 @@ #endif ALIGN_4 -.L70: +L(70): testq $2, M - je .L80 + je L(80) #ifdef LN movq K, %rax @@ -3034,10 +3034,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L85 + je L(85) ALIGN_4 -.L82: +L(82): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -3108,10 +3108,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L85: +L(85): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3120,10 +3120,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -3135,10 +3135,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -3308,9 +3308,9 @@ #endif ALIGN_4 -.L80: +L(80): testq $4, M - je .L90 + je L(90) #ifdef LN movq K, %rax @@ -3353,10 +3353,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3420,10 +3420,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3432,10 +3432,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -3446,10 +3446,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -3697,13 +3697,13 @@ #endif ALIGN_4 -.L90: +L(90): movq M, I sarq $3, I # i = (m >> 3) - jle .L99 + jle L(99) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $3 + BASE_SHIFT, %rax @@ -3750,10 +3750,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3873,10 +3873,10 @@ addq $64 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3885,10 +3885,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -3905,10 +3905,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -4426,10 +4426,10 @@ #endif decq I # i -- - jg .L61 + jg L(61) ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -4451,9 +4451,9 @@ #endif ALIGN_4 -.L100: +L(100): testq $1, N - je .L999 + je L(999) #ifdef LN movq OFFSET, %rax @@ -4489,10 +4489,10 @@ subq KK, %rax #endif sarq $3, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movsd 0 * SIZE(B), %xmm3 movhps 2 * SIZE(B), %xmm3 movsd 4 * SIZE(B), %xmm7 @@ -4521,10 +4521,10 @@ addq $32 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4533,10 +4533,10 @@ #endif andq $7, %rax BRANCH - jle .L110 + jle L(110) ALIGN_4 -.L104: +L(104): movss 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -4546,10 +4546,10 @@ addq $ 1 * SIZE, B addq $ 4 * SIZE, BO decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L110: +L(110): #if defined(LT) || defined(RN) movq A, AO #else @@ -4566,7 +4566,7 @@ #endif testq $1, M - je .L120 + je L(120) #ifdef LN movq K, %rax @@ -4606,10 +4606,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L145 + je L(145) ALIGN_4 -.L142: +L(142): mulss %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -4643,10 +4643,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L142 + jne L(142) ALIGN_4 -.L145: +L(145): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4655,10 +4655,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): mulss %xmm8, %xmm9 movss 1 * SIZE(AO), %xmm8 addss %xmm9, %xmm0 @@ -4667,10 +4667,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): addss %xmm1, %xmm0 addss %xmm3, %xmm2 addss %xmm2, %xmm0 @@ -4755,9 +4755,9 @@ #endif ALIGN_4 -.L120: +L(120): testq $2, M - je .L130 + je L(130) #ifdef LN movq K, %rax @@ -4798,10 +4798,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L135 + je L(135) ALIGN_4 -.L132: +L(132): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -4847,10 +4847,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L132 + jne L(132) ALIGN_4 -.L135: +L(135): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4859,10 +4859,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): mulps %xmm8, %xmm9 movsd 2 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -4871,10 +4871,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): addps %xmm1, %xmm0 #if defined(LN) || defined(RT) @@ -5003,9 +5003,9 @@ #endif ALIGN_4 -.L130: +L(130): testq $4, M - je .L140 + je L(140) #ifdef LN movq K, %rax @@ -5046,10 +5046,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L125 + je L(125) ALIGN_4 -.L122: +L(122): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -5087,10 +5087,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L122 + jne L(122) ALIGN_4 -.L125: +L(125): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -5099,10 +5099,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): mulps %xmm8, %xmm9 movaps 4 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -5111,10 +5111,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): addps %xmm1, %xmm0 addps %xmm3, %xmm2 addps %xmm2, %xmm0 @@ -5313,13 +5313,13 @@ #endif ALIGN_4 -.L140: +L(140): movq M, I sarq $3, I # i = (m >> 3) - jle .L149 + jle L(149) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $3 + BASE_SHIFT, %rax @@ -5365,10 +5365,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulps %xmm9, %xmm8 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -5441,10 +5441,10 @@ addq $64 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -5453,10 +5453,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulps %xmm9, %xmm8 mulps 4 * SIZE(AO), %xmm9 addps %xmm8, %xmm0 @@ -5467,10 +5467,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -5894,10 +5894,10 @@ #endif decq I # i -- - jg .L111 + jg L(111) ALIGN_4 -.L149: +L(149): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -5919,7 +5919,7 @@ #endif ALIGN_4 -.L999: +L(999): movq %rbx, %rsp EMMS movq 0(%rsp), %rbx diff --git a/kernel/x86_64/trsm_kernel_LT_2x8_nehalem.S b/kernel/x86_64/trsm_kernel_LT_2x8_nehalem.S index 39ed586ce4..52987fb0f1 100644 --- a/kernel/x86_64/trsm_kernel_LT_2x8_nehalem.S +++ b/kernel/x86_64/trsm_kernel_LT_2x8_nehalem.S @@ -167,10 +167,10 @@ movq N, J sarq $3, J NOBRANCH - jle .L30 + jle L(30) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -210,10 +210,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -267,10 +267,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm12 @@ -393,10 +393,10 @@ subq $-8 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -405,10 +405,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addpd %xmm1, %xmm12 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm13 @@ -444,10 +444,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -897,13 +897,13 @@ decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M BRANCH - jle .L29 + jle L(29) ALIGN_4 #ifdef LN @@ -937,10 +937,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1002,10 +1002,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1014,10 +1014,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1037,10 +1037,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1378,7 +1378,7 @@ #endif ALIGN_4 -.L29: +L(29): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 8), B @@ -1397,12 +1397,12 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L30: +L(30): testq $4, N - jle .L50 + jle L(50) ALIGN_4 #if defined(LT) || defined(RN) @@ -1440,10 +1440,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -1483,10 +1483,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -1558,10 +1558,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1570,10 +1570,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): addpd %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm9 @@ -1595,10 +1595,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1826,13 +1826,13 @@ decq I BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L40: +L(40): testq $1, M BRANCH - jle .L49 + jle L(49) ALIGN_4 #ifdef LN @@ -1866,10 +1866,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1907,10 +1907,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1919,10 +1919,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1936,10 +1936,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2097,7 +2097,7 @@ #endif ALIGN_4 -.L49: +L(49): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -2115,9 +2115,9 @@ #endif ALIGN_4 -.L50: +L(50): testq $2, N - jle .L70 + jle L(70) ALIGN_4 #if defined(LT) || defined(RN) @@ -2155,10 +2155,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -2194,10 +2194,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -2237,13 +2237,13 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 ALIGN_3 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2252,10 +2252,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addpd %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm9 @@ -2269,10 +2269,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2407,13 +2407,13 @@ decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L69 + jle L(69) ALIGN_4 #ifdef LN @@ -2445,10 +2445,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): mulpd %xmm0, %xmm1 movddup -15 * SIZE(AO), %xmm0 addpd %xmm1, %xmm8 @@ -2474,10 +2474,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2486,10 +2486,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): mulpd %xmm0, %xmm1 movddup -15 * SIZE(AO), %xmm0 addpd %xmm1, %xmm8 @@ -2500,10 +2500,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2606,7 +2606,7 @@ #endif ALIGN_4 -.L69: +L(69): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2624,9 +2624,9 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, N - jle .L999 + jle L(999) ALIGN_4 #if defined(LT) || defined(RN) @@ -2662,10 +2662,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L80 + jle L(80) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -2700,10 +2700,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -2731,12 +2731,12 @@ subq $1, %rax BRANCH - jg .L72 + jg L(72) addpd %xmm9, %xmm8 ALIGN_3 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2745,10 +2745,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addpd %xmm1, %xmm8 movddup -16 * SIZE(BO), %xmm1 mulpd %xmm0, %xmm1 @@ -2759,10 +2759,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2863,13 +2863,13 @@ decq I BRANCH - jg .L71 + jg L(71) ALIGN_4 -.L80: +L(80): testq $1, M BRANCH - jle .L89 + jle L(89) ALIGN_4 #ifdef LN @@ -2903,10 +2903,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): mulpd %xmm0, %xmm1 movsd -14 * SIZE(AO), %xmm0 movhps -13 * SIZE(AO), %xmm0 @@ -2926,12 +2926,12 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) addpd %xmm9, %xmm8 ALIGN_3 -.L85: +L(85): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2940,10 +2940,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): mulsd %xmm0, %xmm1 movsd -15 * SIZE(AO), %xmm0 addsd %xmm1, %xmm8 @@ -2954,10 +2954,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -3029,7 +3029,7 @@ #endif ALIGN_4 -.L89: +L(89): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -3048,7 +3048,7 @@ ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LT_4x2_atom.S b/kernel/x86_64/trsm_kernel_LT_4x2_atom.S index 04b7e2de19..8d63d73568 100644 --- a/kernel/x86_64/trsm_kernel_LT_4x2_atom.S +++ b/kernel/x86_64/trsm_kernel_LT_4x2_atom.S @@ -156,10 +156,10 @@ movq N, J sarq $1, J - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -198,10 +198,10 @@ movq M, I sarq $2, I - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -250,10 +250,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): addsd %xmm2, %xmm13 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps %xmm0, %xmm2 @@ -396,10 +396,10 @@ mulsd %xmm3, %xmm6 movsd 1 * SIZE(BO), %xmm3 - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -408,10 +408,10 @@ #endif andq $3, %rax BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): addsd %xmm2, %xmm13 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -450,10 +450,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): addsd %xmm2, %xmm13 addsd %xmm7, %xmm14 addsd %xmm6, %xmm15 @@ -753,13 +753,13 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - je .L30 + je L(30) #ifdef LN movq K, %rax @@ -799,10 +799,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addsd %xmm2, %xmm9 movaps %xmm0, %xmm2 @@ -880,10 +880,10 @@ addq $8 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -892,10 +892,10 @@ #endif andq $3, %rax BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): addsd %xmm2, %xmm9 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -918,10 +918,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): addsd %xmm2, %xmm9 addsd %xmm6, %xmm11 @@ -1083,9 +1083,9 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #ifdef LN movq K, %rax @@ -1121,10 +1121,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): addsd %xmm5, %xmm8 movsd 2 * SIZE(BO), %xmm5 mulsd %xmm0, %xmm1 @@ -1166,10 +1166,10 @@ addq $8 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1182,10 +1182,10 @@ andq $3, %rax BRANCH BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulsd %xmm0, %xmm1 addq $2 * SIZE, BO mulsd %xmm0, %xmm3 @@ -1199,10 +1199,10 @@ addq $1 * SIZE, AO decq %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1301,7 +1301,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -1319,12 +1319,12 @@ #endif decq J # j -- - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $1, N - je .L999 + je L(999) ALIGN_4 #if defined(LT) || defined(RN) @@ -1359,10 +1359,10 @@ movq M, I sarq $2, I - jle .L50 + jle L(50) ALIGN_4 -.L41: +L(41): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1403,10 +1403,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): addsd %xmm9, %xmm8 movsd 4 * SIZE(AO), %xmm9 mulsd %xmm4, %xmm0 @@ -1480,10 +1480,10 @@ mulsd %xmm5, %xmm15 movsd 1 * SIZE(BO), %xmm5 - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1499,10 +1499,10 @@ andq $3, %rax BRANCH BRANCH - je .L49 + je L(49) ALIGN_4 -.L46: +L(46): mulsd %xmm4, %xmm0 mulsd %xmm4, %xmm1 mulsd %xmm4, %xmm2 @@ -1522,10 +1522,10 @@ addq $1 * SIZE, BO decq %rax BRANCH - jg .L46 + jg L(46) ALIGN_4 -.L49: +L(49): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1678,12 +1678,12 @@ #endif decq I # i -- - jg .L41 + jg L(41) ALIGN_4 -.L50: +L(50): testq $2, M - je .L60 + je L(60) #ifdef LN movq K, %rax @@ -1718,10 +1718,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): addsd %xmm2, %xmm8 movsd 2 * SIZE(AO), %xmm2 mulsd %xmm4, %xmm0 @@ -1762,10 +1762,10 @@ mulsd %xmm5, %xmm3 movsd 1 * SIZE(BO), %xmm5 - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1777,10 +1777,10 @@ andq $3, %rax BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): mulsd %xmm4, %xmm0 mulsd %xmm4, %xmm1 movsd 1 * SIZE(BO), %xmm4 @@ -1794,10 +1794,10 @@ addq $1 * SIZE, BO decq %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1894,9 +1894,9 @@ #endif ALIGN_4 -.L60: +L(60): testq $1, M - je .L69 + je L(69) #ifdef LN movq K, %rax @@ -1933,10 +1933,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): addsd %xmm5, %xmm8 movsd 2 * SIZE(BO), %xmm5 mulsd %xmm0, %xmm1 @@ -1961,13 +1961,13 @@ addq $4 * SIZE, BO decq %rax - jne .L62 + jne L(62) addsd %xmm5, %xmm8 addsd %xmm7, %xmm9 ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1976,10 +1976,10 @@ #endif andq $3, %rax BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): movsd 0 * SIZE(AO), %xmm0 movsd 0 * SIZE(BO), %xmm1 @@ -1991,10 +1991,10 @@ decq %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addsd %xmm9, %xmm8 #if defined(LN) || defined(RT) @@ -2068,7 +2068,7 @@ #endif ALIGN_4 -.L69: +L(69): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -2087,7 +2087,7 @@ #endif ALIGN_2 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LT_4x4_barcelona.S b/kernel/x86_64/trsm_kernel_LT_4x4_barcelona.S index b3712004c3..e85dddf79b 100644 --- a/kernel/x86_64/trsm_kernel_LT_4x4_barcelona.S +++ b/kernel/x86_64/trsm_kernel_LT_4x4_barcelona.S @@ -480,9 +480,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -521,10 +521,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -593,10 +593,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -606,10 +606,10 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) BRANCH - jl .L12 + jl L(12) ALIGN_4 -.L15: +L(15): prefetch -8 * SIZE(BB) subq $-16 * SIZE, BB @@ -620,7 +620,7 @@ subq KK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_4 @@ -633,7 +633,7 @@ subq $-16 * SIZE, AO ALIGN_4 -.L16: +L(16): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -641,7 +641,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L19 + je L(19) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -649,7 +649,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -676,10 +676,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L17 + jl L(17) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1086,18 +1086,18 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $3, M - je .L39 + je L(39) testq $2, M - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -1141,10 +1141,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L26 + je L(26) ALIGN_4 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -1200,10 +1200,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L22 + jl L(22) ALIGN_4 -.L26: +L(26): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1211,7 +1211,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L29 + je L(29) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -1219,7 +1219,7 @@ negq %rax ALIGN_4 -.L27: +L(27): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -1235,10 +1235,10 @@ movddup -11 * SIZE(BO, %rax, 4), %xmm5 addq $SIZE, %rax - jl .L27 + jl L(27) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1456,9 +1456,9 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #ifdef LN movq K, %rax @@ -1503,10 +1503,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L36 + je L(36) ALIGN_4 -.L32: +L(32): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO, %rax, 4), %xmm0 addpd %xmm1, %xmm8 @@ -1534,10 +1534,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L32 + jl L(32) ALIGN_4 -.L36: +L(36): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1545,7 +1545,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L38 + je L(38) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -1553,7 +1553,7 @@ negq %rax ALIGN_4 -.L37: +L(37): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO, %rax, 4), %xmm0 addpd %xmm1, %xmm8 @@ -1562,10 +1562,10 @@ movddup -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L37 + jl L(37) ALIGN_4 -.L38: +L(38): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1746,7 +1746,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1765,12 +1765,12 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $2, N - je .L80 + je L(80) #if defined(LT) || defined(RN) movq A, AO @@ -1806,10 +1806,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1864,10 +1864,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L56 + je L(56) ALIGN_4 -.L52: +L(52): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -1919,10 +1919,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L52 + jl L(52) ALIGN_4 -.L56: +L(56): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1930,7 +1930,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L59 + je L(59) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -1938,7 +1938,7 @@ negq %rax ALIGN_4 -.L57: +L(57): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -1953,10 +1953,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L57 + jl L(57) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2169,12 +2169,12 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) #ifdef LN movq K, %rax @@ -2218,10 +2218,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L66 + je L(66) ALIGN_4 -.L62: +L(62): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -2253,10 +2253,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L62 + jl L(62) ALIGN_4 -.L66: +L(66): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2264,7 +2264,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L69 + je L(69) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -2272,7 +2272,7 @@ negq %rax ALIGN_4 -.L67: +L(67): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -2282,10 +2282,10 @@ movddup -13 * SIZE(BO, %rax, 2), %xmm3 addq $SIZE, %rax - jl .L67 + jl L(67) ALIGN_4 -.L69: +L(69): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -2426,12 +2426,12 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -2474,10 +2474,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L76 + je L(76) ALIGN_4 -.L72: +L(72): mulpd -16 * SIZE(BO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -12 * SIZE(AO, %rax, 1), %xmm0 @@ -2496,10 +2496,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L72 + jl L(72) ALIGN_4 -.L76: +L(76): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2507,7 +2507,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L78 + je L(78) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -2515,16 +2515,16 @@ negq %rax ALIGN_4 -.L77: +L(77): mulpd -16 * SIZE(BO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L77 + jl L(77) ALIGN_4 -.L78: +L(78): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -2627,7 +2627,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2646,9 +2646,9 @@ #endif ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) #if defined(LT) || defined(RN) movq A, AO @@ -2682,10 +2682,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2734,10 +2734,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L96 + je L(96) ALIGN_4 -.L92: +L(92): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -2765,10 +2765,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L92 + jl L(92) ALIGN_4 -.L96: +L(96): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2776,7 +2776,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L99 + je L(99) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -2784,7 +2784,7 @@ negq %rax ALIGN_4 -.L97: +L(97): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -2793,9 +2793,9 @@ movddup -15 * SIZE(BO, %rax, 1), %xmm1 addq $SIZE, %rax - jl .L97 + jl L(97) ALIGN_4 -.L99: +L(99): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -2972,12 +2972,12 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) #ifdef LN movq K, %rax @@ -3020,10 +3020,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L106 + je L(106) ALIGN_4 -.L102: +L(102): mulpd -16 * SIZE(AO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -12 * SIZE(BO, %rax, 1), %xmm0 @@ -3042,10 +3042,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L102 + jl L(102) ALIGN_4 -.L106: +L(106): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3053,7 +3053,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L109 + je L(109) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -3061,16 +3061,16 @@ negq %rax ALIGN_4 -.L107: +L(107): movddup -16 * SIZE(BO, %rax, 1), %xmm0 mulpd -16 * SIZE(AO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 addq $SIZE, %rax - jl .L107 + jl L(107) ALIGN_4 -.L109: +L(109): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -3187,12 +3187,12 @@ #endif ALIGN_4 -.L110: +L(110): testq $1, M - je .L119 + je L(119) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -3230,10 +3230,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L116 + je L(116) ALIGN_4 -.L112: +L(112): mulpd -16 * SIZE(BO, %rax, 1), %xmm0 addpd %xmm0, %xmm8 movapd -12 * SIZE(AO, %rax, 1), %xmm0 @@ -3244,10 +3244,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L112 + jl L(112) ALIGN_4 -.L116: +L(116): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3255,7 +3255,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L118 + je L(118) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -3263,16 +3263,16 @@ negq %rax ALIGN_4 -.L117: +L(117): mulsd -16 * SIZE(BO, %rax, 1), %xmm0 addsd %xmm0, %xmm8 movsd -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L117 + jl L(117) ALIGN_4 -.L118: +L(118): addpd %xmm9, %xmm8 haddpd %xmm8, %xmm8 @@ -3348,7 +3348,7 @@ #endif ALIGN_4 -.L119: +L(119): #ifdef LN leaq (B, K, SIZE), B #endif @@ -3367,7 +3367,7 @@ ALIGN_4 -.L999: +L(999): movq (%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LT_4x4_core2.S b/kernel/x86_64/trsm_kernel_LT_4x4_core2.S index c5fdc52827..de96086af1 100644 --- a/kernel/x86_64/trsm_kernel_LT_4x4_core2.S +++ b/kernel/x86_64/trsm_kernel_LT_4x4_core2.S @@ -171,9 +171,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) -.L01: +L(01): /* Copying to Sub Buffer */ #ifdef LN @@ -210,10 +210,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): prefetcht0 (PREFETCH_R + 0) * SIZE(B) movapd -16 * SIZE(B), %xmm0 movapd -14 * SIZE(B), %xmm1 @@ -269,10 +269,10 @@ subq $-16 * SIZE, B subq $-32 * SIZE, BO subq $1, %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -281,10 +281,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movapd -16 * SIZE(B), %xmm0 movapd -14 * SIZE(B), %xmm1 @@ -301,10 +301,10 @@ addq $4 * SIZE, B addq $8 * SIZE, BO subq $1, %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): leaq (PREFETCH_R + 0) * SIZE(B), BB #if defined(LT) || defined(RN) @@ -326,10 +326,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -381,10 +381,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm2, %xmm10 @@ -504,10 +504,10 @@ mulpd %xmm1, %xmm5 BRANCH - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -516,10 +516,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): movapd -16 * SIZE(AO), %xmm0 addpd %xmm2, %xmm10 movapd -16 * SIZE(BO), %xmm2 @@ -552,10 +552,10 @@ addq $8 * SIZE, BO # boffset1 += 8 subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): addpd %xmm2, %xmm10 addpd %xmm3, %xmm14 addpd %xmm4, %xmm11 @@ -1008,18 +1008,18 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $3, M - je .L39 + je L(39) testq $2, M - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -1053,10 +1053,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1126,10 +1126,10 @@ subq $ -8 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1138,10 +1138,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -1161,10 +1161,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jne .L26 + jne L(26) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1407,12 +1407,12 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1446,10 +1446,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -16 * SIZE(AO), %xmm0 @@ -1519,10 +1519,10 @@ subq $ -4 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1531,10 +1531,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): movsd -16 * SIZE(AO), %xmm0 movsd -16 * SIZE(BO), %xmm2 movsd -14 * SIZE(BO), %xmm3 @@ -1554,10 +1554,10 @@ addq $1 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1733,7 +1733,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1755,18 +1755,18 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $3, N - je .L999 + je L(999) testq $2, N - je .L80 + je L(80) ALIGN_4 -.L41: +L(41): /* Copying to Sub Buffer */ #ifdef LN @@ -1803,10 +1803,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): movddup -16 * SIZE(B), %xmm0 movddup -15 * SIZE(B), %xmm1 movddup -14 * SIZE(B), %xmm2 @@ -1828,10 +1828,10 @@ addq $8 * SIZE, B addq $16 * SIZE, BO subq $1, %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1840,10 +1840,10 @@ #endif andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movddup -16 * SIZE(B), %xmm0 movddup -15 * SIZE(B), %xmm1 @@ -1853,10 +1853,10 @@ addq $2 * SIZE, B addq $4 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): #if defined(LT) || defined(RN) movq A, AO #else @@ -1876,10 +1876,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1923,10 +1923,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -2004,10 +2004,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2016,10 +2016,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): movapd -16 * SIZE(AO), %xmm0 movapd -14 * SIZE(AO), %xmm1 @@ -2041,10 +2041,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2282,15 +2282,15 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -2324,10 +2324,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -2367,10 +2367,10 @@ subq $ -8 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2379,10 +2379,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_4 -.L66: +L(66): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -2396,10 +2396,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L66 + jg L(66) ALIGN_4 -.L69: +L(69): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -2557,12 +2557,12 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -2596,10 +2596,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -16 * SIZE(AO), %xmm0 @@ -2639,10 +2639,10 @@ subq $ -4 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2651,10 +2651,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): movsd -16 * SIZE(AO), %xmm0 movsd -16 * SIZE(BO), %xmm2 movsd -14 * SIZE(BO), %xmm3 @@ -2668,10 +2668,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addsd %xmm10, %xmm8 addsd %xmm11, %xmm9 @@ -2790,7 +2790,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2812,12 +2812,12 @@ #endif ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L81: +L(81): /* Copying to Sub Buffer */ #ifdef LN @@ -2854,10 +2854,10 @@ subq KK, %rax #endif sarq $3, %rax - jle .L83 + jle L(83) ALIGN_4 -.L82: +L(82): movddup -16 * SIZE(B), %xmm0 movddup -15 * SIZE(B), %xmm1 movddup -14 * SIZE(B), %xmm2 @@ -2879,10 +2879,10 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO subq $1, %rax - jne .L82 + jne L(82) ALIGN_4 -.L83: +L(83): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2891,10 +2891,10 @@ #endif andq $7, %rax BRANCH - jle .L90 + jle L(90) ALIGN_4 -.L84: +L(84): movddup -16 * SIZE(B), %xmm0 movapd %xmm0, 0 * SIZE(BO) @@ -2902,10 +2902,10 @@ addq $1 * SIZE, B addq $2 * SIZE, BO subq $1, %rax - jne .L84 + jne L(84) ALIGN_4 -.L90: +L(90): #if defined(LT) || defined(RN) movq A, AO #else @@ -2923,10 +2923,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2965,10 +2965,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -3014,10 +3014,10 @@ subq $-16 * SIZE, AO subq $ -8 * SIZE, BO subq $1, %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3026,10 +3026,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_4 -.L96: +L(96): movapd -16 * SIZE(AO), %xmm0 movapd -14 * SIZE(AO), %xmm1 @@ -3044,10 +3044,10 @@ addq $4 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L96 + jg L(96) ALIGN_4 -.L99: +L(99): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -3248,15 +3248,15 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -3290,10 +3290,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L105 + je L(105) ALIGN_4 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -3321,10 +3321,10 @@ subq $-8 * SIZE, AO subq $-8 * SIZE, BO subq $1, %rax - jne .L102 + jne L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3333,10 +3333,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_4 -.L106: +L(106): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 @@ -3346,10 +3346,10 @@ addq $2 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L106 + jg L(106) ALIGN_4 -.L109: +L(109): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -3479,12 +3479,12 @@ #endif ALIGN_4 -.L110: +L(110): testq $1, M - je .L119 + je L(119) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -3518,10 +3518,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -16 * SIZE(AO), %xmm0 @@ -3547,10 +3547,10 @@ subq $-4 * SIZE, AO subq $-8 * SIZE, BO subq $1, %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3559,10 +3559,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): movsd -16 * SIZE(AO), %xmm0 movsd -16 * SIZE(BO), %xmm2 @@ -3572,10 +3572,10 @@ addq $1 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addsd %xmm10, %xmm8 addsd %xmm11, %xmm9 addsd %xmm9, %xmm8 @@ -3676,7 +3676,7 @@ #endif ALIGN_4 -.L119: +L(119): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -3699,7 +3699,7 @@ ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/trsm_kernel_LT_4x4_penryn.S b/kernel/x86_64/trsm_kernel_LT_4x4_penryn.S index e186b94522..5c43a929f5 100644 --- a/kernel/x86_64/trsm_kernel_LT_4x4_penryn.S +++ b/kernel/x86_64/trsm_kernel_LT_4x4_penryn.S @@ -166,10 +166,10 @@ movq N, J sarq $2, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -209,10 +209,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -279,10 +279,10 @@ #endif sarq $3, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): addpd %xmm3, %xmm11 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -14 * SIZE(BO), %xmm3 @@ -532,10 +532,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -544,10 +544,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addpd %xmm3, %xmm11 movaps -14 * SIZE(BO), %xmm3 addpd %xmm4, %xmm15 @@ -583,10 +583,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax subq $4, %rax @@ -1011,13 +1011,13 @@ decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) ALIGN_4 #ifdef LN @@ -1054,10 +1054,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_4 -.L22: +L(22): addpd %xmm3, %xmm11 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -1123,10 +1123,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1135,10 +1135,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): addpd %xmm3, %xmm11 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -1159,10 +1159,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1390,10 +1390,10 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M BRANCH - jle .L39 + jle L(39) ALIGN_4 #ifdef LN @@ -1429,10 +1429,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) shufps $0x44, %xmm0, %xmm0 @@ -1479,10 +1479,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1491,10 +1491,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): shufps $0x44, %xmm0, %xmm0 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm3 @@ -1510,10 +1510,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1668,7 +1668,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1687,13 +1687,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $2, N BRANCH - jle .L80 + jle L(80) #if defined(LT) || defined(RN) movq A, AO @@ -1734,10 +1734,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1785,10 +1785,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_4 -.L52: +L(52): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -1863,10 +1863,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1875,10 +1875,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -1901,10 +1901,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2126,13 +2126,13 @@ decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M BRANCH - jle .L70 + jle L(70) ALIGN_4 #ifdef LN @@ -2166,10 +2166,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x4e, %xmm2, %xmm7 @@ -2212,10 +2212,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2224,10 +2224,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm7 @@ -2242,10 +2242,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2390,10 +2390,10 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M BRANCH - jle .L79 + jle L(79) ALIGN_4 #ifdef LN @@ -2426,10 +2426,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_4 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) shufps $0x44, %xmm0, %xmm0 @@ -2460,10 +2460,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2472,10 +2472,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): shufps $0x44, %xmm0, %xmm0 mulpd %xmm0, %xmm2 movsd -15 * SIZE(AO), %xmm0 @@ -2487,10 +2487,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2597,7 +2597,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2615,10 +2615,10 @@ #endif ALIGN_4 -.L80: +L(80): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(LT) || defined(RN) movq A, AO @@ -2653,10 +2653,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2695,10 +2695,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_4 -.L92: +L(92): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm2, %xmm3 @@ -2755,10 +2755,10 @@ subq $ -4 * SIZE, BO subq $1, %rax BRANCH - jg .L92 + jg L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2767,10 +2767,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): pshufd $0x44, %xmm2, %xmm3 pshufd $0x44, %xmm2, %xmm4 movsd -15 * SIZE(BO), %xmm2 @@ -2788,10 +2788,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2974,13 +2974,13 @@ decq I BRANCH - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M BRANCH - jle .L110 + jle L(110) ALIGN_4 #ifdef LN @@ -3013,10 +3013,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_4 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm2, %xmm3 @@ -3051,10 +3051,10 @@ subq $-4 * SIZE, BO subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3063,10 +3063,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L108 + je L(108) ALIGN_4 -.L106: +L(106): pshufd $0x44, %xmm2, %xmm3 movsd -15 * SIZE(BO), %xmm2 @@ -3079,10 +3079,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_4 -.L108: +L(108): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3198,10 +3198,10 @@ #endif ALIGN_4 -.L110: +L(110): testq $1, M BRANCH - jle .L119 + jle L(119) #ifdef LN movq K, %rax @@ -3233,10 +3233,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_4 -.L112: +L(112): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulsd %xmm0, %xmm2 @@ -3263,10 +3263,10 @@ subq $-4 * SIZE, BO subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3275,10 +3275,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd %xmm0, %xmm2 addsd %xmm2, %xmm8 movsd -15 * SIZE(AO), %xmm0 @@ -3289,10 +3289,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -3378,7 +3378,7 @@ #endif ALIGN_4 -.L119: +L(119): #ifdef LN leaq (B, K, SIZE), B #endif @@ -3395,7 +3395,7 @@ #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LT_4x4_sse2.S b/kernel/x86_64/trsm_kernel_LT_4x4_sse2.S index 583fb47a5b..25d71c33b6 100644 --- a/kernel/x86_64/trsm_kernel_LT_4x4_sse2.S +++ b/kernel/x86_64/trsm_kernel_LT_4x4_sse2.S @@ -298,9 +298,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) -.L01: +L(01): /* Copying to Sub Buffer */ #ifdef LN @@ -337,12 +337,12 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): PREFETCHNTA 40 * SIZE(B) movsd 0 * SIZE(B), %xmm0 @@ -375,10 +375,10 @@ movsd %xmm7, -1 * SIZE(BO) decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -387,10 +387,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movsd 0 * SIZE(B), %xmm0 movsd 1 * SIZE(B), %xmm1 movsd 2 * SIZE(B), %xmm2 @@ -408,10 +408,10 @@ addq $4 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -431,10 +431,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -489,8 +489,8 @@ #if 1 andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -508,7 +508,7 @@ KERNEL7(16 * 1) KERNEL8(16 * 1) cmpq $64 * 2, %rax - jle .L12 + jle L(12) KERNEL1(16 * 2) KERNEL2(16 * 2) KERNEL3(16 * 2) @@ -526,7 +526,7 @@ KERNEL7(16 * 3) KERNEL8(16 * 3) cmpq $64 * 4, %rax - jle .L12 + jle L(12) KERNEL1(16 * 4) KERNEL2(16 * 4) KERNEL3(16 * 4) @@ -544,7 +544,7 @@ KERNEL7(16 * 5) KERNEL8(16 * 5) cmpq $64 * 6, %rax - jle .L12 + jle L(12) KERNEL1(16 * 6) KERNEL2(16 * 6) KERNEL3(16 * 6) @@ -565,18 +565,18 @@ addq $16 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $64 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 4), BO # * 64 ALIGN_4 #else sarq $3, %rax - je .L15 + je L(15) -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -597,12 +597,12 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jg .L12 + jg L(12) ALIGN_4 #endif -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -611,10 +611,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -643,10 +643,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1158,18 +1158,18 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $3, M - je .L39 + je L(39) testq $2, M - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -1210,10 +1210,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1323,10 +1323,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1335,10 +1335,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -1355,10 +1355,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1628,12 +1628,12 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1674,10 +1674,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulsd %xmm8, %xmm9 addsd %xmm9, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1786,10 +1786,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1798,10 +1798,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulsd %xmm8, %xmm9 addsd %xmm9, %xmm0 movsd 2 * SIZE(BO), %xmm9 @@ -1818,10 +1818,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1997,7 +1997,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -2019,18 +2019,18 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $3, N - je .L999 + je L(999) testq $2, N - je .L80 + je L(80) ALIGN_4 -.L41: +L(41): /* Copying to Sub Buffer */ #ifdef LN @@ -2067,10 +2067,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): PREFETCH 56 * SIZE(B) movsd 0 * SIZE(B), %xmm0 @@ -2103,10 +2103,10 @@ movsd %xmm7, -1 * SIZE(BO) decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2115,10 +2115,10 @@ #endif andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movsd 0 * SIZE(B), %xmm0 movsd 1 * SIZE(B), %xmm1 @@ -2130,10 +2130,10 @@ addq $2 * SIZE, B addq $4 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): #if defined(LT) || defined(RN) movq A, AO #else @@ -2153,10 +2153,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2202,10 +2202,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(BO), %xmm8 @@ -2317,10 +2317,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2329,10 +2329,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 @@ -2349,10 +2349,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2617,15 +2617,15 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -2666,10 +2666,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(BO), %xmm8 @@ -2731,10 +2731,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2743,10 +2743,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_4 -.L66: +L(66): mulpd %xmm8, %xmm9 mulpd 2 * SIZE(BO), %xmm8 addpd %xmm9, %xmm0 @@ -2757,10 +2757,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L69: +L(69): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -2929,12 +2929,12 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -2975,10 +2975,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulsd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulsd 2 * SIZE(BO), %xmm8 @@ -3039,10 +3039,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3051,10 +3051,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulsd %xmm8, %xmm9 mulsd 2 * SIZE(BO), %xmm8 addsd %xmm9, %xmm0 @@ -3065,10 +3065,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addsd %xmm2, %xmm0 addsd %xmm3, %xmm1 @@ -3187,7 +3187,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -3209,12 +3209,12 @@ #endif ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L81: +L(81): /* Copying to Sub Buffer */ #ifdef LN @@ -3251,10 +3251,10 @@ subq KK, %rax #endif sarq $3, %rax - jle .L83 + jle L(83) ALIGN_4 -.L82: +L(82): PREFETCH 56 * SIZE(B) movsd 0 * SIZE(B), %xmm0 @@ -3287,10 +3287,10 @@ movsd %xmm7, -1 * SIZE(BO) decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L83: +L(83): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3299,10 +3299,10 @@ #endif andq $7, %rax BRANCH - jle .L90 + jle L(90) ALIGN_4 -.L84: +L(84): movsd 0 * SIZE(B), %xmm0 movsd %xmm0, 0 * SIZE(BO) @@ -3311,10 +3311,10 @@ addq $1 * SIZE, B addq $2 * SIZE, BO decq %rax - jne .L84 + jne L(84) ALIGN_4 -.L90: +L(90): #if defined(LT) || defined(RN) movq A, AO #else @@ -3332,10 +3332,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -3378,10 +3378,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(AO), %xmm9 @@ -3438,10 +3438,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3450,10 +3450,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_4 -.L96: +L(96): mulpd %xmm9, %xmm8 mulpd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -3464,10 +3464,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L99: +L(99): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -3669,15 +3669,15 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -3715,10 +3715,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L105 + je L(105) ALIGN_4 -.L102: +L(102): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd 2 * SIZE(AO), %xmm8 @@ -3751,10 +3751,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3763,10 +3763,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_4 -.L106: +L(106): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(AO), %xmm8 @@ -3775,10 +3775,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L106 + jg L(106) ALIGN_4 -.L109: +L(109): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -3909,12 +3909,12 @@ #endif ALIGN_4 -.L110: +L(110): testq $1, M - je .L119 + je L(119) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -3952,10 +3952,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulsd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd 1 * SIZE(AO), %xmm8 @@ -3987,10 +3987,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3999,10 +3999,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd %xmm8, %xmm9 movsd 1 * SIZE(AO), %xmm8 addsd %xmm9, %xmm0 @@ -4011,10 +4011,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addsd %xmm2, %xmm0 addsd %xmm3, %xmm1 addsd %xmm1, %xmm0 @@ -4115,7 +4115,7 @@ #endif ALIGN_4 -.L119: +L(119): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -4138,7 +4138,7 @@ ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/trsm_kernel_LT_4x4_sse3.S b/kernel/x86_64/trsm_kernel_LT_4x4_sse3.S index 3856c72098..004f85591e 100644 --- a/kernel/x86_64/trsm_kernel_LT_4x4_sse3.S +++ b/kernel/x86_64/trsm_kernel_LT_4x4_sse3.S @@ -399,10 +399,10 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -441,10 +441,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -497,8 +497,8 @@ #if 1 andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1 (16 * 0) KERNEL2 (16 * 0) KERNEL3 (16 * 0) @@ -517,7 +517,7 @@ KERNEL16(16 * 0) cmpq $128 * 1, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 1) KERNEL2 (16 * 1) KERNEL3 (16 * 1) @@ -536,7 +536,7 @@ KERNEL16(16 * 1) cmpq $128 * 2, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 2) KERNEL2 (16 * 2) KERNEL3 (16 * 2) @@ -555,7 +555,7 @@ KERNEL16(16 * 2) cmpq $128 * 3, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 3) KERNEL2 (16 * 3) KERNEL3 (16 * 3) @@ -574,7 +574,7 @@ KERNEL16(16 * 3) cmpq $128 * 4, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 4) KERNEL2 (16 * 4) KERNEL3 (16 * 4) @@ -593,7 +593,7 @@ KERNEL16(16 * 4) cmpq $128 * 5, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 5) KERNEL2 (16 * 5) KERNEL3 (16 * 5) @@ -612,7 +612,7 @@ KERNEL16(16 * 5) cmpq $128 * 6, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 6) KERNEL2 (16 * 6) KERNEL3 (16 * 6) @@ -631,7 +631,7 @@ KERNEL16(16 * 6) cmpq $128 * 7, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 7) KERNEL2 (16 * 7) KERNEL3 (16 * 7) @@ -652,17 +652,17 @@ addq $32 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $128 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 2), BO # * 64 #else sarq $3, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -889,11 +889,11 @@ addq $32 * SIZE, AO decq %rax BRANCH - jne .L12 + jne L(12) #endif ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -902,10 +902,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 addpd %xmm9, %xmm0 @@ -936,10 +936,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax subq $4, %rax @@ -1344,16 +1344,16 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax @@ -1387,10 +1387,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1500,10 +1500,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1512,10 +1512,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -1533,10 +1533,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1750,12 +1750,12 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1789,10 +1789,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1854,10 +1854,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1866,10 +1866,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -1881,10 +1881,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax @@ -2069,7 +2069,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -2087,12 +2087,12 @@ #endif decq J # j -- - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $2, N - je .L80 + je L(80) ALIGN_4 #if defined(LT) || defined(RN) @@ -2133,10 +2133,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2180,10 +2180,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -2302,10 +2302,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2314,10 +2314,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 addpd %xmm9, %xmm0 @@ -2336,10 +2336,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2556,15 +2556,15 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -2597,10 +2597,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -2662,10 +2662,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2674,10 +2674,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_4 -.L66: +L(66): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -2689,10 +2689,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L69: +L(69): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -2828,12 +2828,12 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -2866,10 +2866,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movddup 1 * SIZE(AO), %xmm8 @@ -2901,10 +2901,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2913,10 +2913,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulpd %xmm8, %xmm9 movddup 1 * SIZE(AO), %xmm8 addpd %xmm9, %xmm0 @@ -2925,10 +2925,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -3045,7 +3045,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -3064,9 +3064,9 @@ #endif ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 #if defined(LT) || defined(RN) @@ -3102,10 +3102,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -3144,10 +3144,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(AO), %xmm9 @@ -3202,10 +3202,10 @@ addq $32 * SIZE, AO addq $8 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3214,10 +3214,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_4 -.L96: +L(96): mulpd %xmm9, %xmm8 mulpd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -3228,10 +3228,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L99: +L(99): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -3414,15 +3414,15 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -3455,10 +3455,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L105 + je L(105) ALIGN_4 -.L102: +L(102): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movddup 1 * SIZE(BO), %xmm9 @@ -3490,10 +3490,10 @@ addq $16 * SIZE, AO addq $ 8 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3502,10 +3502,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_4 -.L106: +L(106): mulpd %xmm9, %xmm8 movddup 1 * SIZE(BO), %xmm9 addpd %xmm8, %xmm0 @@ -3514,10 +3514,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L106 + jg L(106) ALIGN_4 -.L109: +L(109): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -3633,12 +3633,12 @@ #endif ALIGN_4 -.L110: +L(110): testq $1, M - je .L119 + je L(119) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -3671,10 +3671,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulpd %xmm9, %xmm8 movapd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -3693,10 +3693,10 @@ addq $8 * SIZE, AO addq $8 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3705,10 +3705,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd 0 * SIZE(BO), %xmm9 addsd %xmm9, %xmm0 movsd 1 * SIZE(AO), %xmm9 @@ -3716,10 +3716,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addpd %xmm1, %xmm0 haddpd %xmm0, %xmm0 @@ -3808,7 +3808,7 @@ #endif ALIGN_4 -.L119: +L(119): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -3827,7 +3827,7 @@ #endif ALIGN_2 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LT_4x8_nehalem.S b/kernel/x86_64/trsm_kernel_LT_4x8_nehalem.S index 28f38bd617..17043e26fc 100644 --- a/kernel/x86_64/trsm_kernel_LT_4x8_nehalem.S +++ b/kernel/x86_64/trsm_kernel_LT_4x8_nehalem.S @@ -166,10 +166,10 @@ movq N, J sarq $3, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -209,10 +209,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -266,10 +266,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm12 @@ -396,10 +396,10 @@ subq $-16 * SIZE, AO subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -408,10 +408,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addps %xmm1, %xmm12 movaps -32 * SIZE(BO), %xmm1 addps %xmm2, %xmm13 @@ -447,10 +447,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1070,13 +1070,13 @@ decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) #ifdef LN movq K, %rax @@ -1114,10 +1114,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1195,10 +1195,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1207,10 +1207,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1234,10 +1234,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1687,10 +1687,10 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M BRANCH - jle .L39 + jle L(39) #ifdef LN movq K, %rax @@ -1722,10 +1722,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movaps -32 * SIZE(BO), %xmm2 @@ -1765,10 +1765,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1777,10 +1777,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -1795,10 +1795,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2197,7 +2197,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 8), B @@ -2216,12 +2216,12 @@ subq $1, J BRANCH - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $4, N - jle .L70 + jle L(70) #if defined(LT) || defined(RN) movq A, AO @@ -2258,10 +2258,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L50 + jle L(50) ALIGN_4 -.L41: +L(41): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2301,10 +2301,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -2371,10 +2371,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2383,10 +2383,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): addps %xmm1, %xmm8 movaps -32 * SIZE(BO), %xmm1 addps %xmm2, %xmm9 @@ -2407,10 +2407,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2712,13 +2712,13 @@ decq I BRANCH - jg .L41 + jg L(41) ALIGN_4 -.L50: +L(50): testq $2, M BRANCH - jle .L60 + jle L(60) #ifdef LN movq K, %rax @@ -2751,10 +2751,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2796,10 +2796,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2808,10 +2808,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2826,10 +2826,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3033,10 +3033,10 @@ #endif ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L69 + jle L(69) #ifdef LN movq K, %rax @@ -3067,10 +3067,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movaps -32 * SIZE(BO), %xmm2 @@ -3098,11 +3098,11 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) addps %xmm9, %xmm8 ALIGN_3 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3111,10 +3111,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -3126,10 +3126,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3303,7 +3303,7 @@ #endif ALIGN_4 -.L69: +L(69): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -3321,9 +3321,9 @@ #endif ALIGN_4 -.L70: +L(70): testq $2, N - jle .L100 + jle L(100) #if defined(LT) || defined(RN) movq A, AO @@ -3360,10 +3360,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L80 + jle L(80) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -3398,10 +3398,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -3444,10 +3444,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_3 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3456,10 +3456,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addps %xmm1, %xmm8 pshufd $0x00, %xmm3, %xmm1 mulps %xmm0, %xmm1 @@ -3474,10 +3474,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_3 -.L78: +L(78): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3681,13 +3681,13 @@ decq I BRANCH - jg .L71 + jg L(71) ALIGN_4 -.L80: +L(80): testq $2, M BRANCH - jle .L90 + jle L(90) #ifdef LN movq K, %rax @@ -3720,10 +3720,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): addps %xmm1, %xmm8 movsd -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -3753,10 +3753,10 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) ALIGN_3 -.L85: +L(85): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3765,10 +3765,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): addps %xmm1, %xmm8 movsd -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -3780,10 +3780,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_3 -.L88: +L(88): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3910,10 +3910,10 @@ #endif ALIGN_4 -.L90: +L(90): testq $1, M BRANCH - jle .L99 + jle L(99) #ifdef LN movq K, %rax @@ -3944,10 +3944,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_3 -.L92: +L(92): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movsd -32 * SIZE(BO), %xmm2 @@ -3975,11 +3975,11 @@ subq $1, %rax BRANCH - jg .L92 + jg L(92) addps %xmm9, %xmm8 ALIGN_3 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3988,10 +3988,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_3 -.L96: +L(96): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -4003,10 +4003,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_3 -.L98: +L(98): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -4112,7 +4112,7 @@ #endif ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -4130,9 +4130,9 @@ #endif ALIGN_4 -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) #if defined(LT) || defined(RN) movq A, AO @@ -4167,10 +4167,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L110 + jle L(110) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -4201,10 +4201,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_3 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -4235,10 +4235,10 @@ subq $ -4 * SIZE, BO subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_3 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4247,10 +4247,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L108 + je L(108) ALIGN_3 -.L106: +L(106): addps %xmm1, %xmm8 pshufd $0x00, %xmm3, %xmm1 movss -31 * SIZE(BO), %xmm3 @@ -4262,10 +4262,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_3 -.L108: +L(108): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -4433,13 +4433,13 @@ decq I BRANCH - jg .L101 + jg L(101) ALIGN_4 -.L110: +L(110): testq $2, M BRANCH - jle .L120 + jle L(120) #ifdef LN movq K, %rax @@ -4469,10 +4469,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_3 -.L112: +L(112): addps %xmm1, %xmm8 movss -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -4502,10 +4502,10 @@ subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_3 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4514,10 +4514,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_3 -.L116: +L(116): addps %xmm1, %xmm8 movss -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -4529,10 +4529,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_3 -.L118: +L(118): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -4636,10 +4636,10 @@ #endif ALIGN_4 -.L120: +L(120): testq $1, M BRANCH - jle .L129 + jle L(129) #ifdef LN movq K, %rax @@ -4669,10 +4669,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L125 + jle L(125) ALIGN_3 -.L122: +L(122): addss %xmm2, %xmm8 movss -32 * SIZE(BO), %xmm2 mulss %xmm0, %xmm2 @@ -4698,10 +4698,10 @@ subq $1, %rax BRANCH - jg .L122 + jg L(122) ALIGN_3 -.L125: +L(125): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4710,10 +4710,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_3 -.L126: +L(126): addss %xmm2, %xmm8 movss -32 * SIZE(BO), %xmm2 mulss %xmm0, %xmm2 @@ -4724,10 +4724,10 @@ subq $1, %rax BRANCH - jg .L126 + jg L(126) ALIGN_3 -.L128: +L(128): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -4800,7 +4800,7 @@ #endif ALIGN_4 -.L129: +L(129): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -4818,7 +4818,7 @@ #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LT_8x4_sse.S b/kernel/x86_64/trsm_kernel_LT_8x4_sse.S index 887c071ba8..81e69910f9 100644 --- a/kernel/x86_64/trsm_kernel_LT_8x4_sse.S +++ b/kernel/x86_64/trsm_kernel_LT_8x4_sse.S @@ -189,9 +189,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L50 + jle L(50) -.L01: +L(01): /* Copying to Sub Buffer */ #ifdef LN @@ -228,10 +228,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 movaps 8 * SIZE(B), %xmm11 @@ -279,10 +279,10 @@ addq $64 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -291,10 +291,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movaps 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -310,10 +310,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -333,10 +333,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $3 + BASE_SHIFT, %rax @@ -388,10 +388,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 0 * SIZE(BO), %xmm9 @@ -492,10 +492,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -504,9 +504,9 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -535,10 +535,10 @@ addq $8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1201,12 +1201,12 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M - je .L30 + je L(30) #ifdef LN movq K, %rax @@ -1249,10 +1249,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -1366,10 +1366,10 @@ addq $ 32 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1378,10 +1378,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1398,10 +1398,10 @@ addq $ 4 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1725,9 +1725,9 @@ #endif ALIGN_4 -.L30: +L(30): testq $2, M - je .L40 + je L(40) #ifdef LN movq K, %rax @@ -1770,10 +1770,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -1892,10 +1892,10 @@ addq $ 16 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1904,10 +1904,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1925,10 +1925,10 @@ addq $ 2 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2173,9 +2173,9 @@ #endif ALIGN_4 -.L40: +L(40): testq $1, M - je .L49 + je L(49) #ifdef LN movq K, %rax @@ -2217,10 +2217,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): mulss %xmm8, %xmm9 addss %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -2339,10 +2339,10 @@ addq $ 8 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2351,10 +2351,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -2372,10 +2372,10 @@ addq $ 1 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2576,7 +2576,7 @@ #endif ALIGN_4 -.L49: +L(49): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -2598,11 +2598,11 @@ #endif decq J # j -- - jg .L01 + jg L(01) -.L50: +L(50): testq $2, N - je .L100 + je L(100) #ifdef LN movq OFFSET, %rax @@ -2638,10 +2638,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 @@ -2668,10 +2668,10 @@ addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2680,10 +2680,10 @@ #endif andq $3, %rax BRANCH - jle .L60 + jle L(60) ALIGN_4 -.L54: +L(54): movsd 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -2695,10 +2695,10 @@ addq $2 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L54 + jne L(54) ALIGN_4 -.L60: +L(60): #if defined(LT) || defined(RN) movq A, AO #else @@ -2717,10 +2717,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L70 + jle L(70) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $3 + BASE_SHIFT, %rax @@ -2767,10 +2767,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2890,10 +2890,10 @@ addq $64 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2902,10 +2902,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -2922,10 +2922,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3443,12 +3443,12 @@ #endif decq I # i -- - jg .L61 + jg L(61) ALIGN_4 -.L70: +L(70): testq $4, M - je .L80 + je L(80) #ifdef LN movq K, %rax @@ -3491,10 +3491,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3558,10 +3558,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3570,10 +3570,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -3584,10 +3584,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -3835,9 +3835,9 @@ #endif ALIGN_4 -.L80: +L(80): testq $2, M - je .L90 + je L(90) #ifdef LN movq K, %rax @@ -3880,10 +3880,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L85 + je L(85) ALIGN_4 -.L82: +L(82): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -3954,10 +3954,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L85: +L(85): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3966,10 +3966,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -3981,10 +3981,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -4154,9 +4154,9 @@ #endif ALIGN_4 -.L90: +L(90): testq $1, M - je .L99 + je L(99) #ifdef LN movq K, %rax @@ -4198,10 +4198,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -4272,10 +4272,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4284,10 +4284,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -4299,10 +4299,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): addss %xmm2, %xmm0 addss %xmm3, %xmm1 @@ -4428,7 +4428,7 @@ #endif ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -4450,9 +4450,9 @@ #endif ALIGN_4 -.L100: +L(100): testq $1, N - je .L999 + je L(999) #ifdef LN movq OFFSET, %rax @@ -4488,10 +4488,10 @@ subq KK, %rax #endif sarq $3, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movsd 0 * SIZE(B), %xmm3 movhps 2 * SIZE(B), %xmm3 movsd 4 * SIZE(B), %xmm7 @@ -4520,10 +4520,10 @@ addq $32 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4532,10 +4532,10 @@ #endif andq $7, %rax BRANCH - jle .L110 + jle L(110) ALIGN_4 -.L104: +L(104): movss 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -4545,10 +4545,10 @@ addq $ 1 * SIZE, B addq $ 4 * SIZE, BO decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L110: +L(110): #if defined(LT) || defined(RN) movq A, AO #else @@ -4566,10 +4566,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L120 + jle L(120) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $3 + BASE_SHIFT, %rax @@ -4615,10 +4615,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulps %xmm9, %xmm8 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -4691,10 +4691,10 @@ addq $64 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4703,10 +4703,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulps %xmm9, %xmm8 mulps 4 * SIZE(AO), %xmm9 addps %xmm8, %xmm0 @@ -4717,10 +4717,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -5144,12 +5144,12 @@ #endif decq I # i -- - jg .L111 + jg L(111) ALIGN_4 -.L120: +L(120): testq $4, M - je .L130 + je L(130) #ifdef LN movq K, %rax @@ -5190,10 +5190,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L125 + je L(125) ALIGN_4 -.L122: +L(122): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -5231,10 +5231,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L122 + jne L(122) ALIGN_4 -.L125: +L(125): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -5243,10 +5243,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): mulps %xmm8, %xmm9 movaps 4 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -5255,10 +5255,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): addps %xmm1, %xmm0 addps %xmm3, %xmm2 addps %xmm2, %xmm0 @@ -5457,9 +5457,9 @@ #endif ALIGN_4 -.L130: +L(130): testq $2, M - je .L140 + je L(140) #ifdef LN movq K, %rax @@ -5500,10 +5500,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L135 + je L(135) ALIGN_4 -.L132: +L(132): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -5549,10 +5549,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L132 + jne L(132) ALIGN_4 -.L135: +L(135): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -5561,10 +5561,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): mulps %xmm8, %xmm9 movsd 2 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -5573,10 +5573,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): addps %xmm1, %xmm0 #if defined(LN) || defined(RT) @@ -5705,9 +5705,9 @@ #endif ALIGN_4 -.L140: +L(140): testq $1, M - je .L149 + je L(149) #ifdef LN movq K, %rax @@ -5747,10 +5747,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L145 + je L(145) ALIGN_4 -.L142: +L(142): mulss %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -5784,10 +5784,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L142 + jne L(142) ALIGN_4 -.L145: +L(145): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -5796,10 +5796,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): mulss %xmm8, %xmm9 movss 1 * SIZE(AO), %xmm8 addss %xmm9, %xmm0 @@ -5808,10 +5808,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): addss %xmm1, %xmm0 addss %xmm3, %xmm2 addss %xmm2, %xmm0 @@ -5896,7 +5896,7 @@ #endif ALIGN_4 -.L149: +L(149): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -5918,7 +5918,7 @@ #endif ALIGN_4 -.L999: +L(999): movq %rbx, %rsp EMMS movq 0(%rsp), %rbx diff --git a/kernel/x86_64/trsm_kernel_RT_2x8_nehalem.S b/kernel/x86_64/trsm_kernel_RT_2x8_nehalem.S index b8e75e798b..de58f5a281 100644 --- a/kernel/x86_64/trsm_kernel_RT_2x8_nehalem.S +++ b/kernel/x86_64/trsm_kernel_RT_2x8_nehalem.S @@ -165,7 +165,7 @@ #endif testq $1, N - jle .L30 + jle L(30) ALIGN_4 #if defined(LT) || defined(RN) @@ -201,10 +201,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L80 + jle L(80) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -239,10 +239,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -270,12 +270,12 @@ subq $1, %rax BRANCH - jg .L72 + jg L(72) addpd %xmm9, %xmm8 ALIGN_3 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -284,10 +284,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addpd %xmm1, %xmm8 movddup -16 * SIZE(BO), %xmm1 mulpd %xmm0, %xmm1 @@ -298,10 +298,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -402,13 +402,13 @@ decq I BRANCH - jg .L71 + jg L(71) ALIGN_4 -.L80: +L(80): testq $1, M BRANCH - jle .L89 + jle L(89) ALIGN_4 #ifdef LN @@ -442,10 +442,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): mulpd %xmm0, %xmm1 movsd -14 * SIZE(AO), %xmm0 movhps -13 * SIZE(AO), %xmm0 @@ -465,12 +465,12 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) addpd %xmm9, %xmm8 ALIGN_3 -.L85: +L(85): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -479,10 +479,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): mulsd %xmm0, %xmm1 movsd -15 * SIZE(AO), %xmm0 addsd %xmm1, %xmm8 @@ -493,10 +493,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -568,7 +568,7 @@ #endif ALIGN_4 -.L89: +L(89): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -586,9 +586,9 @@ #endif ALIGN_4 -.L30: +L(30): testq $2, N - jle .L50 + jle L(50) ALIGN_4 #if defined(LT) || defined(RN) @@ -626,10 +626,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -665,10 +665,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -708,13 +708,13 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 ALIGN_3 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -723,10 +723,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addpd %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm9 @@ -740,10 +740,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -878,13 +878,13 @@ decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L69 + jle L(69) ALIGN_4 #ifdef LN @@ -916,10 +916,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): mulpd %xmm0, %xmm1 movddup -15 * SIZE(AO), %xmm0 addpd %xmm1, %xmm8 @@ -945,10 +945,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -957,10 +957,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): mulpd %xmm0, %xmm1 movddup -15 * SIZE(AO), %xmm0 addpd %xmm1, %xmm8 @@ -971,10 +971,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1077,7 +1077,7 @@ #endif ALIGN_4 -.L69: +L(69): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -1095,9 +1095,9 @@ #endif ALIGN_4 -.L50: +L(50): testq $4, N - jle .L70 + jle L(70) ALIGN_4 #if defined(LT) || defined(RN) @@ -1135,10 +1135,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -1178,10 +1178,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -1253,10 +1253,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1265,10 +1265,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): addpd %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm9 @@ -1290,10 +1290,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1521,13 +1521,13 @@ decq I BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L40: +L(40): testq $1, M BRANCH - jle .L49 + jle L(49) ALIGN_4 #ifdef LN @@ -1561,10 +1561,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1602,10 +1602,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1614,10 +1614,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1631,10 +1631,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1792,7 +1792,7 @@ #endif ALIGN_4 -.L49: +L(49): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1810,14 +1810,14 @@ #endif ALIGN_4 -.L70: +L(70): movq N, J sarq $3, J NOBRANCH - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -1857,10 +1857,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -1915,10 +1915,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm12 @@ -2041,10 +2041,10 @@ subq $-8 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2053,10 +2053,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addpd %xmm1, %xmm12 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm13 @@ -2092,10 +2092,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2545,13 +2545,13 @@ decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M BRANCH - jle .L29 + jle L(29) ALIGN_4 #ifdef LN @@ -2585,10 +2585,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -2650,10 +2650,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2662,10 +2662,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -2685,10 +2685,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3026,7 +3026,7 @@ #endif ALIGN_4 -.L29: +L(29): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 8), B @@ -3045,10 +3045,10 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_RT_4x2_atom.S b/kernel/x86_64/trsm_kernel_RT_4x2_atom.S index 9b5a93717c..a9740b88b0 100644 --- a/kernel/x86_64/trsm_kernel_RT_4x2_atom.S +++ b/kernel/x86_64/trsm_kernel_RT_4x2_atom.S @@ -155,7 +155,7 @@ #endif testq $1, N - je .L40 + je L(40) ALIGN_4 #if defined(LT) || defined(RN) @@ -190,10 +190,10 @@ movq M, I sarq $2, I - jle .L50 + jle L(50) ALIGN_4 -.L41: +L(41): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -234,10 +234,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): addsd %xmm9, %xmm8 movsd 4 * SIZE(AO), %xmm9 mulsd %xmm4, %xmm0 @@ -311,10 +311,10 @@ mulsd %xmm5, %xmm15 movsd 1 * SIZE(BO), %xmm5 - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -330,10 +330,10 @@ andq $3, %rax BRANCH BRANCH - je .L49 + je L(49) ALIGN_4 -.L46: +L(46): mulsd %xmm4, %xmm0 mulsd %xmm4, %xmm1 mulsd %xmm4, %xmm2 @@ -353,10 +353,10 @@ addq $1 * SIZE, BO decq %rax BRANCH - jg .L46 + jg L(46) ALIGN_4 -.L49: +L(49): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -509,12 +509,12 @@ #endif decq I # i -- - jg .L41 + jg L(41) ALIGN_4 -.L50: +L(50): testq $2, M - je .L60 + je L(60) #ifdef LN movq K, %rax @@ -549,10 +549,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): addsd %xmm2, %xmm8 movsd 2 * SIZE(AO), %xmm2 mulsd %xmm4, %xmm0 @@ -593,10 +593,10 @@ mulsd %xmm5, %xmm3 movsd 1 * SIZE(BO), %xmm5 - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -608,10 +608,10 @@ andq $3, %rax BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): mulsd %xmm4, %xmm0 mulsd %xmm4, %xmm1 movsd 1 * SIZE(BO), %xmm4 @@ -625,10 +625,10 @@ addq $1 * SIZE, BO decq %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -725,9 +725,9 @@ #endif ALIGN_4 -.L60: +L(60): testq $1, M - je .L69 + je L(69) #ifdef LN movq K, %rax @@ -764,10 +764,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): addsd %xmm5, %xmm8 movsd 2 * SIZE(BO), %xmm5 mulsd %xmm0, %xmm1 @@ -792,13 +792,13 @@ addq $4 * SIZE, BO decq %rax - jne .L62 + jne L(62) addsd %xmm5, %xmm8 addsd %xmm7, %xmm9 ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -807,10 +807,10 @@ #endif andq $3, %rax BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): movsd 0 * SIZE(AO), %xmm0 movsd 0 * SIZE(BO), %xmm1 @@ -822,10 +822,10 @@ decq %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addsd %xmm9, %xmm8 #if defined(LN) || defined(RT) @@ -899,7 +899,7 @@ #endif ALIGN_4 -.L69: +L(69): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -918,13 +918,13 @@ #endif ALIGN_2 -.L40: +L(40): movq N, J sarq $1, J - jle .L999 + jle L(999) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -963,10 +963,10 @@ movq M, I sarq $2, I - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1015,10 +1015,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): addsd %xmm2, %xmm13 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps %xmm0, %xmm2 @@ -1161,10 +1161,10 @@ mulsd %xmm3, %xmm6 movsd 1 * SIZE(BO), %xmm3 - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1173,10 +1173,10 @@ #endif andq $3, %rax BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): addsd %xmm2, %xmm13 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -1215,10 +1215,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): addsd %xmm2, %xmm13 addsd %xmm7, %xmm14 addsd %xmm6, %xmm15 @@ -1518,13 +1518,13 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - je .L30 + je L(30) #ifdef LN movq K, %rax @@ -1564,10 +1564,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addsd %xmm2, %xmm9 movaps %xmm0, %xmm2 @@ -1645,10 +1645,10 @@ addq $8 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1657,10 +1657,10 @@ #endif andq $3, %rax BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): addsd %xmm2, %xmm9 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -1683,10 +1683,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): addsd %xmm2, %xmm9 addsd %xmm6, %xmm11 @@ -1848,9 +1848,9 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #ifdef LN movq K, %rax @@ -1886,10 +1886,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): addsd %xmm5, %xmm8 movsd 2 * SIZE(BO), %xmm5 mulsd %xmm0, %xmm1 @@ -1931,10 +1931,10 @@ addq $8 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1947,10 +1947,10 @@ andq $3, %rax BRANCH BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulsd %xmm0, %xmm1 addq $2 * SIZE, BO mulsd %xmm0, %xmm3 @@ -1964,10 +1964,10 @@ addq $1 * SIZE, AO decq %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2066,7 +2066,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2084,10 +2084,10 @@ #endif decq J # j -- - jg .L10 + jg L(10) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_RT_4x4_barcelona.S b/kernel/x86_64/trsm_kernel_RT_4x4_barcelona.S index 08e92dc30c..24f70694df 100644 --- a/kernel/x86_64/trsm_kernel_RT_4x4_barcelona.S +++ b/kernel/x86_64/trsm_kernel_RT_4x4_barcelona.S @@ -479,7 +479,7 @@ #endif testq $1, N - je .L40 + je L(40) #if defined(LT) || defined(RN) movq A, AO @@ -513,10 +513,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -565,10 +565,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L96 + je L(96) ALIGN_4 -.L92: +L(92): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -596,10 +596,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L92 + jl L(92) ALIGN_4 -.L96: +L(96): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -607,7 +607,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L99 + je L(99) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -615,7 +615,7 @@ negq %rax ALIGN_4 -.L97: +L(97): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -624,9 +624,9 @@ movddup -15 * SIZE(BO, %rax, 1), %xmm1 addq $SIZE, %rax - jl .L97 + jl L(97) ALIGN_4 -.L99: +L(99): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -803,12 +803,12 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) #ifdef LN movq K, %rax @@ -851,10 +851,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L106 + je L(106) ALIGN_4 -.L102: +L(102): mulpd -16 * SIZE(AO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -12 * SIZE(BO, %rax, 1), %xmm0 @@ -873,10 +873,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L102 + jl L(102) ALIGN_4 -.L106: +L(106): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -884,7 +884,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L109 + je L(109) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -892,16 +892,16 @@ negq %rax ALIGN_4 -.L107: +L(107): movddup -16 * SIZE(BO, %rax, 1), %xmm0 mulpd -16 * SIZE(AO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 addq $SIZE, %rax - jl .L107 + jl L(107) ALIGN_4 -.L109: +L(109): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -1018,9 +1018,9 @@ #endif ALIGN_4 -.L110: +L(110): testq $1, M - je .L119 + je L(119) #ifdef LN movq K, %rax @@ -1059,10 +1059,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L116 + je L(116) ALIGN_4 -.L112: +L(112): mulpd -16 * SIZE(BO, %rax, 1), %xmm0 addpd %xmm0, %xmm8 movapd -12 * SIZE(AO, %rax, 1), %xmm0 @@ -1073,10 +1073,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L112 + jl L(112) ALIGN_4 -.L116: +L(116): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1084,7 +1084,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L118 + je L(118) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -1092,16 +1092,16 @@ negq %rax ALIGN_4 -.L117: +L(117): mulsd -16 * SIZE(BO, %rax, 1), %xmm0 addsd %xmm0, %xmm8 movsd -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L117 + jl L(117) ALIGN_4 -.L118: +L(118): addpd %xmm9, %xmm8 haddpd %xmm8, %xmm8 @@ -1177,7 +1177,7 @@ #endif ALIGN_4 -.L119: +L(119): #ifdef LN leaq (B, K, SIZE), B #endif @@ -1195,9 +1195,9 @@ #endif ALIGN_4 -.L40: +L(40): testq $2, N - je .L80 + je L(80) #if defined(LT) || defined(RN) movq A, AO @@ -1233,10 +1233,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1291,10 +1291,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L56 + je L(56) ALIGN_4 -.L52: +L(52): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -1346,10 +1346,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L52 + jl L(52) ALIGN_4 -.L56: +L(56): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1357,7 +1357,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L59 + je L(59) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -1365,7 +1365,7 @@ negq %rax ALIGN_4 -.L57: +L(57): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -1380,10 +1380,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L57 + jl L(57) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1596,12 +1596,12 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) #ifdef LN movq K, %rax @@ -1645,10 +1645,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L66 + je L(66) ALIGN_4 -.L62: +L(62): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -1680,10 +1680,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L62 + jl L(62) ALIGN_4 -.L66: +L(66): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1691,7 +1691,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L69 + je L(69) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -1699,7 +1699,7 @@ negq %rax ALIGN_4 -.L67: +L(67): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -1709,10 +1709,10 @@ movddup -13 * SIZE(BO, %rax, 2), %xmm3 addq $SIZE, %rax - jl .L67 + jl L(67) ALIGN_4 -.L69: +L(69): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1853,12 +1853,12 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1901,10 +1901,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L76 + je L(76) ALIGN_4 -.L72: +L(72): mulpd -16 * SIZE(BO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -12 * SIZE(AO, %rax, 1), %xmm0 @@ -1923,10 +1923,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L72 + jl L(72) ALIGN_4 -.L76: +L(76): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1934,7 +1934,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L78 + je L(78) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -1942,16 +1942,16 @@ negq %rax ALIGN_4 -.L77: +L(77): mulpd -16 * SIZE(BO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L77 + jl L(77) ALIGN_4 -.L78: +L(78): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -2054,7 +2054,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2073,12 +2073,12 @@ #endif ALIGN_4 -.L80: +L(80): movq N, J sarq $2, J # j = (n >> 2) - jle .L999 + jle L(999) -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -2118,10 +2118,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2192,10 +2192,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -2205,10 +2205,10 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) BRANCH - jl .L12 + jl L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2216,7 +2216,7 @@ subq KK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_4 @@ -2229,7 +2229,7 @@ subq $-16 * SIZE, AO ALIGN_4 -.L16: +L(16): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2237,7 +2237,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L19 + je L(19) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -2245,7 +2245,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -2272,10 +2272,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L17 + jl L(17) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2682,18 +2682,18 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $3, M - je .L39 + je L(39) testq $2, M - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -2737,10 +2737,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L26 + je L(26) ALIGN_4 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -2796,10 +2796,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L22 + jl L(22) ALIGN_4 -.L26: +L(26): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2807,7 +2807,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L29 + je L(29) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -2815,7 +2815,7 @@ negq %rax ALIGN_4 -.L27: +L(27): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -2831,10 +2831,10 @@ movddup -11 * SIZE(BO, %rax, 4), %xmm5 addq $SIZE, %rax - jl .L27 + jl L(27) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3052,9 +3052,9 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #ifdef LN movq K, %rax @@ -3099,10 +3099,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L36 + je L(36) ALIGN_4 -.L32: +L(32): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO, %rax, 4), %xmm0 addpd %xmm1, %xmm8 @@ -3130,10 +3130,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L32 + jl L(32) ALIGN_4 -.L36: +L(36): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3141,7 +3141,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L38 + je L(38) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -3149,7 +3149,7 @@ negq %rax ALIGN_4 -.L37: +L(37): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO, %rax, 4), %xmm0 addpd %xmm1, %xmm8 @@ -3158,10 +3158,10 @@ movddup -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L37 + jl L(37) ALIGN_4 -.L38: +L(38): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -3342,7 +3342,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -3361,10 +3361,10 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq (%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_RT_4x4_core2.S b/kernel/x86_64/trsm_kernel_RT_4x4_core2.S index 64e0342de0..2b1bb8d9f6 100644 --- a/kernel/x86_64/trsm_kernel_RT_4x4_core2.S +++ b/kernel/x86_64/trsm_kernel_RT_4x4_core2.S @@ -170,10 +170,10 @@ #endif testq $1, N - je .L40 + je L(40) ALIGN_4 -.L81: +L(81): /* Copying to Sub Buffer */ #ifdef LN @@ -210,10 +210,10 @@ subq KK, %rax #endif sarq $3, %rax - jle .L83 + jle L(83) ALIGN_4 -.L82: +L(82): movddup -16 * SIZE(B), %xmm0 movddup -15 * SIZE(B), %xmm1 movddup -14 * SIZE(B), %xmm2 @@ -235,10 +235,10 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO subq $1, %rax - jne .L82 + jne L(82) ALIGN_4 -.L83: +L(83): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -247,10 +247,10 @@ #endif andq $7, %rax BRANCH - jle .L90 + jle L(90) ALIGN_4 -.L84: +L(84): movddup -16 * SIZE(B), %xmm0 movapd %xmm0, 0 * SIZE(BO) @@ -258,10 +258,10 @@ addq $1 * SIZE, B addq $2 * SIZE, BO subq $1, %rax - jne .L84 + jne L(84) ALIGN_4 -.L90: +L(90): #if defined(LT) || defined(RN) movq A, AO #else @@ -279,10 +279,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -321,10 +321,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -370,10 +370,10 @@ subq $-16 * SIZE, AO subq $ -8 * SIZE, BO subq $1, %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -382,10 +382,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_4 -.L96: +L(96): movapd -16 * SIZE(AO), %xmm0 movapd -14 * SIZE(AO), %xmm1 @@ -400,10 +400,10 @@ addq $4 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L96 + jg L(96) ALIGN_4 -.L99: +L(99): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -604,15 +604,15 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -646,10 +646,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L105 + je L(105) ALIGN_4 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -677,10 +677,10 @@ subq $-8 * SIZE, AO subq $-8 * SIZE, BO subq $1, %rax - jne .L102 + jne L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -689,10 +689,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_4 -.L106: +L(106): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 @@ -702,10 +702,10 @@ addq $2 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L106 + jg L(106) ALIGN_4 -.L109: +L(109): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -835,12 +835,12 @@ #endif ALIGN_4 -.L110: +L(110): testq $1, M - je .L119 + je L(119) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -874,10 +874,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -16 * SIZE(AO), %xmm0 @@ -903,10 +903,10 @@ subq $-4 * SIZE, AO subq $-8 * SIZE, BO subq $1, %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -915,10 +915,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): movsd -16 * SIZE(AO), %xmm0 movsd -16 * SIZE(BO), %xmm2 @@ -928,10 +928,10 @@ addq $1 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addsd %xmm10, %xmm8 addsd %xmm11, %xmm9 addsd %xmm9, %xmm8 @@ -1032,7 +1032,7 @@ #endif ALIGN_4 -.L119: +L(119): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -1054,12 +1054,12 @@ #endif ALIGN_4 -.L40: +L(40): testq $2, N - je .L80 + je L(80) ALIGN_4 -.L41: +L(41): /* Copying to Sub Buffer */ #ifdef LN @@ -1096,10 +1096,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): movddup -16 * SIZE(B), %xmm0 movddup -15 * SIZE(B), %xmm1 movddup -14 * SIZE(B), %xmm2 @@ -1121,10 +1121,10 @@ addq $8 * SIZE, B addq $16 * SIZE, BO subq $1, %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1133,10 +1133,10 @@ #endif andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movddup -16 * SIZE(B), %xmm0 movddup -15 * SIZE(B), %xmm1 @@ -1146,10 +1146,10 @@ addq $2 * SIZE, B addq $4 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): #if defined(LT) || defined(RN) movq A, AO #else @@ -1169,10 +1169,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1216,10 +1216,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1297,10 +1297,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1309,10 +1309,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): movapd -16 * SIZE(AO), %xmm0 movapd -14 * SIZE(AO), %xmm1 @@ -1334,10 +1334,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1575,15 +1575,15 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -1617,10 +1617,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1660,10 +1660,10 @@ subq $ -8 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1672,10 +1672,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_4 -.L66: +L(66): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -1689,10 +1689,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L66 + jg L(66) ALIGN_4 -.L69: +L(69): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1850,12 +1850,12 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1889,10 +1889,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -16 * SIZE(AO), %xmm0 @@ -1932,10 +1932,10 @@ subq $ -4 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1944,10 +1944,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): movsd -16 * SIZE(AO), %xmm0 movsd -16 * SIZE(BO), %xmm2 movsd -14 * SIZE(BO), %xmm3 @@ -1961,10 +1961,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addsd %xmm10, %xmm8 addsd %xmm11, %xmm9 @@ -2083,7 +2083,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2105,12 +2105,12 @@ #endif ALIGN_4 -.L80: +L(80): movq N, J sarq $2, J # j = (n >> 2) - jle .L999 + jle L(999) -.L01: +L(01): /* Copying to Sub Buffer */ #ifdef LN @@ -2147,10 +2147,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): prefetcht0 (PREFETCH_R + 0) * SIZE(B) movapd -16 * SIZE(B), %xmm0 movapd -14 * SIZE(B), %xmm1 @@ -2206,10 +2206,10 @@ subq $-16 * SIZE, B subq $-32 * SIZE, BO subq $1, %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2218,10 +2218,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movapd -16 * SIZE(B), %xmm0 movapd -14 * SIZE(B), %xmm1 @@ -2238,10 +2238,10 @@ addq $4 * SIZE, B addq $8 * SIZE, BO subq $1, %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): leaq (PREFETCH_R + 0) * SIZE(B), BB #if defined(LT) || defined(RN) @@ -2263,10 +2263,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2329,10 +2329,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm2, %xmm10 @@ -2452,10 +2452,10 @@ mulpd %xmm1, %xmm5 BRANCH - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2464,10 +2464,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): movapd -16 * SIZE(AO), %xmm0 addpd %xmm2, %xmm10 movapd -16 * SIZE(BO), %xmm2 @@ -2500,10 +2500,10 @@ addq $8 * SIZE, BO # boffset1 += 8 subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): addpd %xmm2, %xmm10 addpd %xmm3, %xmm14 addpd %xmm4, %xmm11 @@ -2956,18 +2956,18 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $3, M - je .L39 + je L(39) testq $2, M - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -3001,10 +3001,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -3074,10 +3074,10 @@ subq $ -8 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3086,10 +3086,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -3109,10 +3109,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jne .L26 + jne L(26) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3355,12 +3355,12 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -3394,10 +3394,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -16 * SIZE(AO), %xmm0 @@ -3467,10 +3467,10 @@ subq $ -4 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3479,10 +3479,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): movsd -16 * SIZE(AO), %xmm0 movsd -16 * SIZE(BO), %xmm2 movsd -14 * SIZE(BO), %xmm3 @@ -3502,10 +3502,10 @@ addq $1 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3681,7 +3681,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -3703,10 +3703,10 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/trsm_kernel_RT_4x4_penryn.S b/kernel/x86_64/trsm_kernel_RT_4x4_penryn.S index f95200ae55..e87e680854 100644 --- a/kernel/x86_64/trsm_kernel_RT_4x4_penryn.S +++ b/kernel/x86_64/trsm_kernel_RT_4x4_penryn.S @@ -165,7 +165,7 @@ testq $1, N BRANCH - jle .L40 + jle L(40) #if defined(LT) || defined(RN) movq A, AO @@ -200,10 +200,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -242,10 +242,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_4 -.L92: +L(92): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm2, %xmm3 @@ -302,10 +302,10 @@ subq $ -4 * SIZE, BO subq $1, %rax BRANCH - jg .L92 + jg L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -314,10 +314,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): pshufd $0x44, %xmm2, %xmm3 pshufd $0x44, %xmm2, %xmm4 movsd -15 * SIZE(BO), %xmm2 @@ -335,10 +335,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -521,13 +521,13 @@ decq I BRANCH - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M BRANCH - jle .L110 + jle L(110) ALIGN_4 #ifdef LN @@ -560,10 +560,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_4 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm2, %xmm3 @@ -598,10 +598,10 @@ subq $-4 * SIZE, BO subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -610,10 +610,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L108 + je L(108) ALIGN_4 -.L106: +L(106): pshufd $0x44, %xmm2, %xmm3 movsd -15 * SIZE(BO), %xmm2 @@ -626,10 +626,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_4 -.L108: +L(108): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -745,10 +745,10 @@ #endif ALIGN_4 -.L110: +L(110): testq $1, M BRANCH - jle .L119 + jle L(119) #ifdef LN movq K, %rax @@ -780,10 +780,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_4 -.L112: +L(112): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulsd %xmm0, %xmm2 @@ -810,10 +810,10 @@ subq $-4 * SIZE, BO subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -822,10 +822,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd %xmm0, %xmm2 addsd %xmm2, %xmm8 movsd -15 * SIZE(AO), %xmm0 @@ -836,10 +836,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -925,7 +925,7 @@ #endif ALIGN_4 -.L119: +L(119): #ifdef LN leaq (B, K, SIZE), B #endif @@ -942,10 +942,10 @@ #endif ALIGN_4 -.L40: +L(40): testq $2, N BRANCH - jle .L80 + jle L(80) #if defined(LT) || defined(RN) movq A, AO @@ -987,10 +987,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1038,10 +1038,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_4 -.L52: +L(52): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -1116,10 +1116,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1128,10 +1128,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -1154,10 +1154,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1379,13 +1379,13 @@ decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M BRANCH - jle .L70 + jle L(70) ALIGN_4 #ifdef LN @@ -1419,10 +1419,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x4e, %xmm2, %xmm7 @@ -1465,10 +1465,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1477,10 +1477,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm7 @@ -1495,10 +1495,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1643,10 +1643,10 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M BRANCH - jle .L79 + jle L(79) ALIGN_4 #ifdef LN @@ -1679,10 +1679,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_4 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) shufps $0x44, %xmm0, %xmm0 @@ -1713,10 +1713,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1725,10 +1725,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): shufps $0x44, %xmm0, %xmm0 mulpd %xmm0, %xmm2 movsd -15 * SIZE(AO), %xmm0 @@ -1740,10 +1740,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1850,7 +1850,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -1868,14 +1868,14 @@ #endif ALIGN_4 -.L80: +L(80): movq N, J sarq $2, J NOBRANCH - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -1916,10 +1916,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1986,10 +1986,10 @@ #endif sarq $3, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): addpd %xmm3, %xmm11 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -14 * SIZE(BO), %xmm3 @@ -2239,10 +2239,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2251,10 +2251,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addpd %xmm3, %xmm11 movaps -14 * SIZE(BO), %xmm3 addpd %xmm4, %xmm15 @@ -2290,10 +2290,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax subq $4, %rax @@ -2718,13 +2718,13 @@ decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) ALIGN_4 #ifdef LN @@ -2761,10 +2761,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_4 -.L22: +L(22): addpd %xmm3, %xmm11 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -2830,10 +2830,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2842,10 +2842,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): addpd %xmm3, %xmm11 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -2866,10 +2866,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3097,10 +3097,10 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M BRANCH - jle .L39 + jle L(39) ALIGN_4 #ifdef LN @@ -3136,10 +3136,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) shufps $0x44, %xmm0, %xmm0 @@ -3186,10 +3186,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3198,10 +3198,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): shufps $0x44, %xmm0, %xmm0 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm3 @@ -3217,10 +3217,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3375,7 +3375,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -3394,10 +3394,10 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_RT_4x4_sse2.S b/kernel/x86_64/trsm_kernel_RT_4x4_sse2.S index 49a5fe6c25..a3725cbba7 100644 --- a/kernel/x86_64/trsm_kernel_RT_4x4_sse2.S +++ b/kernel/x86_64/trsm_kernel_RT_4x4_sse2.S @@ -297,10 +297,10 @@ #endif testq $1, N - je .L40 + je L(40) ALIGN_4 -.L81: +L(81): /* Copying to Sub Buffer */ #ifdef LN @@ -337,10 +337,10 @@ subq KK, %rax #endif sarq $3, %rax - jle .L83 + jle L(83) ALIGN_4 -.L82: +L(82): PREFETCH 56 * SIZE(B) movsd 0 * SIZE(B), %xmm0 @@ -373,10 +373,10 @@ movsd %xmm7, -1 * SIZE(BO) decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L83: +L(83): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -385,10 +385,10 @@ #endif andq $7, %rax BRANCH - jle .L90 + jle L(90) ALIGN_4 -.L84: +L(84): movsd 0 * SIZE(B), %xmm0 movsd %xmm0, 0 * SIZE(BO) @@ -397,10 +397,10 @@ addq $1 * SIZE, B addq $2 * SIZE, BO decq %rax - jne .L84 + jne L(84) ALIGN_4 -.L90: +L(90): #if defined(LT) || defined(RN) movq A, AO #else @@ -418,10 +418,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -464,10 +464,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(AO), %xmm9 @@ -524,10 +524,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -536,10 +536,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_4 -.L96: +L(96): mulpd %xmm9, %xmm8 mulpd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -550,10 +550,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L99: +L(99): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -755,15 +755,15 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -801,10 +801,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L105 + je L(105) ALIGN_4 -.L102: +L(102): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd 2 * SIZE(AO), %xmm8 @@ -837,10 +837,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -849,10 +849,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_4 -.L106: +L(106): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(AO), %xmm8 @@ -861,10 +861,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L106 + jg L(106) ALIGN_4 -.L109: +L(109): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -995,12 +995,12 @@ #endif ALIGN_4 -.L110: +L(110): testq $1, M - je .L119 + je L(119) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1038,10 +1038,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulsd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd 1 * SIZE(AO), %xmm8 @@ -1073,10 +1073,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1085,10 +1085,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd %xmm8, %xmm9 movsd 1 * SIZE(AO), %xmm8 addsd %xmm9, %xmm0 @@ -1097,10 +1097,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addsd %xmm2, %xmm0 addsd %xmm3, %xmm1 addsd %xmm1, %xmm0 @@ -1201,7 +1201,7 @@ #endif ALIGN_4 -.L119: +L(119): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -1224,12 +1224,12 @@ ALIGN_4 -.L40: +L(40): testq $2, N - je .L80 + je L(80) ALIGN_4 -.L41: +L(41): /* Copying to Sub Buffer */ #ifdef LN @@ -1266,10 +1266,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): PREFETCH 56 * SIZE(B) movsd 0 * SIZE(B), %xmm0 @@ -1302,10 +1302,10 @@ movsd %xmm7, -1 * SIZE(BO) decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1314,10 +1314,10 @@ #endif andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movsd 0 * SIZE(B), %xmm0 movsd 1 * SIZE(B), %xmm1 @@ -1329,10 +1329,10 @@ addq $2 * SIZE, B addq $4 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): #if defined(LT) || defined(RN) movq A, AO #else @@ -1352,10 +1352,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1401,10 +1401,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(BO), %xmm8 @@ -1516,10 +1516,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1528,10 +1528,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 @@ -1548,10 +1548,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1816,15 +1816,15 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -1865,10 +1865,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(BO), %xmm8 @@ -1930,10 +1930,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1942,10 +1942,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_4 -.L66: +L(66): mulpd %xmm8, %xmm9 mulpd 2 * SIZE(BO), %xmm8 addpd %xmm9, %xmm0 @@ -1956,10 +1956,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L69: +L(69): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -2128,12 +2128,12 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -2174,10 +2174,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulsd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulsd 2 * SIZE(BO), %xmm8 @@ -2238,10 +2238,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2250,10 +2250,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulsd %xmm8, %xmm9 mulsd 2 * SIZE(BO), %xmm8 addsd %xmm9, %xmm0 @@ -2264,10 +2264,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addsd %xmm2, %xmm0 addsd %xmm3, %xmm1 @@ -2386,7 +2386,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2408,12 +2408,12 @@ #endif ALIGN_4 -.L80: +L(80): movq N, J sarq $2, J # j = (n >> 2) - jle .L999 + jle L(999) -.L01: +L(01): /* Copying to Sub Buffer */ #ifdef LN @@ -2450,12 +2450,12 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): PREFETCHNTA 40 * SIZE(B) movsd 0 * SIZE(B), %xmm0 @@ -2488,10 +2488,10 @@ movsd %xmm7, -1 * SIZE(BO) decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2500,10 +2500,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movsd 0 * SIZE(B), %xmm0 movsd 1 * SIZE(B), %xmm1 movsd 2 * SIZE(B), %xmm2 @@ -2521,10 +2521,10 @@ addq $4 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -2544,10 +2544,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2600,8 +2600,8 @@ #endif andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -2619,7 +2619,7 @@ KERNEL7(16 * 1) KERNEL8(16 * 1) cmpq $64 * 2, %rax - jle .L12 + jle L(12) KERNEL1(16 * 2) KERNEL2(16 * 2) KERNEL3(16 * 2) @@ -2637,7 +2637,7 @@ KERNEL7(16 * 3) KERNEL8(16 * 3) cmpq $64 * 4, %rax - jle .L12 + jle L(12) KERNEL1(16 * 4) KERNEL2(16 * 4) KERNEL3(16 * 4) @@ -2655,7 +2655,7 @@ KERNEL7(16 * 5) KERNEL8(16 * 5) cmpq $64 * 6, %rax - jle .L12 + jle L(12) KERNEL1(16 * 6) KERNEL2(16 * 6) KERNEL3(16 * 6) @@ -2676,14 +2676,14 @@ addq $16 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $64 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 4), BO # * 64 ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2692,10 +2692,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -2724,10 +2724,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3239,18 +3239,18 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $3, M - je .L39 + je L(39) testq $2, M - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -3291,10 +3291,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3404,10 +3404,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3416,10 +3416,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -3436,10 +3436,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3709,12 +3709,12 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -3755,10 +3755,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulsd %xmm8, %xmm9 addsd %xmm9, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3867,10 +3867,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3879,10 +3879,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulsd %xmm8, %xmm9 addsd %xmm9, %xmm0 movsd 2 * SIZE(BO), %xmm9 @@ -3899,10 +3899,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -4078,7 +4078,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -4100,10 +4100,10 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/trsm_kernel_RT_4x4_sse3.S b/kernel/x86_64/trsm_kernel_RT_4x4_sse3.S index b6c56e056b..c4601cb7ab 100644 --- a/kernel/x86_64/trsm_kernel_RT_4x4_sse3.S +++ b/kernel/x86_64/trsm_kernel_RT_4x4_sse3.S @@ -396,7 +396,7 @@ #endif testq $1, N - je .L80 + je L(80) ALIGN_4 #if defined(LT) || defined(RN) @@ -432,10 +432,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -474,10 +474,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(AO), %xmm9 @@ -532,10 +532,10 @@ addq $32 * SIZE, AO addq $8 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -544,10 +544,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_4 -.L96: +L(96): mulpd %xmm9, %xmm8 mulpd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -558,10 +558,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L99: +L(99): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -744,15 +744,15 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -785,10 +785,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L105 + je L(105) ALIGN_4 -.L102: +L(102): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movddup 1 * SIZE(BO), %xmm9 @@ -820,10 +820,10 @@ addq $16 * SIZE, AO addq $ 8 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -832,10 +832,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_4 -.L106: +L(106): mulpd %xmm9, %xmm8 movddup 1 * SIZE(BO), %xmm9 addpd %xmm8, %xmm0 @@ -844,10 +844,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L106 + jg L(106) ALIGN_4 -.L109: +L(109): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -963,12 +963,12 @@ #endif ALIGN_4 -.L110: +L(110): testq $1, M - je .L119 + je L(119) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1001,10 +1001,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulpd %xmm9, %xmm8 movapd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -1023,10 +1023,10 @@ addq $8 * SIZE, AO addq $8 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1035,10 +1035,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd 0 * SIZE(BO), %xmm9 addsd %xmm9, %xmm0 movsd 1 * SIZE(AO), %xmm9 @@ -1046,10 +1046,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addpd %xmm1, %xmm0 haddpd %xmm0, %xmm0 @@ -1138,7 +1138,7 @@ #endif ALIGN_4 -.L119: +L(119): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -1157,9 +1157,9 @@ #endif ALIGN_2 -.L80: +L(80): testq $2, N - je .L40 + je L(40) ALIGN_4 #if defined(LT) || defined(RN) @@ -1196,10 +1196,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1240,10 +1240,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1362,10 +1362,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1374,10 +1374,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 addpd %xmm9, %xmm0 @@ -1396,10 +1396,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1616,15 +1616,15 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -1657,10 +1657,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1722,10 +1722,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1734,10 +1734,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_4 -.L66: +L(66): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -1749,10 +1749,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L69: +L(69): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -1888,12 +1888,12 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1926,10 +1926,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movddup 1 * SIZE(AO), %xmm8 @@ -1961,10 +1961,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1973,10 +1973,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulpd %xmm8, %xmm9 movddup 1 * SIZE(AO), %xmm8 addpd %xmm9, %xmm0 @@ -1985,10 +1985,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -2105,7 +2105,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2124,13 +2124,13 @@ #endif ALIGN_4 -.L40: +L(40): movq N, J sarq $2, J # j = (n >> 2) - jle .L999 + jle L(999) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -2165,10 +2165,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax @@ -2219,8 +2219,8 @@ #if 1 andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1 (16 * 0) KERNEL2 (16 * 0) KERNEL3 (16 * 0) @@ -2239,7 +2239,7 @@ KERNEL16(16 * 0) cmpq $128 * 1, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 1) KERNEL2 (16 * 1) KERNEL3 (16 * 1) @@ -2258,7 +2258,7 @@ KERNEL16(16 * 1) cmpq $128 * 2, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 2) KERNEL2 (16 * 2) KERNEL3 (16 * 2) @@ -2277,7 +2277,7 @@ KERNEL16(16 * 2) cmpq $128 * 3, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 3) KERNEL2 (16 * 3) KERNEL3 (16 * 3) @@ -2296,7 +2296,7 @@ KERNEL16(16 * 3) cmpq $128 * 4, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 4) KERNEL2 (16 * 4) KERNEL3 (16 * 4) @@ -2315,7 +2315,7 @@ KERNEL16(16 * 4) cmpq $128 * 5, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 5) KERNEL2 (16 * 5) KERNEL3 (16 * 5) @@ -2334,7 +2334,7 @@ KERNEL16(16 * 5) cmpq $128 * 6, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 6) KERNEL2 (16 * 6) KERNEL3 (16 * 6) @@ -2353,7 +2353,7 @@ KERNEL16(16 * 6) cmpq $128 * 7, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 7) KERNEL2 (16 * 7) KERNEL3 (16 * 7) @@ -2374,18 +2374,18 @@ addq $32 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $128 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 2), BO # * 64 #else sarq $3, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -2612,11 +2612,11 @@ addq $32 * SIZE, AO decq %rax BRANCH - jne .L12 + jne L(12) #endif ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2625,10 +2625,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 addpd %xmm9, %xmm0 @@ -2659,10 +2659,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax @@ -3068,16 +3068,16 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax @@ -3111,10 +3111,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -3224,10 +3224,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3236,10 +3236,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -3257,10 +3257,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3474,12 +3474,12 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -3513,10 +3513,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -3578,10 +3578,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3590,10 +3590,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -3605,10 +3605,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax @@ -3793,7 +3793,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -3811,11 +3811,11 @@ #endif decq J # j -- - jg .L10 + jg L(10) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_RT_4x8_nehalem.S b/kernel/x86_64/trsm_kernel_RT_4x8_nehalem.S index 4942f46710..326bc0ee36 100644 --- a/kernel/x86_64/trsm_kernel_RT_4x8_nehalem.S +++ b/kernel/x86_64/trsm_kernel_RT_4x8_nehalem.S @@ -164,7 +164,7 @@ #endif testq $1, N - jle .L40 + jle L(40) #if defined(LT) || defined(RN) movq A, AO @@ -199,10 +199,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L110 + jle L(110) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -233,10 +233,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_3 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -267,10 +267,10 @@ subq $ -4 * SIZE, BO subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_3 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -279,10 +279,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L108 + je L(108) ALIGN_3 -.L106: +L(106): addps %xmm1, %xmm8 pshufd $0x00, %xmm3, %xmm1 movss -31 * SIZE(BO), %xmm3 @@ -294,10 +294,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_3 -.L108: +L(108): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -465,13 +465,13 @@ decq I BRANCH - jg .L101 + jg L(101) ALIGN_4 -.L110: +L(110): testq $2, M BRANCH - jle .L120 + jle L(120) #ifdef LN movq K, %rax @@ -501,10 +501,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_3 -.L112: +L(112): addps %xmm1, %xmm8 movss -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -534,10 +534,10 @@ subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_3 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -546,10 +546,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_3 -.L116: +L(116): addps %xmm1, %xmm8 movss -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -561,10 +561,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_3 -.L118: +L(118): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -668,10 +668,10 @@ #endif ALIGN_4 -.L120: +L(120): testq $1, M BRANCH - jle .L129 + jle L(129) #ifdef LN movq K, %rax @@ -701,10 +701,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L125 + jle L(125) ALIGN_3 -.L122: +L(122): addss %xmm2, %xmm8 movss -32 * SIZE(BO), %xmm2 mulss %xmm0, %xmm2 @@ -730,10 +730,10 @@ subq $1, %rax BRANCH - jg .L122 + jg L(122) ALIGN_3 -.L125: +L(125): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -742,10 +742,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_3 -.L126: +L(126): addss %xmm2, %xmm8 movss -32 * SIZE(BO), %xmm2 mulss %xmm0, %xmm2 @@ -756,10 +756,10 @@ subq $1, %rax BRANCH - jg .L126 + jg L(126) ALIGN_3 -.L128: +L(128): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -832,7 +832,7 @@ #endif ALIGN_4 -.L129: +L(129): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -850,9 +850,9 @@ #endif ALIGN_4 -.L40: +L(40): testq $2, N - jle .L70 + jle L(70) #if defined(LT) || defined(RN) movq A, AO @@ -889,10 +889,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L80 + jle L(80) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -927,10 +927,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -973,10 +973,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_3 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -985,10 +985,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addps %xmm1, %xmm8 pshufd $0x00, %xmm3, %xmm1 mulps %xmm0, %xmm1 @@ -1003,10 +1003,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_3 -.L78: +L(78): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1210,13 +1210,13 @@ decq I BRANCH - jg .L71 + jg L(71) ALIGN_4 -.L80: +L(80): testq $2, M BRANCH - jle .L90 + jle L(90) #ifdef LN movq K, %rax @@ -1249,10 +1249,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): addps %xmm1, %xmm8 movsd -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -1282,10 +1282,10 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) ALIGN_3 -.L85: +L(85): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1294,10 +1294,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): addps %xmm1, %xmm8 movsd -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -1309,10 +1309,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_3 -.L88: +L(88): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1439,10 +1439,10 @@ #endif ALIGN_4 -.L90: +L(90): testq $1, M BRANCH - jle .L99 + jle L(99) #ifdef LN movq K, %rax @@ -1473,10 +1473,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_3 -.L92: +L(92): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movsd -32 * SIZE(BO), %xmm2 @@ -1504,11 +1504,11 @@ subq $1, %rax BRANCH - jg .L92 + jg L(92) addps %xmm9, %xmm8 ALIGN_3 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1517,10 +1517,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_3 -.L96: +L(96): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -1532,10 +1532,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_3 -.L98: +L(98): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1641,7 +1641,7 @@ #endif ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -1659,9 +1659,9 @@ #endif ALIGN_4 -.L70: +L(70): testq $4, N - jle .L100 + jle L(100) #if defined(LT) || defined(RN) movq A, AO @@ -1698,10 +1698,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L50 + jle L(50) ALIGN_4 -.L41: +L(41): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1741,10 +1741,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -1811,10 +1811,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1823,10 +1823,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): addps %xmm1, %xmm8 movaps -32 * SIZE(BO), %xmm1 addps %xmm2, %xmm9 @@ -1847,10 +1847,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2152,13 +2152,13 @@ decq I BRANCH - jg .L41 + jg L(41) ALIGN_4 -.L50: +L(50): testq $2, M BRANCH - jle .L60 + jle L(60) #ifdef LN movq K, %rax @@ -2191,10 +2191,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2236,10 +2236,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2248,10 +2248,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2266,10 +2266,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2473,10 +2473,10 @@ #endif ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L69 + jle L(69) #ifdef LN movq K, %rax @@ -2507,10 +2507,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movaps -32 * SIZE(BO), %xmm2 @@ -2538,11 +2538,11 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) addps %xmm9, %xmm8 ALIGN_3 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2551,10 +2551,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -2566,10 +2566,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2743,7 +2743,7 @@ #endif ALIGN_4 -.L69: +L(69): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -2761,14 +2761,14 @@ #endif ALIGN_4 -.L100: +L(100): movq N, J sarq $3, J NOBRANCH - jle .L999 + jle L(999) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -2808,10 +2808,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2865,10 +2865,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm12 @@ -2995,10 +2995,10 @@ subq $-16 * SIZE, AO subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3007,10 +3007,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addps %xmm1, %xmm12 movaps -32 * SIZE(BO), %xmm1 addps %xmm2, %xmm13 @@ -3046,10 +3046,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3669,13 +3669,13 @@ decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) #ifdef LN movq K, %rax @@ -3713,10 +3713,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -3794,10 +3794,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3806,10 +3806,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -3833,10 +3833,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -4286,10 +4286,10 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M BRANCH - jle .L39 + jle L(39) #ifdef LN movq K, %rax @@ -4321,10 +4321,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movaps -32 * SIZE(BO), %xmm2 @@ -4364,10 +4364,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4376,10 +4376,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -4394,10 +4394,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -4796,7 +4796,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 8), B @@ -4815,10 +4815,10 @@ subq $1, J BRANCH - jg .L10 + jg L(10) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_RT_8x4_sse.S b/kernel/x86_64/trsm_kernel_RT_8x4_sse.S index c854b93088..831c301d45 100644 --- a/kernel/x86_64/trsm_kernel_RT_8x4_sse.S +++ b/kernel/x86_64/trsm_kernel_RT_8x4_sse.S @@ -186,7 +186,7 @@ #endif testq $1, N - je .L50 + je L(50) #ifdef LN movq OFFSET, %rax @@ -222,10 +222,10 @@ subq KK, %rax #endif sarq $3, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movsd 0 * SIZE(B), %xmm3 movhps 2 * SIZE(B), %xmm3 movsd 4 * SIZE(B), %xmm7 @@ -254,10 +254,10 @@ addq $32 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -266,10 +266,10 @@ #endif andq $7, %rax BRANCH - jle .L110 + jle L(110) ALIGN_4 -.L104: +L(104): movss 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -279,10 +279,10 @@ addq $ 1 * SIZE, B addq $ 4 * SIZE, BO decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L110: +L(110): #if defined(LT) || defined(RN) movq A, AO #else @@ -300,10 +300,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L120 + jle L(120) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $3 + BASE_SHIFT, %rax @@ -349,10 +349,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulps %xmm9, %xmm8 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -425,10 +425,10 @@ addq $64 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -437,10 +437,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulps %xmm9, %xmm8 mulps 4 * SIZE(AO), %xmm9 addps %xmm8, %xmm0 @@ -451,10 +451,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -878,12 +878,12 @@ #endif decq I # i -- - jg .L111 + jg L(111) ALIGN_4 -.L120: +L(120): testq $4, M - je .L130 + je L(130) #ifdef LN movq K, %rax @@ -924,10 +924,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L125 + je L(125) ALIGN_4 -.L122: +L(122): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -965,10 +965,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L122 + jne L(122) ALIGN_4 -.L125: +L(125): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -977,10 +977,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): mulps %xmm8, %xmm9 movaps 4 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -989,10 +989,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): addps %xmm1, %xmm0 addps %xmm3, %xmm2 addps %xmm2, %xmm0 @@ -1191,9 +1191,9 @@ #endif ALIGN_4 -.L130: +L(130): testq $2, M - je .L140 + je L(140) #ifdef LN movq K, %rax @@ -1236,10 +1236,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L135 + je L(135) ALIGN_4 -.L132: +L(132): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1285,10 +1285,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L132 + jne L(132) ALIGN_4 -.L135: +L(135): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1297,10 +1297,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): mulps %xmm8, %xmm9 movsd 2 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -1309,10 +1309,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): addps %xmm1, %xmm0 #if defined(LN) || defined(RT) @@ -1441,9 +1441,9 @@ #endif ALIGN_4 -.L140: +L(140): testq $1, M - je .L149 + je L(149) #ifdef LN movq K, %rax @@ -1483,10 +1483,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L145 + je L(145) ALIGN_4 -.L142: +L(142): mulss %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1520,10 +1520,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L142 + jne L(142) ALIGN_4 -.L145: +L(145): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1532,10 +1532,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): mulss %xmm8, %xmm9 movss 1 * SIZE(AO), %xmm8 addss %xmm9, %xmm0 @@ -1544,10 +1544,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): addss %xmm1, %xmm0 addss %xmm3, %xmm2 addss %xmm2, %xmm0 @@ -1632,7 +1632,7 @@ #endif ALIGN_4 -.L149: +L(149): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -1654,9 +1654,9 @@ #endif ALIGN_4 -.L50: +L(50): testq $2, N - je .L100 + je L(100) #ifdef LN movq OFFSET, %rax @@ -1692,10 +1692,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 @@ -1722,10 +1722,10 @@ addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1734,10 +1734,10 @@ #endif andq $3, %rax BRANCH - jle .L60 + jle L(60) ALIGN_4 -.L54: +L(54): movsd 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -1749,10 +1749,10 @@ addq $2 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L54 + jne L(54) ALIGN_4 -.L60: +L(60): #if defined(LT) || defined(RN) movq A, AO #else @@ -1771,10 +1771,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L70 + jle L(70) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $3 + BASE_SHIFT, %rax @@ -1821,10 +1821,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1944,10 +1944,10 @@ addq $64 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1956,10 +1956,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -1976,10 +1976,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2497,12 +2497,12 @@ #endif decq I # i -- - jg .L61 + jg L(61) ALIGN_4 -.L70: +L(70): testq $4, M - je .L80 + je L(80) #ifdef LN movq K, %rax @@ -2545,10 +2545,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2612,10 +2612,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2624,10 +2624,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -2638,10 +2638,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -2889,9 +2889,9 @@ #endif ALIGN_4 -.L80: +L(80): testq $2, M - je .L90 + je L(90) #ifdef LN movq K, %rax @@ -2952,10 +2952,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L85 + je L(85) ALIGN_4 -.L82: +L(82): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -3026,10 +3026,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L85: +L(85): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3038,10 +3038,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -3053,10 +3053,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -3226,9 +3226,9 @@ #endif ALIGN_4 -.L90: +L(90): testq $1, M - je .L99 + je L(99) #ifdef LN movq K, %rax @@ -3270,10 +3270,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -3344,10 +3344,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3356,10 +3356,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -3371,10 +3371,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): addss %xmm2, %xmm0 addss %xmm3, %xmm1 @@ -3500,7 +3500,7 @@ #endif ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -3522,12 +3522,12 @@ #endif ALIGN_4 -.L100: +L(100): movq N, J sarq $2, J # j = (n >> 2) - jle .L999 + jle L(999) -.L01: +L(01): /* Copying to Sub Buffer */ #ifdef LN @@ -3564,10 +3564,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 movaps 8 * SIZE(B), %xmm11 @@ -3615,10 +3615,10 @@ addq $64 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3627,10 +3627,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movaps 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -3646,10 +3646,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -3669,10 +3669,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $3 + BASE_SHIFT, %rax @@ -3724,10 +3724,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 0 * SIZE(BO), %xmm9 @@ -3828,10 +3828,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3840,9 +3840,9 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -3871,10 +3871,10 @@ addq $8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -4537,12 +4537,12 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M - je .L30 + je L(30) #ifdef LN movq K, %rax @@ -4585,10 +4585,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -4702,10 +4702,10 @@ addq $ 32 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4714,10 +4714,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -4734,10 +4734,10 @@ addq $ 4 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -5061,9 +5061,9 @@ #endif ALIGN_4 -.L30: +L(30): testq $2, M - je .L40 + je L(40) #ifdef LN movq K, %rax @@ -5112,10 +5112,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -5234,10 +5234,10 @@ addq $ 16 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -5246,10 +5246,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -5267,10 +5267,10 @@ addq $ 2 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -5515,9 +5515,9 @@ #endif ALIGN_4 -.L40: +L(40): testq $1, M - je .L49 + je L(49) #ifdef LN movq K, %rax @@ -5559,10 +5559,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): mulss %xmm8, %xmm9 addss %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -5681,10 +5681,10 @@ addq $ 8 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -5693,10 +5693,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -5714,10 +5714,10 @@ addq $ 1 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -5918,7 +5918,7 @@ #endif ALIGN_4 -.L49: +L(49): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -5940,11 +5940,11 @@ #endif decq J # j -- - jg .L01 + jg L(01) -.L999: +L(999): movq %rbx, %rsp EMMS movq 0(%rsp), %rbx diff --git a/kernel/x86_64/xdot.S b/kernel/x86_64/xdot.S index c4b4734947..d41846d34d 100644 --- a/kernel/x86_64/xdot.S +++ b/kernel/x86_64/xdot.S @@ -77,7 +77,7 @@ movl STACK_INCY, INCY testl N, N - jle .L88 + jle L(88) sall $ZBASE_SHIFT, INCX sall $ZBASE_SHIFT, INCY @@ -88,16 +88,16 @@ fldz cmpl $2 * SIZE, INCX - jne .L14 + jne L(14) cmpl $2 * SIZE, INCY - jne .L14 + jne L(14) movl N, %eax sarl $1, %eax - jle .L15 + jle L(15) ALIGN_3 -.L16: +L(16): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -146,16 +146,16 @@ addl $4 * SIZE, X addl $4 * SIZE, Y decl %eax - jg .L16 + jg L(16) ALIGN_3 -.L15: +L(15): movl N, %eax andl $1, %eax - jle .L27 + jle L(27) ALIGN_3 -.L22: +L(22): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -174,17 +174,17 @@ FLD 1 * SIZE(Y) fmulp %st, %st(1) faddp %st, %st(4) - jmp .L27 + jmp L(27) ALIGN_3 -.L14: +L(14): movl N, %eax sarl $1, %eax - jle .L30 + jle L(30) ALIGN_3 -.L31: +L(31): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -228,16 +228,16 @@ addl INCY, Y decl %eax - jg .L31 + jg L(31) ALIGN_3 -.L30: +L(30): movl N, %eax andl $1, %eax - jle .L27 + jle L(27) ALIGN_3 -.L37: +L(37): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -258,7 +258,7 @@ faddp %st, %st(4) ALIGN_3 -.L27: +L(27): movl RESULT, %eax #ifndef CONJ @@ -278,7 +278,7 @@ ret ALIGN_3 -.L88: +L(88): movl RESULT, %eax fldz diff --git a/kernel/x86_64/xgemm3m_kernel_2x2.S b/kernel/x86_64/xgemm3m_kernel_2x2.S index 1d0b23c402..1d6c16e081 100644 --- a/kernel/x86_64/xgemm3m_kernel_2x2.S +++ b/kernel/x86_64/xgemm3m_kernel_2x2.S @@ -106,10 +106,10 @@ movq N, %rax sarq $1, %rax movq %rax, J - je .L30 + je L(30) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -123,10 +123,10 @@ movq M, I sarq $1, I - je .L20 + je L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -167,10 +167,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -251,20 +251,20 @@ addq $8 * SIZE,BO decq %rax - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif and $3, %rax - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -287,10 +287,10 @@ addq $2 * SIZE,BO decq %rax - jne .L16 + jne L(16) ALIGN_4 -.L18: +L(18): #ifndef TRMMKERNEL FLD ALPHA_I FLD ALPHA_R @@ -364,16 +364,16 @@ addq $4 * SIZE, CO decq I - jne .L11 + jne L(11) ALIGN_4 -.L20: +L(20): movq M, %rax andq $1, %rax - je .L29 + je L(29) ALIGN_4 -.L21: +L(21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -404,10 +404,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -454,20 +454,20 @@ addq $8 * SIZE,BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif and $3, %rax - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -482,10 +482,10 @@ addq $2 * SIZE,BO decq %rax - jne .L26 + jne L(26) ALIGN_4 -.L28: +L(28): #ifndef TRMMKERNEL FLD ALPHA_I FLD ALPHA_R @@ -534,20 +534,20 @@ addq $1 * SIZE, CO ALIGN_4 -.L29: +L(29): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif movq BO, B decq J - jne .L01 + jne L(01) ALIGN_4 -.L30: +L(30): movq N, %rax testq $1, %rax - je .L999 + je L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -561,10 +561,10 @@ movq M, I sarq $1, I - je .L40 + je L(40) ALIGN_4 -.L31: +L(31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -601,10 +601,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(BO) @@ -647,20 +647,20 @@ addq $4 * SIZE,BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif and $3, %rax - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): FLD -8 * SIZE(BO) FLD -8 * SIZE(AO) @@ -675,10 +675,10 @@ addq $1 * SIZE,BO decq %rax - jne .L36 + jne L(36) ALIGN_4 -.L38: +L(38): #ifndef TRMMKERNEL FLD ALPHA_I FLD ALPHA_R @@ -726,16 +726,16 @@ addq $4 * SIZE, CO decq I - jne .L31 + jne L(31) ALIGN_4 -.L40: +L(40): movq M, %rax andq $1, %rax - je .L49 + je L(49) ALIGN_4 -.L41: +L(41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -765,10 +765,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -795,20 +795,20 @@ addq $4 * SIZE,BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif and $3, %rax - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -819,10 +819,10 @@ addq $1 * SIZE,BO decq %rax - jne .L46 + jne L(46) ALIGN_4 -.L48: +L(48): #ifndef TRMMKERNEL FLD ALPHA_I FLD ALPHA_R @@ -858,7 +858,7 @@ addq $1 * SIZE, CO ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addq $1, KK #endif @@ -866,7 +866,7 @@ movq BO, B ALIGN_4 -.L999: +L(999): EMMS movq 0(%rsp), %rbx diff --git a/kernel/x86_64/xgemm_kernel_1x1.S b/kernel/x86_64/xgemm_kernel_1x1.S index ee67d8d430..9b196717c2 100644 --- a/kernel/x86_64/xgemm_kernel_1x1.S +++ b/kernel/x86_64/xgemm_kernel_1x1.S @@ -123,15 +123,15 @@ salq $ZBASE_SHIFT, LDC cmpq $0, M - jle .L999 + jle L(999) movq N, %rax movq %rax, J testq %rax, %rax - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -145,7 +145,7 @@ movq M, I ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -184,10 +184,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -268,20 +268,20 @@ addq $8 * SIZE,BO decq %rax - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif and $3, %rax - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -304,10 +304,10 @@ addq $2 * SIZE,BO decq %rax - jne .L16 + jne L(16) ALIGN_4 -.L18: +L(18): faddp %st, %st(3) faddp %st, %st(1) @@ -352,7 +352,7 @@ addq $2 * SIZE, CO decq I - jne .L11 + jne L(11) #if defined(TRMMKERNEL) && !defined(LEFT) addq $1, KK @@ -360,10 +360,10 @@ movq BO, B decq J - jne .L01 + jne L(01) ALIGN_4 -.L999: +L(999): EMMS movq 0(%rsp), %rbx diff --git a/kernel/x86_64/xgemv_n.S b/kernel/x86_64/xgemv_n.S index b66f28d586..ede06c7677 100644 --- a/kernel/x86_64/xgemv_n.S +++ b/kernel/x86_64/xgemv_n.S @@ -101,9 +101,9 @@ movq $0, IS test M, M - jle .L79 + jle L(79) test N, N - jle .L79 + jle L(79) movq LDA, %rax imulq $P, %rax # P * lda @@ -114,7 +114,7 @@ salq $ZBASE_SHIFT, LDA ALIGN_2 -.L32: +L(32): movq $P, %rax movq N, MIN_N subq IS, MIN_N @@ -126,17 +126,17 @@ leaq (X,XP, 1), XP cmpq $2 * SIZE, INCX - je .L34 + je L(34) movq BUFFER, XP movq XP, X1 movq MIN_N, I sarq $1, I - jle .L35 + jle L(35) ALIGN_2 -.L36: +L(36): FLD 0 * SIZE(X) FLD 1 * SIZE(X) addq INCX,X # x += incx @@ -151,13 +151,13 @@ addq $4 * SIZE, X1 # xp += 4 decq I - jg .L36 + jg L(36) ALIGN_3 -.L35: +L(35): movq MIN_N, I andq $1, I - jle .L34 + jle L(34) FLD 0 * SIZE(X) FLD 1 * SIZE(X) @@ -167,12 +167,12 @@ ALIGN_3 /* Main Routine */ -.L34: +L(34): movq Y, Y1 # c_offset movq M, J # j = m ALIGN_3 -.L61: +L(61): movq A, A1 # a_offset = a addq $2 * SIZE, A # a++ @@ -186,10 +186,10 @@ movq MIN_N, I sarq $1, I - jle .L64 + jle L(64) ALIGN_3 -.L65: +L(65): FLD 0 * SIZE(A1) # at1 = *(a_offset + 0) fmul %st(1) # at1 *= bt1 faddp %st, %st(2) # ct1 += at1 @@ -241,15 +241,15 @@ addq LDA, A1 # a_offset += lda decq I - jg .L65 + jg L(65) -.L64: +L(64): movq MIN_N, I andq $1, I - jle .L70 + jle L(70) ALIGN_2 -.L71: +L(71): FLD 0 * SIZE(A1) # at1 = *(a_offset + 0) fmul %st(1) # at1 *= bt1 faddp %st, %st(2) # ct1 += at1 @@ -273,7 +273,7 @@ fldz ALIGN_2 -.L70: +L(70): ffreep %st(0) #ifndef XCONJ @@ -315,15 +315,15 @@ addq INCY, Y1 decq J - jg .L61 + jg L(61) -.L60: +L(60): addq PLDA_M, A addq $P, IS cmpq N, IS - jl .L32 + jl L(32) -.L79: +L(79): ffreep %st ffreep %st diff --git a/kernel/x86_64/xgemv_t.S b/kernel/x86_64/xgemv_t.S index d6d37010d1..0844f966a2 100644 --- a/kernel/x86_64/xgemv_t.S +++ b/kernel/x86_64/xgemv_t.S @@ -100,9 +100,9 @@ movq $0, IS test M, M - jle .L79 # goto END + jle L(79) # goto END test N, N - jle .L79 # goto END + jle L(79) # goto END movq N, %rax imulq LDA, %rax @@ -113,7 +113,7 @@ salq $ZBASE_SHIFT, LDA ALIGN_2 -.L32: +L(32): movq $P, %rax movq M, MIN_M subq IS , MIN_M @@ -127,17 +127,17 @@ movq X1, XP cmpq $2 * SIZE, INCX - je .L34 + je L(34) movq BUFFER, X1 movq X1, XP movq MIN_M, I sarq $1, I - jle .L35 + jle L(35) ALIGN_3 -.L36: +L(36): FLD 0 * SIZE(X) FLD 1 * SIZE(X) addq INCX,X # x += incx @@ -152,13 +152,13 @@ addq $4 * SIZE, X1 # xp += 4 decq I - jg .L36 + jg L(36) ALIGN_3 -.L35: +L(35): movq MIN_M, I andq $1,I - jle .L34 + jle L(34) FLD 0 * SIZE(X) FLD 1 * SIZE(X) @@ -169,13 +169,13 @@ /* Main Routine */ -.L34: +L(34): movq Y, Y1 # coffset = y movq N, J ALIGN_2 -.L61: +L(61): movq A, A1 # a_offset = a fldz # ct1 = ZERO fldz # ct1 = ZERO @@ -190,10 +190,10 @@ movq MIN_M, I sarq $1, I - jle .L64 + jle L(64) ALIGN_3 -.L65: +L(65): FLD 0 * SIZE(A1) # at1 = *(a_offset + 0) fmul %st(1) # at1 *= bt1 faddp %st, %st(2) # ct1 += at1 @@ -241,16 +241,16 @@ addq $4 * SIZE, X1 addq $4 * SIZE, A1 decq I - jg .L65 + jg L(65) ALIGN_3 -.L64: +L(64): movq MIN_M, I andq $1, I - jle .L70 + jle L(70) ALIGN_3 -.L71: +L(71): FLD 0 * SIZE(A1) # at1 = *(a_offset + 0) fmul %st(1) # at1 *= bt1 faddp %st, %st(2) # ct1 += at1 @@ -274,7 +274,7 @@ fldz ALIGN_3 -.L70: +L(70): ffreep %st(0) #ifndef XCONJ @@ -316,18 +316,18 @@ addq INCY, Y1 decq J - jg .L61 + jg L(61) ALIGN_3 -.L60: +L(60): addq NLDA, A addq $P, IS cmpq M, IS - jl .L32 + jl L(32) ALIGN_3 -.L79: +L(79): ffreep %st ffreep %st diff --git a/kernel/x86_64/xtrsm_kernel_LT_1x1.S b/kernel/x86_64/xtrsm_kernel_LT_1x1.S index 875206363f..3f08dd0a55 100644 --- a/kernel/x86_64/xtrsm_kernel_LT_1x1.S +++ b/kernel/x86_64/xtrsm_kernel_LT_1x1.S @@ -141,15 +141,15 @@ #endif cmpq $0, M - jle .L999 + jle L(999) movq N, %rax movq %rax, J testq %rax, %rax - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -183,7 +183,7 @@ movq M, I ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -218,10 +218,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -302,10 +302,10 @@ addq $8 * SIZE,BO decq %rax - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -313,10 +313,10 @@ subq KK, %rax #endif and $3, %rax - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -339,10 +339,10 @@ addq $2 * SIZE,BO decq %rax - jne .L16 + jne L(16) ALIGN_4 -.L18: +L(18): faddp %st, %st(3) faddp %st, %st(1) @@ -453,7 +453,7 @@ #endif decq I - jne .L11 + jne L(11) #ifdef LN movq K, %rax @@ -474,10 +474,10 @@ #endif decq J - jne .L01 + jne L(01) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zamax.S b/kernel/x86_64/zamax.S index 5cb2f60198..9c8f576020 100644 --- a/kernel/x86_64/zamax.S +++ b/kernel/x86_64/zamax.S @@ -64,9 +64,9 @@ fldz testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) ffreep %st @@ -77,17 +77,17 @@ faddp %st, %st(1) addq INCX, X decq M - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX - jne .L40 + jne L(40) movq M, I sarq $2, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -135,17 +135,17 @@ addq $8 * SIZE, X decq I - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): movq M, I andq $3, I - jle .L999 + jle L(999) ALIGN_4 -.L21: +L(21): FLD 0 * SIZE(X) fabs FLD 1 * SIZE(X) @@ -158,17 +158,17 @@ addq $2 * SIZE, X decq I - jg .L21 - jmp .L999 + jg L(21) + jmp L(999) ALIGN_4 -.L40: +L(40): movq M, I sarq $2, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): FLD 0 * SIZE(X) fabs FLD 1 * SIZE(X) @@ -214,16 +214,16 @@ ffreep %st decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): movq M, I andq $3, I - jle .L999 + jle L(999) ALIGN_4 -.L61: +L(61): FLD 0 * SIZE(X) fabs FLD 1 * SIZE(X) @@ -236,10 +236,10 @@ addq INCX, X decq I - jg .L61 + jg L(61) ALIGN_4 -.L999: +L(999): ret EPILOGUE diff --git a/kernel/x86_64/zamax_atom.S b/kernel/x86_64/zamax_atom.S index 8b4e144f05..c23b8df848 100644 --- a/kernel/x86_64/zamax_atom.S +++ b/kernel/x86_64/zamax_atom.S @@ -60,10 +60,10 @@ salq $ZBASE_SHIFT, INCX testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) pcmpeqb %xmm15, %xmm15 psrlq $1, %xmm15 @@ -77,16 +77,16 @@ addsd %xmm4, %xmm0 decq M - jle .L999 + jle L(999) movaps %xmm0, %xmm1 cmpq $2 * SIZE, INCX - jne .L20 + jne L(20) movq M, I sarq $2, I - jle .L15 + jle L(15) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -101,10 +101,10 @@ addsd %xmm4, %xmm5 movsd 7 * SIZE(X), %xmm11 decq I - jle .L13 + jle L(13) ALIGN_4 -.L12: +L(12): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -139,10 +139,10 @@ addq $8 * SIZE, X decq I - jg .L12 + jg L(12) ALIGN_4 -.L13: +L(13): andps %xmm15, %xmm6 andps %xmm15, %xmm7 addsd %xmm6, %xmm7 @@ -163,9 +163,9 @@ addq $8 * SIZE, X ALIGN_4 -.L15: +L(15): testq $2, M - jle .L17 + jle L(17) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -185,9 +185,9 @@ maxsd %xmm7, %xmm1 ALIGN_3 -.L17: +L(17): testq $1, M - jle .L998 + jle L(998) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -197,13 +197,13 @@ addsd %xmm4, %xmm5 maxsd %xmm5, %xmm0 - jmp .L998 + jmp L(998) ALIGN_3 -.L20: +L(20): movq M, I sarq $2, I - jle .L25 + jle L(25) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -223,10 +223,10 @@ addq INCX, X decq I - jle .L23 + jle L(23) ALIGN_4 -.L22: +L(22): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -264,10 +264,10 @@ addsd %xmm4, %xmm5 decq I - jg .L22 + jg L(22) ALIGN_4 -.L23: +L(23): andps %xmm15, %xmm6 andps %xmm15, %xmm7 addsd %xmm6, %xmm7 @@ -286,9 +286,9 @@ maxsd %xmm11, %xmm1 ALIGN_4 -.L25: +L(25): testq $2, M - jle .L27 + jle L(27) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -309,9 +309,9 @@ maxsd %xmm7, %xmm1 ALIGN_3 -.L27: +L(27): testq $1, M - jle .L998 + jle L(998) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -323,11 +323,11 @@ maxsd %xmm5, %xmm0 ALIGN_3 -.L998: +L(998): maxsd %xmm1, %xmm0 ALIGN_4 -.L999: +L(999): RESTOREREGISTERS diff --git a/kernel/x86_64/zamax_sse.S b/kernel/x86_64/zamax_sse.S index 5f8a1f1c6d..b3caefd666 100644 --- a/kernel/x86_64/zamax_sse.S +++ b/kernel/x86_64/zamax_sse.S @@ -61,7 +61,7 @@ salq $ZBASE_SHIFT, INCX testq M, M - jle .L999 + jle L(999) pcmpeqb %xmm15, %xmm15 psrld $1, %xmm15 @@ -76,15 +76,15 @@ shufps $0, %xmm0, %xmm0 movaps %xmm0, %xmm1 cmpq $2 * SIZE, INCX - jne .L40 + jne L(40) -.L30: +L(30): movq M, I sarq $3, I - jle .L35 + jle L(35) ALIGN_4 -.L31: +L(31): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -120,15 +120,15 @@ addq $16 * SIZE, X decq I - jg .L31 + jg L(31) ALIGN_4 -.L35: +L(35): andq $7, M - jle .L998 + jle L(998) testq $4, M - je .L36 + je L(36) movsd 0 * SIZE(X), %xmm4 movhps 2 * SIZE(X), %xmm4 @@ -147,9 +147,9 @@ addq $8 * SIZE, X ALIGN_3 -.L36: +L(36): testq $2, M - je .L37 + je L(37) movss 0 * SIZE(X), %xmm4 movss 1 * SIZE(X), %xmm5 @@ -166,9 +166,9 @@ addq $4 * SIZE, X ALIGN_3 -.L37: +L(37): testq $1, M - je .L998 + je L(998) movss 0 * SIZE(X), %xmm4 movss 1 * SIZE(X), %xmm5 @@ -176,17 +176,17 @@ andps %xmm15, %xmm5 addps %xmm5, %xmm4 maxss %xmm4, %xmm0 - jmp .L998 + jmp L(998) ALIGN_4 -.L40: +L(40): movq M, I sarq $3, I - jle .L45 + jle L(45) ALIGN_4 -.L41: +L(41): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -229,15 +229,15 @@ maxps %xmm7, %xmm0 decq I - jg .L41 + jg L(41) ALIGN_4 -.L45: +L(45): andq $7, M - jle .L998 + jle L(998) testq $4, M - je .L46 + je L(46) movsd 0 * SIZE(X), %xmm4 addq INCX, X @@ -258,9 +258,9 @@ maxps %xmm4, %xmm0 ALIGN_3 -.L46: +L(46): testq $2, M - je .L47 + je L(47) movss 0 * SIZE(X), %xmm4 movss 1 * SIZE(X), %xmm5 @@ -278,9 +278,9 @@ maxss %xmm6, %xmm1 ALIGN_3 -.L47: +L(47): testq $1, M - je .L998 + je L(998) movss 0 * SIZE(X), %xmm4 movss 1 * SIZE(X), %xmm5 @@ -288,10 +288,10 @@ andps %xmm15, %xmm5 addps %xmm5, %xmm4 maxss %xmm4, %xmm0 - jmp .L998 + jmp L(998) ALIGN_4 -.L998: +L(998): maxps %xmm1, %xmm0 movaps %xmm0, %xmm1 movhlps %xmm0, %xmm0 @@ -301,7 +301,7 @@ maxss %xmm1, %xmm0 ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/zamax_sse2.S b/kernel/x86_64/zamax_sse2.S index bde290be98..4732f0b5c0 100644 --- a/kernel/x86_64/zamax_sse2.S +++ b/kernel/x86_64/zamax_sse2.S @@ -59,9 +59,9 @@ pxor %xmm0, %xmm0 testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) salq $ZBASE_SHIFT, INCX @@ -81,15 +81,15 @@ movapd %xmm0, %xmm3 cmpq $2 * SIZE, INCX - jne .L40 + jne L(40) -.L30: +L(30): movq M, I sarq $3, I - jle .L35 + jle L(35) ALIGN_4 -.L31: +L(31): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -138,15 +138,15 @@ addq $16 * SIZE, X decq I - jg .L31 + jg L(31) ALIGN_4 -.L35: +L(35): andq $7, M - jle .L998 + jle L(998) testq $4, M - je .L36 + je L(36) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -170,9 +170,9 @@ addq $8 * SIZE, X ALIGN_3 -.L36: +L(36): testq $2, M - je .L37 + je L(37) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -186,9 +186,9 @@ maxpd %xmm4, %xmm0 ALIGN_3 -.L37: +L(37): testq $1, M - je .L998 + je L(998) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -196,17 +196,17 @@ andpd %xmm15, %xmm5 addpd %xmm5, %xmm4 maxsd %xmm4, %xmm2 - jmp .L998 + jmp L(998) ALIGN_4 -.L40: +L(40): movq M, I sarq $3, I - jle .L45 + jle L(45) ALIGN_4 -.L41: +L(41): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -262,15 +262,15 @@ maxpd %xmm6, %xmm3 decq I - jg .L41 + jg L(41) ALIGN_4 -.L45: +L(45): andq $7, M - jle .L998 + jle L(998) testq $4, M - je .L46 + je L(46) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -295,9 +295,9 @@ maxpd %xmm6, %xmm1 ALIGN_3 -.L46: +L(46): testq $2, M - je .L47 + je L(47) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -311,9 +311,9 @@ maxpd %xmm4, %xmm2 ALIGN_3 -.L47: +L(47): testq $1, M - je .L998 + je L(998) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -321,10 +321,10 @@ andpd %xmm15, %xmm5 addpd %xmm5, %xmm4 maxsd %xmm4, %xmm3 - jmp .L998 + jmp L(998) ALIGN_4 -.L998: +L(998): maxpd %xmm1, %xmm0 maxpd %xmm3, %xmm2 maxpd %xmm2, %xmm0 @@ -333,7 +333,7 @@ maxsd %xmm1, %xmm0 ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/zasum.S b/kernel/x86_64/zasum.S index 3460fcea30..7b57747eba 100644 --- a/kernel/x86_64/zasum.S +++ b/kernel/x86_64/zasum.S @@ -56,9 +56,9 @@ fldz testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) salq $ZBASE_SHIFT, INCX @@ -66,14 +66,14 @@ fldz fldz cmpq $SIZE * 2, INCX - jne .L40 + jne L(40) movq M, I sarq $2, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -109,16 +109,16 @@ faddp %st, %st(1) decq I - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): andq $3, M - jle .L998 + jle L(998) ALIGN_4 -.L21: +L(21): FLD 0 * SIZE(X) fabs FLD 1 * SIZE(X) @@ -127,17 +127,17 @@ faddp %st,%st(1) addq $2 * SIZE, X decq M - jg .L21 - jmp .L998 + jg L(21) + jmp L(998) ALIGN_4 -.L40: +L(40): movq M, I sarq $2, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): FLD 0 * SIZE(X) fabs FLD 1 * SIZE(X) @@ -171,16 +171,16 @@ faddp %st, %st(1) decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): andq $3, M - jle .L998 + jle L(998) ALIGN_4 -.L61: +L(61): FLD 0 * SIZE(X) fabs FLD 1 * SIZE(X) @@ -189,16 +189,16 @@ faddp %st,%st(3) faddp %st,%st(1) decq M - jg .L61 + jg L(61) ALIGN_4 -.L998: +L(998): faddp %st,%st(2) faddp %st,%st(1) faddp %st,%st(1) ALIGN_4 -.L999: +L(999): ret EPILOGUE diff --git a/kernel/x86_64/zasum_atom.S b/kernel/x86_64/zasum_atom.S index 888dbbb80c..1f63f5b294 100644 --- a/kernel/x86_64/zasum_atom.S +++ b/kernel/x86_64/zasum_atom.S @@ -55,9 +55,9 @@ xorps %xmm0, %xmm0 testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) xorps %xmm1, %xmm1 xorps %xmm2, %xmm2 @@ -69,12 +69,12 @@ xorps %xmm13, %xmm13 cmpq $2 * SIZE, INCX - jne .L20 + jne L(20) addq M, M testq $SIZE, X - je .L05 + je L(05) movsd (X), %xmm0 addq $SIZE, X @@ -82,12 +82,12 @@ decq M ALIGN_3 -.L05: +L(05): subq $-16 * SIZE, X movq M, I sarq $4, I - jle .L12 + jle L(12) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -100,10 +100,10 @@ movaps -2 * SIZE(X), %xmm11 decq I - jle .L11 + jle L(11) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -162,10 +162,10 @@ subq $-16 * SIZE, X decq I - jg .L10 + jg L(10) ALIGN_4 -.L11: +L(11): andps %xmm15, %xmm4 addsd %xmm13, %xmm3 pshufd $0x4e, %xmm4, %xmm12 @@ -210,12 +210,12 @@ subq $-16 * SIZE, X ALIGN_3 -.L12: +L(12): andq $15, M - jle .L998 + jle L(998) testq $8, M - je .L13 + je L(13) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -241,9 +241,9 @@ addsd %xmm13, %xmm3 ALIGN_3 -.L13: +L(13): testq $4, M - je .L14 + je L(14) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -259,9 +259,9 @@ addsd %xmm13, %xmm3 ALIGN_3 -.L14: +L(14): testq $2, M - je .L15 + je L(15) movaps -16 * SIZE(X), %xmm4 addq $2 * SIZE, X @@ -272,20 +272,20 @@ addsd %xmm5, %xmm3 ALIGN_3 -.L15: +L(15): testq $1, M - je .L998 + je L(998) movsd -16 * SIZE(X), %xmm4 andps %xmm15, %xmm4 addsd %xmm4, %xmm0 - jmp .L998 + jmp L(998) ALIGN_3 -.L20: +L(20): movq M, I sarq $2, I - jle .L25 + jle L(25) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -301,10 +301,10 @@ movsd 1 * SIZE(X), %xmm11 decq I - jle .L23 + jle L(23) ALIGN_4 -.L22: +L(22): andps %xmm15, %xmm4 addq INCX, X addsd %xmm4, %xmm0 @@ -336,10 +336,10 @@ movsd 1 * SIZE(X), %xmm11 decq I - jg .L22 + jg L(22) ALIGN_4 -.L23: +L(23): andps %xmm15, %xmm4 addq INCX, X addsd %xmm4, %xmm0 @@ -360,9 +360,9 @@ addsd %xmm11, %xmm3 ALIGN_3 -.L25: +L(25): testq $2, M - je .L26 + je L(26) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -381,9 +381,9 @@ addsd %xmm7, %xmm3 ALIGN_3 -.L26: +L(26): testq $1, M - je .L998 + je L(998) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -396,13 +396,13 @@ addsd %xmm5, %xmm1 ALIGN_3 -.L998: +L(998): addsd %xmm1, %xmm0 addsd %xmm3, %xmm2 addsd %xmm2, %xmm0 ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/zasum_sse.S b/kernel/x86_64/zasum_sse.S index 44d6da5617..3ce7ee275b 100644 --- a/kernel/x86_64/zasum_sse.S +++ b/kernel/x86_64/zasum_sse.S @@ -54,9 +54,9 @@ pxor %xmm0, %xmm0 testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) pxor %xmm1, %xmm1 pxor %xmm2, %xmm2 @@ -68,26 +68,26 @@ salq $ZBASE_SHIFT, INCX cmpq $2 * SIZE, INCX - jne .L100 + jne L(100) subq $-32 * SIZE, X addq M, M cmpq $3, M - jle .L18 + jle L(18) testq $4, X - je .L05 + je L(05) movss -32 * SIZE(X), %xmm0 andps %xmm15, %xmm0 addq $SIZE, X decq M - jle .L998 + jle L(998) ALIGN_3 -.L05: +L(05): testq $8, X - je .L10 + je L(10) #ifdef movsd xorps %xmm1, %xmm1 @@ -96,13 +96,13 @@ andps %xmm15, %xmm1 addq $2 * SIZE, X subq $2, M - jle .L998 + jle L(998) ALIGN_3 -.L10: +L(10): movq M, I sarq $5, I - jle .L14 + jle L(14) movaps -32 * SIZE(X), %xmm4 movaps -28 * SIZE(X), %xmm5 @@ -114,10 +114,10 @@ movaps -8 * SIZE(X), %xmm10 movaps -4 * SIZE(X), %xmm11 decq I - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -160,10 +160,10 @@ subq $-32 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): andps %xmm15, %xmm4 addps %xmm4, %xmm0 andps %xmm15, %xmm5 @@ -187,13 +187,13 @@ addq $32 * SIZE, X ALIGN_3 -.L14: +L(14): testq $31, M - jle .L998 + jle L(998) -.L15: +L(15): testq $16, M - je .L16 + je L(16) movaps -32 * SIZE(X), %xmm4 andps %xmm15, %xmm4 @@ -214,9 +214,9 @@ addq $16 * SIZE, X ALIGN_3 -.L16: +L(16): testq $8, M - je .L17 + je L(17) movaps -32 * SIZE(X), %xmm4 andps %xmm15, %xmm4 @@ -229,9 +229,9 @@ addq $8 * SIZE, X ALIGN_3 -.L17: +L(17): testq $4, M - je .L18 + je L(18) movaps -32 * SIZE(X), %xmm6 andps %xmm15, %xmm6 @@ -239,9 +239,9 @@ addq $4 * SIZE, X ALIGN_3 -.L18: +L(18): testq $2, M - je .L19 + je L(19) #ifdef movsd xorps %xmm7, %xmm7 @@ -252,23 +252,23 @@ addq $2 * SIZE, X ALIGN_3 -.L19: +L(19): testq $1, M - je .L998 + je L(998) movss -32 * SIZE(X), %xmm6 andps %xmm15, %xmm6 addps %xmm6, %xmm2 - jmp .L998 + jmp L(998) ALIGN_4 -.L100: +L(100): movq M, I sarq $2, I - jle .L105 + jle L(105) ALIGN_4 -.L101: +L(101): movsd (X), %xmm4 addq INCX, X movhps (X), %xmm4 @@ -286,27 +286,27 @@ addps %xmm5, %xmm1 decq I - jg .L101 + jg L(101) ALIGN_4 -.L105: +L(105): #ifdef movsd xorps %xmm4, %xmm4 #endif andq $3, M - jle .L998 + jle L(998) ALIGN_4 -.L106: +L(106): movsd (X), %xmm4 andps %xmm15, %xmm4 addps %xmm4, %xmm0 addq INCX, X decq M - jg .L106 + jg L(106) ALIGN_4 -.L998: +L(998): addps %xmm1, %xmm0 addps %xmm3, %xmm2 addps %xmm2, %xmm0 @@ -324,7 +324,7 @@ #endif ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/zasum_sse2.S b/kernel/x86_64/zasum_sse2.S index d1e076c311..d8e424cdb5 100644 --- a/kernel/x86_64/zasum_sse2.S +++ b/kernel/x86_64/zasum_sse2.S @@ -54,9 +54,9 @@ xorps %xmm0, %xmm0 testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) xorps %xmm1, %xmm1 xorps %xmm2, %xmm2 @@ -68,13 +68,13 @@ salq $ZBASE_SHIFT, INCX cmpq $2 * SIZE, INCX - jne .L40 + jne L(40) subq $-16 * SIZE, X addq M, M testq $SIZE, X - je .L05 + je L(05) #ifdef movsd xorps %xmm0, %xmm0 @@ -84,13 +84,13 @@ andps %xmm15, %xmm0 subq $1, M - jle .L999 + jle L(999) ALIGN_3 -.L05: +L(05): movq M, I sarq $4, I - jle .L20 + jle L(20) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -103,10 +103,10 @@ movaps -2 * SIZE(X), %xmm11 decq I - jle .L11 + jle L(11) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -149,10 +149,10 @@ subq $-16 * SIZE, X decq I - jg .L10 + jg L(10) ALIGN_4 -.L11: +L(11): andps %xmm15, %xmm4 andps %xmm15, %xmm5 andps %xmm15, %xmm6 @@ -176,12 +176,12 @@ subq $-16 * SIZE, X ALIGN_3 -.L20: +L(20): andq $15, M - jle .L998 + jle L(998) testq $8, M - je .L21 + je L(21) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -200,9 +200,9 @@ addq $8 * SIZE, X ALIGN_3 -.L21: +L(21): testq $4, M - je .L22 + je L(22) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -215,18 +215,18 @@ addq $4 * SIZE, X ALIGN_3 -.L22: +L(22): testq $2, M - je .L23 + je L(23) movaps -16 * SIZE(X), %xmm6 andps %xmm15, %xmm6 addpd %xmm6, %xmm3 addq $2 * SIZE, X -.L23: +L(23): testq $1, M - je .L998 + je L(998) #ifdef movsd xorps %xmm4, %xmm4 @@ -234,17 +234,17 @@ movsd -16 * SIZE(X), %xmm4 andps %xmm15, %xmm4 addsd %xmm4, %xmm0 - jmp .L998 + jmp L(998) ALIGN_3 -.L40: +L(40): movq M, I sarq $2, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) prefetcht0 PREFETCHSIZE * SIZE(X) #endif @@ -278,26 +278,26 @@ addpd %xmm7, %xmm3 decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): andq $3, M - jle .L998 + jle L(998) ALIGN_4 -.L61: +L(61): movsd 0 * SIZE(X), %xmm4 movhpd 1 * SIZE(X), %xmm4 andpd %xmm15, %xmm4 addpd %xmm4, %xmm0 addq INCX, X decq M - jg .L61 + jg L(61) ALIGN_4 -.L998: +L(998): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -310,7 +310,7 @@ #endif ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/zaxpy.S b/kernel/x86_64/zaxpy.S index 1758ca9431..e0f800db05 100644 --- a/kernel/x86_64/zaxpy.S +++ b/kernel/x86_64/zaxpy.S @@ -70,19 +70,19 @@ salq $ZBASE_SHIFT, INCY testq M, M - jle .L40 + jle L(40) cmpq $2 * SIZE, INCX - jne .L14 + jne L(14) cmpq $2 * SIZE, INCY - jne .L14 + jne L(14) movq M, %rax sarq $2, %rax - jle .L15 + jle L(15) ALIGN_3 -.L16: +L(16): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -166,16 +166,16 @@ addq $8 * SIZE, X addq $8 * SIZE, Y decq %rax - jg .L16 + jg L(16) ALIGN_3 -.L15: +L(15): movq M, %rax andq $3, %rax - jle .L40 + jle L(40) ALIGN_3 -.L22: +L(22): FLD 0 * SIZE(X) fmul %st(1), %st FLD 1 * SIZE(X) @@ -197,17 +197,17 @@ addq $2 * SIZE, X addq $2 * SIZE, Y decq %rax - jg .L22 - jmp .L40 + jg L(22) + jmp L(40) ALIGN_3 -.L14: +L(14): movq M, %rax sarq $2, %rax - jle .L28 + jle L(28) ALIGN_3 -.L29: +L(29): FLD 0 * SIZE(X) fmul %st(1), %st FLD 1 * SIZE(X) @@ -293,16 +293,16 @@ addq INCY, Y decq %rax - jg .L29 + jg L(29) ALIGN_3 -.L28: +L(28): movq M, %rax andq $3, %rax - jle .L40 + jle L(40) ALIGN_3 -.L35: +L(35): FLD 0 * SIZE(X) fmul %st(1), %st FLD 1 * SIZE(X) @@ -325,10 +325,10 @@ addq INCY, Y decq %rax - jg .L35 + jg L(35) ALIGN_3 -.L40: +L(40): ffreep %st(0) ffreep %st(0) ret diff --git a/kernel/x86_64/zaxpy_atom.S b/kernel/x86_64/zaxpy_atom.S index 2fe2756fb9..8073c3c5fd 100644 --- a/kernel/x86_64/zaxpy_atom.S +++ b/kernel/x86_64/zaxpy_atom.S @@ -94,16 +94,16 @@ movaps %xmm1, ALPHA_I testq M, M - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX - jne .L20 + jne L(20) cmpq $2 * SIZE, INCY - jne .L20 + jne L(20) movq M, %rax sarq $2, %rax - jle .L15 + jle L(15) movsd 0 * SIZE(X), %xmm0 movsd 1 * SIZE(X), %xmm1 @@ -133,10 +133,10 @@ movsd 5 * SIZE(X), %xmm1 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -228,10 +228,10 @@ addq $8 * SIZE, X addq $8 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): mulsd ALPHA_I, %xmm7 movsd 4 * SIZE(Y), %xmm12 ADD1 %xmm3, %xmm8 @@ -299,10 +299,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L15: +L(15): movq M, %rax andq $2, %rax - jle .L17 + jle L(17) movsd 0 * SIZE(X), %xmm0 movsd 1 * SIZE(X), %xmm1 @@ -348,10 +348,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L17: +L(17): movq M, %rax andq $1, %rax - jle .L999 + jle L(999) movsd 0 * SIZE(X), %xmm0 movsd 1 * SIZE(X), %xmm1 @@ -372,15 +372,15 @@ movsd %xmm8, 0 * SIZE(Y) movsd %xmm9, 1 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L20: +L(20): movq Y, YY movq M, %rax sarq $2, %rax - jle .L25 + jle L(25) movsd 0 * SIZE(X), %xmm0 movsd 1 * SIZE(X), %xmm1 @@ -415,10 +415,10 @@ addq INCX, X decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): mulsd ALPHA_I, %xmm7 movsd 0 * SIZE(Y), %xmm12 ADD1 %xmm3, %xmm8 @@ -512,10 +512,10 @@ addq INCX, X decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): mulsd ALPHA_I, %xmm7 movsd 0 * SIZE(Y), %xmm12 ADD1 %xmm3, %xmm8 @@ -587,10 +587,10 @@ addq INCY, YY ALIGN_3 -.L25: +L(25): movq M, %rax andq $2, %rax - jle .L27 + jle L(27) movsd 0 * SIZE(X), %xmm0 movsd 1 * SIZE(X), %xmm1 @@ -639,10 +639,10 @@ addq INCY, YY ALIGN_3 -.L27: +L(27): movq M, %rax andq $1, %rax - jle .L999 + jle L(999) movsd 0 * SIZE(X), %xmm0 movsd 1 * SIZE(X), %xmm1 @@ -665,7 +665,7 @@ movsd %xmm9, 1 * SIZE(YY) ALIGN_3 -.L999: +L(999): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/zaxpy_sse.S b/kernel/x86_64/zaxpy_sse.S index 0a12e244a5..f83d7d0982 100644 --- a/kernel/x86_64/zaxpy_sse.S +++ b/kernel/x86_64/zaxpy_sse.S @@ -81,12 +81,12 @@ salq $ZBASE_SHIFT, INCY testq M, M - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX - jne .L100 + jne L(100) cmpq $2 * SIZE, INCY - jne .L100 + jne L(100) pcmpeqb %xmm7, %xmm7 psllq $63, %xmm7 @@ -105,7 +105,7 @@ subq $-32 * SIZE, Y testq $2 * SIZE, Y - je .L10 + je L(10) movsd -32 * SIZE(X), %xmm0 movsd -32 * SIZE(Y), %xmm1 @@ -122,19 +122,19 @@ addq $2 * SIZE, X addq $2 * SIZE, Y decq M - jle .L999 + jle L(999) ALIGN_2 -.L10: +L(10): testq $SIZE, Y - jne .L50 + jne L(50) testq $3 * SIZE, X - jne .L20 + jne L(20) movq M, %rax sarq $4, %rax - jle .L15 + jle L(15) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -142,10 +142,10 @@ movaps -20 * SIZE(X), %xmm3 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): movaps -16 * SIZE(X), %xmm4 movaps -12 * SIZE(X), %xmm5 @@ -233,10 +233,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movaps -16 * SIZE(X), %xmm4 movaps -12 * SIZE(X), %xmm5 @@ -303,9 +303,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $8, M - jle .L16 + jle L(16) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -345,9 +345,9 @@ addq $16 * SIZE, Y ALIGN_2 -.L16: +L(16): testq $4, M - jle .L17 + jle L(17) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -370,9 +370,9 @@ addq $8 * SIZE, Y ALIGN_2 -.L17: +L(17): testq $2, M - jle .L18 + jle L(18) movaps -32 * SIZE(X), %xmm0 @@ -388,9 +388,9 @@ addq $4 * SIZE, Y ALIGN_2 -.L18: +L(18): testq $1, M - jle .L999 + jle L(999) movsd -32 * SIZE(X), %xmm0 @@ -402,14 +402,14 @@ addps %xmm1, %xmm0 addps %xmm8, %xmm0 movlps %xmm0, -32 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L20: +L(20): #ifdef ALIGNED_ACCESS testq $2 * SIZE, X - jne .L30 + jne L(30) subq $1 * SIZE, X @@ -417,7 +417,7 @@ movq M, %rax sarq $4, %rax - jle .L25 + jle L(25) movaps -28 * SIZE(X), %xmm1 movaps -24 * SIZE(X), %xmm2 @@ -425,10 +425,10 @@ movaps -16 * SIZE(X), %xmm4 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): movaps -12 * SIZE(X), %xmm5 movaps -8 * SIZE(X), %xmm6 @@ -532,10 +532,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): movaps -12 * SIZE(X), %xmm5 movaps -8 * SIZE(X), %xmm6 @@ -618,9 +618,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $8, M - jle .L26 + jle L(26) movaps -28 * SIZE(X), %xmm1 movaps -24 * SIZE(X), %xmm2 @@ -668,9 +668,9 @@ addq $16 * SIZE, Y ALIGN_2 -.L26: +L(26): testq $4, M - jle .L27 + jle L(27) movaps -28 * SIZE(X), %xmm1 movaps -24 * SIZE(X), %xmm2 @@ -699,9 +699,9 @@ addq $8 * SIZE, Y ALIGN_2 -.L27: +L(27): testq $2, M - jle .L28 + jle L(28) movaps -28 * SIZE(X), %xmm1 @@ -721,9 +721,9 @@ addq $4 * SIZE, Y ALIGN_2 -.L28: +L(28): testq $1, M - jle .L999 + jle L(999) pshufd $0x06, %xmm0, %xmm8 pshufd $0x09, %xmm0, %xmm0 @@ -736,17 +736,17 @@ movlps %xmm0, -32 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L30: +L(30): testq $1 * SIZE, X - jne .L40 + jne L(40) #endif movq M, %rax sarq $4, %rax - jle .L35 + jle L(35) movsd -32 * SIZE(X), %xmm0 movhps -30 * SIZE(X), %xmm0 @@ -758,10 +758,10 @@ movhps -18 * SIZE(X), %xmm3 decq %rax - jle .L32 + jle L(32) ALIGN_3 -.L31: +L(31): movsd -16 * SIZE(X), %xmm4 movhps -14 * SIZE(X), %xmm4 @@ -861,10 +861,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L32: +L(32): movsd -16 * SIZE(X), %xmm4 movhps -14 * SIZE(X), %xmm4 @@ -937,9 +937,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L35: +L(35): testq $8, M - jle .L36 + jle L(36) movsd -32 * SIZE(X), %xmm0 movhps -30 * SIZE(X), %xmm0 @@ -983,9 +983,9 @@ addq $16 * SIZE, Y ALIGN_2 -.L36: +L(36): testq $4, M - jle .L37 + jle L(37) movsd -32 * SIZE(X), %xmm0 movhps -30 * SIZE(X), %xmm0 @@ -1010,9 +1010,9 @@ addq $8 * SIZE, Y ALIGN_2 -.L37: +L(37): testq $2, M - jle .L38 + jle L(38) movsd -32 * SIZE(X), %xmm0 movhps -30 * SIZE(X), %xmm0 @@ -1029,9 +1029,9 @@ addq $4 * SIZE, Y ALIGN_2 -.L38: +L(38): testq $1, M - jle .L999 + jle L(999) movsd -32 * SIZE(X), %xmm0 @@ -1044,19 +1044,19 @@ addps %xmm1, %xmm0 addps %xmm8, %xmm0 movlps %xmm0, -32 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 #ifdef ALIGNED_ACCESS -.L40: +L(40): subq $3 * SIZE, X movaps -32 * SIZE(X), %xmm0 movq M, %rax sarq $4, %rax - jle .L45 + jle L(45) movaps -28 * SIZE(X), %xmm1 movaps -24 * SIZE(X), %xmm2 @@ -1064,10 +1064,10 @@ movaps -16 * SIZE(X), %xmm4 decq %rax - jle .L42 + jle L(42) ALIGN_3 -.L41: +L(41): movaps -12 * SIZE(X), %xmm5 movaps -8 * SIZE(X), %xmm6 @@ -1171,10 +1171,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L42: +L(42): movaps -12 * SIZE(X), %xmm5 movaps -8 * SIZE(X), %xmm6 @@ -1257,9 +1257,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L45: +L(45): testq $8, M - jle .L46 + jle L(46) movaps -28 * SIZE(X), %xmm1 movaps -24 * SIZE(X), %xmm2 @@ -1307,9 +1307,9 @@ addq $16 * SIZE, Y ALIGN_2 -.L46: +L(46): testq $4, M - jle .L47 + jle L(47) movaps -28 * SIZE(X), %xmm1 movaps -24 * SIZE(X), %xmm2 @@ -1338,9 +1338,9 @@ addq $8 * SIZE, Y ALIGN_2 -.L47: +L(47): testq $2, M - jle .L48 + jle L(48) movaps -28 * SIZE(X), %xmm1 @@ -1360,9 +1360,9 @@ addq $4 * SIZE, Y ALIGN_2 -.L48: +L(48): testq $1, M - jle .L999 + jle L(999) movaps -28 * SIZE(X), %xmm1 movsd -32 * SIZE(Y), %xmm2 @@ -1377,21 +1377,21 @@ addps %xmm2, %xmm0 movlps %xmm0, -32 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 #endif -.L50: +L(50): xorps %xmm0, %xmm0 subq $1 * SIZE, Y testq $3 * SIZE, X - jne .L60 + jne L(60) movq M, %rax sarq $4, %rax - jle .L55 + jle L(55) movaps -32 * SIZE(X), %xmm1 movaps -28 * SIZE(X), %xmm2 @@ -1399,10 +1399,10 @@ movaps -20 * SIZE(X), %xmm4 decq %rax - jle .L52 + jle L(52) ALIGN_3 -.L51: +L(51): movaps -16 * SIZE(X), %xmm5 movaps -12 * SIZE(X), %xmm6 @@ -1506,10 +1506,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L51 + jg L(51) ALIGN_3 -.L52: +L(52): movaps -16 * SIZE(X), %xmm5 movaps -12 * SIZE(X), %xmm6 @@ -1592,9 +1592,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L55: +L(55): testq $8, M - jle .L56 + jle L(56) movaps -32 * SIZE(X), %xmm1 movaps -28 * SIZE(X), %xmm2 @@ -1642,9 +1642,9 @@ addq $16 * SIZE, Y ALIGN_2 -.L56: +L(56): testq $4, M - jle .L57 + jle L(57) movaps -32 * SIZE(X), %xmm1 movaps -28 * SIZE(X), %xmm2 @@ -1673,9 +1673,9 @@ addq $8 * SIZE, Y ALIGN_2 -.L57: +L(57): testq $2, M - jle .L58 + jle L(58) movaps -32 * SIZE(X), %xmm1 @@ -1694,9 +1694,9 @@ addq $4 * SIZE, Y ALIGN_2 -.L58: +L(58): testq $1, M - jle .L59 + jle L(59) #ifdef movsd xorps %xmm1, %xmm1 @@ -1712,22 +1712,22 @@ addps -32 * SIZE(Y), %xmm0 movaps %xmm0, -32 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L59: +L(59): shufps $0x93, %xmm0, %xmm0 addss -32 * SIZE(Y), %xmm0 movss %xmm0, -32 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L60: +L(60): #ifdef ALIGNED_ACCESS testq $2 * SIZE, X - jne .L70 + jne L(70) subq $1 * SIZE, X @@ -1735,16 +1735,16 @@ movq M, %rax sarq $4, %rax - jle .L65 + jle L(65) movaps -28 * SIZE(X), %xmm2 movaps -24 * SIZE(X), %xmm3 decq %rax - jle .L62 + jle L(62) ALIGN_3 -.L61: +L(61): movaps -20 * SIZE(X), %xmm4 movaps -16 * SIZE(X), %xmm5 @@ -1865,10 +1865,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L61 + jg L(61) ALIGN_3 -.L62: +L(62): movaps -20 * SIZE(X), %xmm4 movaps -16 * SIZE(X), %xmm5 @@ -1970,9 +1970,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L65: +L(65): testq $8, M - jle .L66 + jle L(66) movaps -28 * SIZE(X), %xmm2 movaps -24 * SIZE(X), %xmm3 @@ -2028,9 +2028,9 @@ addq $16 * SIZE, Y ALIGN_2 -.L66: +L(66): testq $4, M - jle .L67 + jle L(67) movaps -28 * SIZE(X), %xmm2 movaps -24 * SIZE(X), %xmm3 @@ -2064,9 +2064,9 @@ addq $8 * SIZE, Y ALIGN_2 -.L67: +L(67): testq $2, M - jle .L68 + jle L(68) movaps -28 * SIZE(X), %xmm2 @@ -2088,9 +2088,9 @@ addq $4 * SIZE, Y ALIGN_2 -.L68: +L(68): testq $1, M - jle .L69 + jle L(69) movaps -28 * SIZE(X), %xmm2 @@ -2107,24 +2107,24 @@ movlps %xmm0, -32 * SIZE(Y) movhlps %xmm0, %xmm0 movss %xmm0, -30 * SIZE(Y) - jmp .L999 + jmp L(999) -.L69: +L(69): shufps $0x93, %xmm0, %xmm0 addss -32 * SIZE(Y), %xmm0 movss %xmm0, -32 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L70: +L(70): testq $1 * SIZE, X - jne .L80 + jne L(80) #endif movq M, %rax sarq $4, %rax - jle .L75 + jle L(75) movsd -32 * SIZE(X), %xmm1 movhps -30 * SIZE(X), %xmm1 @@ -2136,10 +2136,10 @@ movhps -18 * SIZE(X), %xmm4 decq %rax - jle .L72 + jle L(72) ALIGN_3 -.L71: +L(71): movsd -16 * SIZE(X), %xmm5 movhps -14 * SIZE(X), %xmm5 movsd -12 * SIZE(X), %xmm6 @@ -2251,10 +2251,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L71 + jg L(71) ALIGN_3 -.L72: +L(72): movsd -16 * SIZE(X), %xmm5 movhps -14 * SIZE(X), %xmm5 movsd -12 * SIZE(X), %xmm6 @@ -2341,9 +2341,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L75: +L(75): testq $8, M - jle .L76 + jle L(76) movsd -32 * SIZE(X), %xmm1 movhps -30 * SIZE(X), %xmm1 @@ -2395,9 +2395,9 @@ addq $16 * SIZE, Y ALIGN_2 -.L76: +L(76): testq $4, M - jle .L77 + jle L(77) movsd -32 * SIZE(X), %xmm1 movhps -30 * SIZE(X), %xmm1 @@ -2429,9 +2429,9 @@ addq $8 * SIZE, Y ALIGN_2 -.L77: +L(77): testq $2, M - jle .L78 + jle L(78) movsd -32 * SIZE(X), %xmm1 movhps -30 * SIZE(X), %xmm1 @@ -2451,9 +2451,9 @@ addq $4 * SIZE, Y ALIGN_2 -.L78: +L(78): testq $1, M - jle .L79 + jle L(79) #ifdef movsd xorps %xmm1, %xmm1 @@ -2469,36 +2469,36 @@ addps -32 * SIZE(Y), %xmm0 movaps %xmm0, -32 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L79: +L(79): shufps $0x93, %xmm0, %xmm0 addss -32 * SIZE(Y), %xmm0 movss %xmm0, -32 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 #ifdef ALIGNED_ACCESS -.L80: +L(80): subq $3 * SIZE, X movaps -32 * SIZE(X), %xmm1 movq M, %rax sarq $4, %rax - jle .L85 + jle L(85) movaps -28 * SIZE(X), %xmm2 movaps -24 * SIZE(X), %xmm3 decq %rax - jle .L82 + jle L(82) ALIGN_3 -.L81: +L(81): movaps -20 * SIZE(X), %xmm4 movaps -16 * SIZE(X), %xmm5 @@ -2618,10 +2618,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L81 + jg L(81) ALIGN_3 -.L82: +L(82): movaps -20 * SIZE(X), %xmm4 movaps -16 * SIZE(X), %xmm5 @@ -2723,9 +2723,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L85: +L(85): testq $8, M - jle .L86 + jle L(86) movaps -28 * SIZE(X), %xmm2 movaps -24 * SIZE(X), %xmm3 @@ -2781,9 +2781,9 @@ addq $16 * SIZE, Y ALIGN_2 -.L86: +L(86): testq $4, M - jle .L87 + jle L(87) movaps -28 * SIZE(X), %xmm2 movaps -24 * SIZE(X), %xmm3 @@ -2817,9 +2817,9 @@ addq $8 * SIZE, Y ALIGN_2 -.L87: +L(87): testq $2, M - jle .L88 + jle L(88) movaps -28 * SIZE(X), %xmm2 @@ -2841,9 +2841,9 @@ addq $4 * SIZE, Y ALIGN_2 -.L88: +L(88): testq $1, M - jle .L89 + jle L(89) movaps -28 * SIZE(X), %xmm2 @@ -2860,18 +2860,18 @@ movlps %xmm0, -32 * SIZE(Y) movhlps %xmm0, %xmm0 movss %xmm0, -30 * SIZE(Y) - jmp .L999 + jmp L(999) -.L89: +L(89): shufps $0x93, %xmm0, %xmm0 addss -32 * SIZE(Y), %xmm0 movss %xmm0, -32 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 #endif -.L100: +L(100): #ifndef CONJ pshufd $0, %xmm0, %xmm14 pshufd $0, %xmm1, %xmm15 @@ -2895,18 +2895,18 @@ //If incx==0 || incy==0, avoid unloop and jump to end. cmpq $0, INCX - je .L200 + je L(200) cmpq $0, INCY - je .L200 + je L(200) movq Y, YY movq M, %rax sarq $3, %rax - jle .L105 + jle L(105) ALIGN_3 -.L102: +L(102): movsd (X), %xmm0 addq INCX, X movhps (X), %xmm0 @@ -2997,12 +2997,12 @@ addq INCY, YY decq %rax - jg .L102 + jg L(102) ALIGN_3 -.L105: +L(105): testq $4, M - jle .L106 + jle L(106) movsd (X), %xmm0 addq INCX, X @@ -3054,9 +3054,9 @@ addq INCY, YY ALIGN_3 -.L106: +L(106): testq $2, M - jle .L107 + jle L(107) movsd (X), %xmm0 addq INCX, X @@ -3088,9 +3088,9 @@ addq INCY, YY ALIGN_3 -.L107: +L(107): testq $1, M - jle .L999 + jle L(999) movsd (X), %xmm0 @@ -3111,16 +3111,16 @@ addps %xmm1, %xmm8 movsd %xmm8, (Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L200: +L(200): movq M, %rax cmpq $0, %rax - jle .L999 + jle L(999) ALIGN_3 -.L201: +L(201): movsd (X), %xmm0 addq INCX, X @@ -3144,10 +3144,10 @@ addq INCY, Y decq %rax - jg .L201 + jg L(201) ALIGN_3 -.L999: +L(999): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/zaxpy_sse2.S b/kernel/x86_64/zaxpy_sse2.S index 3776c8910d..2d185a65b5 100644 --- a/kernel/x86_64/zaxpy_sse2.S +++ b/kernel/x86_64/zaxpy_sse2.S @@ -90,12 +90,12 @@ salq $ZBASE_SHIFT, INCY testq M, M - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX - jne .L50 + jne L(50) cmpq $2 * SIZE, INCY - jne .L50 + jne L(50) subq $-16 * SIZE, X subq $-16 * SIZE, Y @@ -120,14 +120,14 @@ #endif testq $SIZE, Y - jne .L30 + jne L(30) testq $SIZE, X - jne .L20 + jne L(20) movq M, %rax sarq $3, %rax - jle .L15 + jle L(15) movaps -16 * SIZE(X), %xmm0 movaps -14 * SIZE(X), %xmm1 @@ -135,10 +135,10 @@ movaps -10 * SIZE(X), %xmm3 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): movaps -8 * SIZE(X), %xmm4 movaps -6 * SIZE(X), %xmm5 @@ -266,10 +266,10 @@ subq $-16 * SIZE, X subq $-16 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movaps -8 * SIZE(X), %xmm4 movaps -6 * SIZE(X), %xmm5 @@ -336,10 +336,10 @@ subq $-16 * SIZE, Y ALIGN_3 -.L15: +L(15): movq M, %rax andq $4, %rax - jle .L16 + jle L(16) movaps -16 * SIZE(X), %xmm0 movaps -14 * SIZE(X), %xmm1 @@ -378,10 +378,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L16: +L(16): movq M, %rax andq $2, %rax - jle .L17 + jle L(17) movaps -16 * SIZE(X), %xmm0 movaps -14 * SIZE(X), %xmm1 @@ -404,10 +404,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L17: +L(17): movq M, %rax andq $1, %rax - jle .L999 + jle L(999) movaps -16 * SIZE(X), %xmm0 @@ -417,13 +417,13 @@ addpd -16 * SIZE(Y), %xmm0 addpd %xmm8, %xmm0 movaps %xmm0, -16 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L20: +L(20): movq M, %rax sarq $3, %rax - jle .L25 + jle L(25) movsd -16 * SIZE(X), %xmm0 movhps -15 * SIZE(X), %xmm0 @@ -435,10 +435,10 @@ movhps -9 * SIZE(X), %xmm3 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): movsd -8 * SIZE(X), %xmm4 movhps -7 * SIZE(X), %xmm4 @@ -538,10 +538,10 @@ subq $-16 * SIZE, X subq $-16 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): movsd -8 * SIZE(X), %xmm4 movhps -7 * SIZE(X), %xmm4 @@ -614,10 +614,10 @@ subq $-16 * SIZE, Y ALIGN_3 -.L25: +L(25): movq M, %rax andq $4, %rax - jle .L26 + jle L(26) movsd -16 * SIZE(X), %xmm0 movhps -15 * SIZE(X), %xmm0 @@ -661,10 +661,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L26: +L(26): movq M, %rax andq $2, %rax - jle .L27 + jle L(27) movsd -16 * SIZE(X), %xmm0 movhps -15 * SIZE(X), %xmm0 @@ -690,10 +690,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L27: +L(27): movq M, %rax andq $1, %rax - jle .L999 + jle L(999) movsd -16 * SIZE(X), %xmm0 movhps -15 * SIZE(X), %xmm0 @@ -704,12 +704,12 @@ addpd -16 * SIZE(Y), %xmm0 addpd %xmm8, %xmm0 movaps %xmm0, -16 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L30: +L(30): testq $SIZE, X - jne .L40 + jne L(40) movaps -16 * SIZE(X), %xmm1 @@ -730,21 +730,21 @@ addq $2 * SIZE, X addq $1 * SIZE, Y decq M - jle .L39 + jle L(39) movq M, %rax sarq $3, %rax - jle .L35 + jle L(35) movaps -16 * SIZE(X), %xmm1 movaps -14 * SIZE(X), %xmm2 movaps -12 * SIZE(X), %xmm3 decq %rax - jle .L32 + jle L(32) ALIGN_3 -.L31: +L(31): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -844,10 +844,10 @@ subq $-16 * SIZE, X subq $-16 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L32: +L(32): pshufd $0x4e, %xmm1, %xmm8 mulpd ALPHA_R, %xmm1 mulpd ALPHA_I, %xmm8 @@ -929,10 +929,10 @@ subq $-16 * SIZE, Y ALIGN_3 -.L35: +L(35): movq M, %rax andq $4, %rax - jle .L36 + jle L(36) movaps -16 * SIZE(X), %xmm1 movaps -14 * SIZE(X), %xmm2 @@ -980,10 +980,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L36: +L(36): movq M, %rax andq $2, %rax - jle .L37 + jle L(37) movaps -16 * SIZE(X), %xmm1 movaps -14 * SIZE(X), %xmm2 @@ -1011,10 +1011,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L37: +L(37): movq M, %rax andq $1, %rax - jle .L39 + jle L(39) movaps -16 * SIZE(X), %xmm1 @@ -1032,15 +1032,15 @@ addq $2 * SIZE, Y ALIGN_3 -.L39: +L(39): SHUFPD_1 %xmm0, %xmm0 addsd -16 * SIZE(Y), %xmm0 movlps %xmm0, -16 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L40: +L(40): movsd -16 * SIZE(X), %xmm1 movhps -15 * SIZE(X), %xmm1 @@ -1061,11 +1061,11 @@ addq $2 * SIZE, X addq $1 * SIZE, Y decq M - jle .L49 + jle L(49) movq M, %rax sarq $3, %rax - jle .L45 + jle L(45) movsd -16 * SIZE(X), %xmm1 movhps -15 * SIZE(X), %xmm1 @@ -1075,10 +1075,10 @@ movhps -11 * SIZE(X), %xmm3 decq %rax - jle .L42 + jle L(42) ALIGN_3 -.L41: +L(41): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -1186,10 +1186,10 @@ subq $-16 * SIZE, X subq $-16 * SIZE, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L42: +L(42): pshufd $0x4e, %xmm1, %xmm8 mulpd ALPHA_R, %xmm1 mulpd ALPHA_I, %xmm8 @@ -1276,10 +1276,10 @@ subq $-16 * SIZE, Y ALIGN_3 -.L45: +L(45): movq M, %rax andq $4, %rax - jle .L46 + jle L(46) movsd -16 * SIZE(X), %xmm1 movhps -15 * SIZE(X), %xmm1 @@ -1331,10 +1331,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L46: +L(46): movq M, %rax andq $2, %rax - jle .L47 + jle L(47) movsd -16 * SIZE(X), %xmm1 movhps -15 * SIZE(X), %xmm1 @@ -1364,10 +1364,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L47: +L(47): movq M, %rax andq $1, %rax - jle .L49 + jle L(49) movsd -16 * SIZE(X), %xmm1 movhps -15 * SIZE(X), %xmm1 @@ -1386,15 +1386,15 @@ addq $2 * SIZE, Y ALIGN_3 -.L49: +L(49): SHUFPD_1 %xmm0, %xmm0 addsd -16 * SIZE(Y), %xmm0 movlps %xmm0, -16 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L50: +L(50): #ifndef CONJ movaps %xmm0, %xmm14 # a 0 @@ -1418,12 +1418,12 @@ movq M, %rax //If incx==0 || incy==0, avoid unloop and jump to end. cmpq $0, INCX - jne .L59 + jne L(59) cmpq $0, INCY - je .L58 -.L59: + je L(58) +L(59): sarq $3, %rax - jle .L55 + jle L(55) MOVDDUP( 0 * SIZE, X, %xmm0) MOVDDUP( 1 * SIZE, X, %xmm1) @@ -1457,10 +1457,10 @@ mulpd %xmm14, %xmm6 decq %rax - jle .L52 + jle L(52) ALIGN_3 -.L51: +L(51): addpd %xmm0, %xmm8 mulpd %xmm15, %xmm1 addpd %xmm2, %xmm9 @@ -1578,10 +1578,10 @@ addq INCY, Y decq %rax - jg .L51 + jg L(51) ALIGN_3 -.L52: +L(52): addpd %xmm0, %xmm8 mulpd %xmm15, %xmm1 addpd %xmm2, %xmm9 @@ -1668,10 +1668,10 @@ addq INCY, YY ALIGN_3 -.L55: +L(55): movq M, %rax andq $4, %rax - jle .L56 + jle L(56) MOVDDUP( 0 * SIZE, X, %xmm0) MOVDDUP( 1 * SIZE, X, %xmm1) @@ -1733,10 +1733,10 @@ addq INCY, YY ALIGN_3 -.L56: +L(56): movq M, %rax andq $2, %rax - jle .L57 + jle L(57) MOVDDUP( 0 * SIZE, X, %xmm0) MOVDDUP( 1 * SIZE, X, %xmm1) @@ -1770,12 +1770,12 @@ addq INCY, YY ALIGN_3 -.L57: +L(57): movq M, %rax andq $1, %rax - jle .L999 + jle L(999) -.L58: +L(58): MOVDDUP( 0 * SIZE, X, %xmm0) MOVDDUP( 1 * SIZE, X, %xmm1) @@ -1790,10 +1790,10 @@ movhpd %xmm8, 1 * SIZE(YY) decq %rax - jg .L58 + jg L(58) ALIGN_3 -.L999: +L(999): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/zcopy.S b/kernel/x86_64/zcopy.S index 3cc4e185b4..35b9f2221a 100644 --- a/kernel/x86_64/zcopy.S +++ b/kernel/x86_64/zcopy.S @@ -66,19 +66,19 @@ salq $ZBASE_SHIFT, INCY testq N, N # if m == 0 goto End - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX # if incx != 1 - jne .L100 + jne L(100) cmpq $2 * SIZE, INCY # if incy != 1 - jne .L100 + jne L(100) movq N, %rax # i = m sarq $2, %rax - jle .L20 + jle L(20) ALIGN_2 -.L11: +L(11): #ifdef XDOUBLE #ifdef PREFETCH @@ -201,16 +201,16 @@ addq $8 * SIZE, X addq $8 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_2 -.L20: +L(20): movq N, %rax # i = m andq $3, %rax - jle .L99 + jle L(99) ALIGN_2 -.L21: +L(21): #ifdef XDOUBLE movq 0(X), %mm0 movq %mm0, 0(Y) @@ -233,21 +233,21 @@ addq $2 * SIZE, X addq $2 * SIZE, Y decq %rax - jg .L21 + jg L(21) -.L99: +L(99): xorq %rax,%rax EMMS ret ALIGN_3 -.L100: +L(100): movq N, %rax sarq $2, %rax - jle .L120 + jle L(120) ALIGN_2 -.L111: +L(111): #ifdef XDOUBLE movq 0(X), %mm0 movq %mm0, 0(Y) @@ -343,15 +343,15 @@ #endif decq %rax - jg .L111 + jg L(111) -.L120: +L(120): movq N, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_2 -.L121: +L(121): #ifdef XDOUBLE movq 0(X), %mm0 movq %mm0, 0(Y) @@ -378,9 +378,9 @@ #endif decq %rax - jg .L121 + jg L(121) -.L999: +L(999): xorq %rax,%rax EMMS ret diff --git a/kernel/x86_64/zcopy_sse.S b/kernel/x86_64/zcopy_sse.S index 018a56ff23..a36ce0339a 100644 --- a/kernel/x86_64/zcopy_sse.S +++ b/kernel/x86_64/zcopy_sse.S @@ -70,19 +70,19 @@ salq $ZBASE_SHIFT, INCY cmpq $2 * SIZE, INCX - jne .L100 + jne L(100) cmpq $2 * SIZE, INCY - jne .L100 + jne L(100) cmpq $3, M - jle .L106 + jle L(106) subq $-32 * SIZE, X subq $-32 * SIZE, Y addq M, M testq $SIZE, Y - je .L05 + je L(05) movss -32 * SIZE(X), %xmm0 movss %xmm0, -32 * SIZE(Y) @@ -91,25 +91,25 @@ decq M ALIGN_4 -.L05: +L(05): testq $2 * SIZE, Y - je .L10 + je L(10) movsd -32 * SIZE(X), %xmm0 movlps %xmm0, -32 * SIZE(Y) addq $2 * SIZE, X addq $2 * SIZE, Y subq $2, M - jle .L19 + jle L(19) ALIGN_4 -.L10: +L(10): testq $3 * SIZE, X - jne .L20 + jne L(20) movq M, %rax sarq $5, %rax - jle .L13 + jle L(13) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -121,10 +121,10 @@ movaps -4 * SIZE(X), %xmm7 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -164,10 +164,10 @@ subq $-32 * SIZE, Y subq $-32 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movaps %xmm0, -32 * SIZE(Y) movaps %xmm1, -28 * SIZE(Y) movaps %xmm2, -24 * SIZE(Y) @@ -181,9 +181,9 @@ subq $-32 * SIZE, X ALIGN_3 -.L13: +L(13): testq $16, M - jle .L14 + jle L(14) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -199,9 +199,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $8, M - jle .L15 + jle L(15) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -213,9 +213,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $4, M - jle .L16 + jle L(16) movaps -32 * SIZE(X), %xmm0 movaps %xmm0, -32 * SIZE(Y) @@ -224,9 +224,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $2, M - jle .L17 + jle L(17) movsd -32 * SIZE(X), %xmm0 movlps %xmm0, -32 * SIZE(Y) @@ -235,15 +235,15 @@ addq $2 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $1, M - jle .L19 + jle L(19) movss -32 * SIZE(X), %xmm0 movss %xmm0, -32 * SIZE(Y) ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -252,15 +252,15 @@ ALIGN_3 -.L20: +L(20): testq $SIZE, X - jne .L30 + jne L(30) movhps -32 * SIZE(X), %xmm0 movq M, %rax sarq $5, %rax - jle .L23 + jle L(23) movaps -30 * SIZE(X), %xmm1 movaps -26 * SIZE(X), %xmm2 @@ -271,10 +271,10 @@ movaps -6 * SIZE(X), %xmm7 decq %rax - jle .L22 + jle L(22) ALIGN_4 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) @@ -327,10 +327,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): shufps $0x4e, %xmm1, %xmm0 movaps %xmm0, -32 * SIZE(Y) movaps -2 * SIZE(X), %xmm0 @@ -360,9 +360,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L23: +L(23): testq $16, M - jle .L24 + jle L(24) ALIGN_3 movaps -30 * SIZE(X), %xmm1 @@ -385,9 +385,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $8, M - jle .L25 + jle L(25) ALIGN_3 movaps -30 * SIZE(X), %xmm1 @@ -404,9 +404,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $4, M - jle .L26 + jle L(26) ALIGN_3 movaps -30 * SIZE(X), %xmm1 @@ -417,9 +417,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $2, M - jle .L27 + jle L(27) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -430,9 +430,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $1, M - jle .L29 + jle L(29) ALIGN_3 movss -32 * SIZE(X), %xmm0 @@ -440,7 +440,7 @@ addq $SIZE, Y ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -448,15 +448,15 @@ ret ALIGN_3 -.L30: +L(30): testq $2 * SIZE, X - jne .L40 + jne L(40) movaps -33 * SIZE(X), %xmm0 movq M, %rax sarq $5, %rax - jle .L33 + jle L(33) movaps -29 * SIZE(X), %xmm1 movaps -25 * SIZE(X), %xmm2 @@ -467,10 +467,10 @@ movaps -5 * SIZE(X), %xmm7 decq %rax - jle .L32 + jle L(32) ALIGN_4 -.L31: +L(31): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -530,10 +530,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L32: +L(32): movss %xmm1, %xmm0 shufps $0x39, %xmm0, %xmm0 movaps %xmm0, -32 * SIZE(Y) @@ -571,9 +571,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L33: +L(33): testq $16, M - jle .L34 + jle L(34) ALIGN_3 movaps -29 * SIZE(X), %xmm1 @@ -603,9 +603,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L34: +L(34): testq $8, M - jle .L35 + jle L(35) ALIGN_3 movaps -29 * SIZE(X), %xmm1 @@ -624,9 +624,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L35: +L(35): testq $4, M - jle .L36 + jle L(36) ALIGN_3 movaps -29 * SIZE(X), %xmm1 @@ -640,9 +640,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L36: +L(36): testq $2, M - jle .L37 + jle L(37) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -652,9 +652,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L37: +L(37): testq $1, M - jle .L39 + jle L(39) ALIGN_3 movss -32 * SIZE(X), %xmm0 @@ -662,7 +662,7 @@ addq $SIZE, Y ALIGN_3 -.L39: +L(39): xorq %rax,%rax RESTOREREGISTERS @@ -670,12 +670,12 @@ ret ALIGN_3 -.L40: +L(40): movaps -35 * SIZE(X), %xmm0 movq M, %rax sarq $5, %rax - jle .L43 + jle L(43) movaps -31 * SIZE(X), %xmm1 movaps -27 * SIZE(X), %xmm2 @@ -686,10 +686,10 @@ movaps -7 * SIZE(X), %xmm7 decq %rax - jle .L42 + jle L(42) ALIGN_4 -.L41: +L(41): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -749,10 +749,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L42: +L(42): movss %xmm1, %xmm0 shufps $0x93, %xmm1, %xmm0 movaps %xmm0, -32 * SIZE(Y) @@ -790,9 +790,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L43: +L(43): testq $16, M - jle .L44 + jle L(44) ALIGN_3 movaps -31 * SIZE(X), %xmm1 @@ -822,9 +822,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L44: +L(44): testq $8, M - jle .L45 + jle L(45) ALIGN_3 movaps -31 * SIZE(X), %xmm1 @@ -844,9 +844,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L45: +L(45): testq $4, M - jle .L46 + jle L(46) ALIGN_3 movaps -31 * SIZE(X), %xmm1 @@ -860,9 +860,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L46: +L(46): testq $2, M - jle .L47 + jle L(47) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -872,9 +872,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L47: +L(47): testq $1, M - jle .L49 + jle L(49) ALIGN_3 movss -32 * SIZE(X), %xmm0 @@ -882,7 +882,7 @@ addq $SIZE, Y ALIGN_3 -.L49: +L(49): xorq %rax,%rax RESTOREREGISTERS @@ -890,13 +890,13 @@ ret ALIGN_4 -.L100: +L(100): movq M, %rax sarq $3, %rax - jle .L105 + jle L(105) ALIGN_3 -.L102: +L(102): movsd (X), %xmm0 addq INCX, X movhps (X), %xmm0 @@ -932,12 +932,12 @@ addq INCY, Y decq %rax - jg .L102 + jg L(102) ALIGN_3 -.L105: +L(105): testq $4, M - jle .L106 + jle L(106) movsd (X), %xmm0 addq INCX, X @@ -958,9 +958,9 @@ addq INCY, Y ALIGN_3 -.L106: +L(106): testq $2, M - jle .L107 + jle L(107) movsd (X), %xmm0 addq INCX, X @@ -973,15 +973,15 @@ addq INCY, Y ALIGN_3 -.L107: +L(107): testq $1, M - jle .L999 + jle L(999) movsd (X), %xmm0 movsd %xmm0, (Y) ALIGN_3 -.L999: +L(999): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/zcopy_sse2.S b/kernel/x86_64/zcopy_sse2.S index c3a99a57b3..c06e65e1d4 100644 --- a/kernel/x86_64/zcopy_sse2.S +++ b/kernel/x86_64/zcopy_sse2.S @@ -70,9 +70,9 @@ salq $ZBASE_SHIFT, INCY cmpq $2 * SIZE, INCX - jne .L50 + jne L(50) cmpq $2 * SIZE, INCY - jne .L50 + jne L(50) addq M, M @@ -81,17 +81,17 @@ #else testq $SIZE, X #endif - je .L10 + je L(10) movsd (X), %xmm0 movsd %xmm0, (Y) addq $1 * SIZE, X addq $1 * SIZE, Y decq M - jle .L19 + jle L(19) ALIGN_4 -.L10: +L(10): subq $-16 * SIZE, X subq $-16 * SIZE, Y @@ -100,11 +100,11 @@ #else testq $SIZE, Y #endif - jne .L20 + jne L(20) movq M, %rax sarq $4, %rax - jle .L13 + jle L(13) movaps -16 * SIZE(X), %xmm0 movaps -14 * SIZE(X), %xmm1 @@ -116,10 +116,10 @@ movaps -2 * SIZE(X), %xmm7 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -159,10 +159,10 @@ subq $-16 * SIZE, Y subq $-16 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movaps %xmm0, -16 * SIZE(Y) movaps %xmm1, -14 * SIZE(Y) movaps %xmm2, -12 * SIZE(Y) @@ -176,9 +176,9 @@ subq $-16 * SIZE, X ALIGN_3 -.L13: +L(13): testq $8, M - jle .L14 + jle L(14) ALIGN_3 movaps -16 * SIZE(X), %xmm0 @@ -195,9 +195,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $4, M - jle .L15 + jle L(15) ALIGN_3 movaps -16 * SIZE(X), %xmm0 @@ -210,9 +210,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $2, M - jle .L16 + jle L(16) ALIGN_3 movaps -16 * SIZE(X), %xmm0 @@ -222,16 +222,16 @@ addq $2 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $1, M - jle .L19 + jle L(19) ALIGN_3 movsd -16 * SIZE(X), %xmm0 movsd %xmm0, -16 * SIZE(Y) ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -239,14 +239,14 @@ ret ALIGN_3 -.L20: +L(20): #ifdef ALIGNED_ACCESS movhps -16 * SIZE(X), %xmm0 movq M, %rax sarq $4, %rax - jle .L23 + jle L(23) movaps -15 * SIZE(X), %xmm1 movaps -13 * SIZE(X), %xmm2 @@ -257,10 +257,10 @@ movaps -3 * SIZE(X), %xmm7 decq %rax - jle .L22 + jle L(22) ALIGN_4 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -312,10 +312,10 @@ subq $-16 * SIZE, X subq $-16 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): SHUFPD_1 %xmm1, %xmm0 movaps %xmm0, -16 * SIZE(Y) LOAD(-1 * SIZE, X, %xmm0) @@ -342,9 +342,9 @@ subq $-16 * SIZE, Y ALIGN_3 -.L23: +L(23): testq $8, M - jle .L24 + jle L(24) ALIGN_3 movaps -15 * SIZE(X), %xmm1 @@ -370,9 +370,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $4, M - jle .L25 + jle L(25) ALIGN_3 movaps -15 * SIZE(X), %xmm1 @@ -389,9 +389,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $2, M - jle .L26 + jle L(26) ALIGN_3 movaps -15 * SIZE(X), %xmm1 @@ -403,16 +403,16 @@ addq $2 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $1, M - jle .L29 + jle L(29) ALIGN_3 movsd -16 * SIZE(X), %xmm0 movsd %xmm0, -16 * SIZE(Y) ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -424,7 +424,7 @@ movq M, %rax sarq $4, %rax - jle .L23 + jle L(23) movaps -16 * SIZE(X), %xmm0 movaps -14 * SIZE(X), %xmm1 @@ -436,10 +436,10 @@ movaps -2 * SIZE(X), %xmm7 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -487,10 +487,10 @@ subq $-16 * SIZE, Y subq $-16 * SIZE, X decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): movlps %xmm0, -16 * SIZE(Y) movhps %xmm0, -15 * SIZE(Y) movlps %xmm1, -14 * SIZE(Y) @@ -512,9 +512,9 @@ subq $-16 * SIZE, X ALIGN_3 -.L23: +L(23): testq $8, M - jle .L24 + jle L(24) ALIGN_3 movaps -16 * SIZE(X), %xmm0 @@ -534,9 +534,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $4, M - jle .L25 + jle L(25) ALIGN_3 movaps -16 * SIZE(X), %xmm0 @@ -550,9 +550,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $2, M - jle .L26 + jle L(26) ALIGN_3 movaps -16 * SIZE(X), %xmm0 @@ -563,16 +563,16 @@ addq $2 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $1, M - jle .L29 + jle L(29) ALIGN_3 movsd -16 * SIZE(X), %xmm0 movsd %xmm0, -16 * SIZE(Y) ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -582,13 +582,13 @@ #endif -.L50: +L(50): movq M, %rax sarq $2, %rax - jle .L55 + jle L(55) ALIGN_3 -.L51: +L(51): movsd 0 * SIZE(X), %xmm0 movhps 1 * SIZE(X), %xmm0 addq INCX, X @@ -623,16 +623,16 @@ addq INCY, Y decq %rax - jg .L51 + jg L(51) ALIGN_3 -.L55: +L(55): movq M, %rax andq $3, %rax - jle .L57 + jle L(57) ALIGN_3 -.L56: +L(56): movsd 0 * SIZE(X), %xmm0 movhps 1 * SIZE(X), %xmm0 addq INCX, X @@ -642,10 +642,10 @@ addq INCY, Y decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L57: +L(57): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/zdot.S b/kernel/x86_64/zdot.S index 87c08d7c80..fcae4bf1d7 100644 --- a/kernel/x86_64/zdot.S +++ b/kernel/x86_64/zdot.S @@ -61,7 +61,7 @@ #endif testq N, N - jle .L88 + jle L(88) salq $ZBASE_SHIFT, INCX salq $ZBASE_SHIFT, INCY @@ -72,16 +72,16 @@ fldz cmpq $2 * SIZE, INCX - jne .L14 + jne L(14) cmpq $2 * SIZE, INCY - jne .L14 + jne L(14) movq N, %rax sarq $1, %rax - jle .L15 + jle L(15) ALIGN_3 -.L16: +L(16): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -130,16 +130,16 @@ addq $4 * SIZE, X addq $4 * SIZE, Y decq %rax - jg .L16 + jg L(16) ALIGN_3 -.L15: +L(15): movq N, %rax andq $1, %rax - jle .L27 + jle L(27) ALIGN_3 -.L22: +L(22): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -158,17 +158,17 @@ FLD 1 * SIZE(Y) fmulp %st, %st(1) faddp %st, %st(4) - jmp .L27 + jmp L(27) ALIGN_3 -.L14: +L(14): movq N, %rax sarq $1, %rax - jle .L30 + jle L(30) ALIGN_3 -.L31: +L(31): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -212,16 +212,16 @@ addq INCY, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L30: +L(30): movq N, %rax andq $1, %rax - jle .L27 + jle L(27) ALIGN_3 -.L37: +L(37): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -242,7 +242,7 @@ faddp %st, %st(4) ALIGN_3 -.L27: +L(27): #ifndef CONJ fsubp %st, %st(3) faddp %st, %st(1) @@ -255,7 +255,7 @@ ret ALIGN_3 -.L88: +L(88): fldz fldz diff --git a/kernel/x86_64/zdot_atom.S b/kernel/x86_64/zdot_atom.S index 9a8239c8d4..4884903e21 100644 --- a/kernel/x86_64/zdot_atom.S +++ b/kernel/x86_64/zdot_atom.S @@ -68,16 +68,16 @@ pxor %xmm2, %xmm2 cmpq $0, N pxor %xmm3, %xmm3 - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX - jne .L20 + jne L(20) cmpq $2 * SIZE, INCY - jne .L20 + jne L(20) movq N, %rax sarq $2, %rax - jle .L15 + jle L(15) movsd 0 * SIZE(X), %xmm4 movsd 0 * SIZE(Y), %xmm6 @@ -96,10 +96,10 @@ movsd 3 * SIZE(Y), %xmm13 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -172,10 +172,10 @@ addq $8 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): addsd %xmm4, %xmm0 movaps %xmm10, %xmm14 mulsd %xmm11, %xmm10 @@ -226,10 +226,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L15: +L(15): movq N, %rax andq $2, %rax - jle .L17 + jle L(17) movsd 0 * SIZE(X), %xmm4 movsd 0 * SIZE(Y), %xmm6 @@ -267,10 +267,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L17: +L(17): movq N, %rax andq $1, %rax - jle .L999 + jle L(999) movsd 0 * SIZE(X), %xmm4 movsd 0 * SIZE(Y), %xmm6 @@ -288,16 +288,16 @@ addsd %xmm8, %xmm1 addsd %xmm5, %xmm2 addsd %xmm9, %xmm3 - jmp .L999 + jmp L(999) ALIGN_3 -.L20: +L(20): movq N, %rax sarq $2, %rax - jle .L25 + jle L(25) ALIGN_3 -.L23: +L(23): movsd 0 * SIZE(X), %xmm4 movsd 0 * SIZE(Y), %xmm6 movsd 1 * SIZE(X), %xmm5 @@ -372,16 +372,16 @@ addsd %xmm9, %xmm3 decq %rax - jg .L23 + jg L(23) ALIGN_3 -.L25: +L(25): testq $3, N - je .L999 + je L(999) movq N, %rax andq $2, %rax - jle .L27 + jle L(27) movsd 0 * SIZE(X), %xmm4 movsd 0 * SIZE(Y), %xmm6 @@ -422,10 +422,10 @@ ALIGN_3 -.L27: +L(27): movq N, %rax andq $1, %rax - jle .L999 + jle L(999) movsd 0 * SIZE(X), %xmm4 movsd 0 * SIZE(Y), %xmm6 @@ -445,7 +445,7 @@ addsd %xmm9, %xmm3 ALIGN_3 -.L999: +L(999): #ifndef CONJ subsd %xmm2, %xmm0 addsd %xmm3, %xmm1 diff --git a/kernel/x86_64/zdot_sse.S b/kernel/x86_64/zdot_sse.S index f53e04cfe6..92e9ab7807 100644 --- a/kernel/x86_64/zdot_sse.S +++ b/kernel/x86_64/zdot_sse.S @@ -69,22 +69,22 @@ xorps %xmm3, %xmm3 testq N, N - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX - jne .L200 + jne L(200) cmpq $2 * SIZE, INCY - jne .L200 + jne L(200) subq $-32 * SIZE, X subq $-32 * SIZE, Y testq $SIZE, X - jne .L50 + jne L(50) -.L0x: +L(0x): testq $2 * SIZE, X - je .L10 + je L(10) #ifdef movsd xorps %xmm4, %xmm4 @@ -100,13 +100,13 @@ decq N ALIGN_3 -.L10: +L(10): testq $3 * SIZE, Y - jne .L20 + jne L(20) movq N, %rax sarq $4, %rax - jle .L15 + jle L(15) movaps -32 * SIZE(X), %xmm4 movaps -28 * SIZE(X), %xmm5 @@ -118,10 +118,10 @@ movaps -20 * SIZE(Y), %xmm11 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -206,10 +206,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): pshufd $0xb1, %xmm8, %xmm12 mulps %xmm4, %xmm8 addps %xmm8, %xmm0 @@ -270,9 +270,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $8, N - jle .L16 + jle L(16) movaps -32 * SIZE(X), %xmm4 movaps -32 * SIZE(Y), %xmm8 @@ -314,9 +314,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $4, N - jle .L17 + jle L(17) movaps -32 * SIZE(X), %xmm4 movaps -32 * SIZE(Y), %xmm8 @@ -339,9 +339,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $2, N - jle .L18 + jle L(18) movaps -32 * SIZE(X), %xmm4 movaps -32 * SIZE(Y), %xmm8 @@ -356,9 +356,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L18: +L(18): testq $1, N - jle .L98 + jle L(98) #ifdef movsd xorps %xmm4, %xmm4 @@ -374,14 +374,14 @@ addps %xmm8, %xmm0 mulps %xmm4, %xmm12 addps %xmm12, %xmm1 - jmp .L98 + jmp L(98) ALIGN_3 -.L20: +L(20): #ifdef ALIGNED_ACCESS testq $2 * SIZE, Y - jne .L30 + jne L(30) movaps -33 * SIZE(Y), %xmm8 addq $3 * SIZE, Y @@ -390,7 +390,7 @@ movq N, %rax sarq $4, %rax - jle .L25 + jle L(25) movaps -32 * SIZE(X), %xmm4 movaps -32 * SIZE(Y), %xmm9 @@ -401,10 +401,10 @@ movaps -20 * SIZE(X), %xmm7 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -505,10 +505,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): movss %xmm9, %xmm8 pshufd $0xb1, %xmm4, %xmm12 shufps $0x39, %xmm8, %xmm8 @@ -586,9 +586,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $8, N - jle .L26 + jle L(26) movaps -32 * SIZE(X), %xmm4 movaps -32 * SIZE(Y), %xmm9 @@ -637,9 +637,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $4, N - jle .L27 + jle L(27) movaps -32 * SIZE(X), %xmm4 movaps -32 * SIZE(Y), %xmm9 @@ -669,9 +669,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $2, N - jle .L28 + jle L(28) movaps -32 * SIZE(X), %xmm4 movaps -32 * SIZE(Y), %xmm9 @@ -689,9 +689,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L28: +L(28): testq $1, N - jle .L29 + jle L(29) #ifdef movsd xorps %xmm4, %xmm4 @@ -706,21 +706,21 @@ addps %xmm12, %xmm1 ALIGN_3 -.L29: +L(29): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm3, %xmm3 - jmp .L98 + jmp L(98) ALIGN_3 -.L30: +L(30): testq $SIZE, Y - jne .L40 + jne L(40) #endif movq N, %rax sarq $4, %rax - jle .L35 + jle L(35) movaps -32 * SIZE(X), %xmm4 movsd -32 * SIZE(Y), %xmm8 @@ -737,10 +737,10 @@ movhps -18 * SIZE(Y), %xmm11 decq %rax - jle .L32 + jle L(32) ALIGN_3 -.L31: +L(31): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -833,10 +833,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L32: +L(32): pshufd $0xb1, %xmm8, %xmm12 mulps %xmm4, %xmm8 addps %xmm8, %xmm0 @@ -901,9 +901,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L35: +L(35): testq $8, N - jle .L36 + jle L(36) movaps -32 * SIZE(X), %xmm4 movsd -32 * SIZE(Y), %xmm8 @@ -949,9 +949,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L36: +L(36): testq $4, N - jle .L37 + jle L(37) movaps -32 * SIZE(X), %xmm4 movsd -32 * SIZE(Y), %xmm8 @@ -977,9 +977,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L37: +L(37): testq $2, N - jle .L38 + jle L(38) movaps -32 * SIZE(X), %xmm4 movsd -32 * SIZE(Y), %xmm8 @@ -995,9 +995,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L38: +L(38): testq $1, N - jle .L98 + jle L(98) #ifdef movsd xorps %xmm4, %xmm4 @@ -1013,11 +1013,11 @@ addps %xmm8, %xmm0 mulps %xmm4, %xmm12 addps %xmm12, %xmm1 - jmp .L98 + jmp L(98) ALIGN_3 #ifdef ALIGNED_ACCESS -.L40: +L(40): movaps -35 * SIZE(Y), %xmm8 addq $1 * SIZE, Y @@ -1025,7 +1025,7 @@ movq N, %rax sarq $4, %rax - jle .L45 + jle L(45) movaps -32 * SIZE(X), %xmm4 movaps -32 * SIZE(Y), %xmm9 @@ -1036,10 +1036,10 @@ movaps -20 * SIZE(X), %xmm7 decq %rax - jle .L42 + jle L(42) ALIGN_3 -.L41: +L(41): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -1140,10 +1140,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L42: +L(42): movss %xmm9, %xmm8 pshufd $0xb1, %xmm4, %xmm12 shufps $0x93, %xmm9, %xmm8 @@ -1221,9 +1221,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L45: +L(45): testq $8, N - jle .L46 + jle L(46) movaps -32 * SIZE(X), %xmm4 movaps -32 * SIZE(Y), %xmm9 @@ -1272,9 +1272,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L46: +L(46): testq $4, N - jle .L47 + jle L(47) movaps -32 * SIZE(X), %xmm4 movaps -32 * SIZE(Y), %xmm9 @@ -1304,9 +1304,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L47: +L(47): testq $2, N - jle .L48 + jle L(48) movaps -32 * SIZE(X), %xmm4 movaps -32 * SIZE(Y), %xmm9 @@ -1324,9 +1324,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L48: +L(48): testq $1, N - jle .L49 + jle L(49) #ifdef movsd xorps %xmm4, %xmm4 @@ -1343,21 +1343,21 @@ addps %xmm12, %xmm1 ALIGN_3 -.L49: +L(49): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm3, %xmm3 - jmp .L98 + jmp L(98) ALIGN_3 #endif -.L50: +L(50): testq $SIZE, Y - jne .L70 + jne L(70) #ifdef ALIGNED_ACCESS testq $2 * SIZE, Y - je .L50x + je L(50x) #ifdef movsd xorps %xmm0, %xmm0 @@ -1377,9 +1377,9 @@ decq N ALIGN_3 -.L50x: +L(50x): testq $2 * SIZE, X - jne .L60 + jne L(60) movaps -33 * SIZE(X), %xmm8 addq $3 * SIZE, X @@ -1388,7 +1388,7 @@ movq N, %rax sarq $4, %rax - jle .L55 + jle L(55) movaps -32 * SIZE(Y), %xmm4 movaps -32 * SIZE(X), %xmm9 @@ -1399,10 +1399,10 @@ movaps -20 * SIZE(Y), %xmm7 decq %rax - jle .L52 + jle L(52) ALIGN_3 -.L51: +L(51): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -1503,10 +1503,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L51 + jg L(51) ALIGN_3 -.L52: +L(52): movss %xmm9, %xmm8 pshufd $0xb1, %xmm4, %xmm12 shufps $0x39, %xmm8, %xmm8 @@ -1584,9 +1584,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L55: +L(55): testq $8, N - jle .L56 + jle L(56) movaps -32 * SIZE(Y), %xmm4 movaps -32 * SIZE(X), %xmm9 @@ -1635,9 +1635,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L56: +L(56): testq $4, N - jle .L57 + jle L(57) movaps -32 * SIZE(Y), %xmm4 movaps -32 * SIZE(X), %xmm9 @@ -1667,9 +1667,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L57: +L(57): testq $2, N - jle .L58 + jle L(58) movaps -32 * SIZE(Y), %xmm4 movaps -32 * SIZE(X), %xmm9 @@ -1687,9 +1687,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L58: +L(58): testq $1, N - jle .L98 + jle L(98) #ifdef movsd xorps %xmm4, %xmm4 @@ -1702,10 +1702,10 @@ addps %xmm4, %xmm0 mulps %xmm8, %xmm12 addps %xmm12, %xmm1 - jmp .L98 + jmp L(98) ALIGN_3 -.L60: +L(60): movaps -35 * SIZE(X), %xmm8 addq $1 * SIZE, X @@ -1713,7 +1713,7 @@ movq N, %rax sarq $4, %rax - jle .L65 + jle L(65) movaps -32 * SIZE(Y), %xmm4 movaps -32 * SIZE(X), %xmm9 @@ -1724,10 +1724,10 @@ movaps -20 * SIZE(Y), %xmm7 decq %rax - jle .L62 + jle L(62) ALIGN_3 -.L61: +L(61): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -1828,10 +1828,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L61 + jg L(61) ALIGN_3 -.L62: +L(62): movss %xmm9, %xmm8 pshufd $0xb1, %xmm4, %xmm12 shufps $0x93, %xmm9, %xmm8 @@ -1909,9 +1909,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L65: +L(65): testq $8, N - jle .L66 + jle L(66) movaps -32 * SIZE(Y), %xmm4 movaps -32 * SIZE(X), %xmm9 @@ -1960,9 +1960,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L66: +L(66): testq $4, N - jle .L67 + jle L(67) movaps -32 * SIZE(Y), %xmm4 movaps -32 * SIZE(X), %xmm9 @@ -1992,9 +1992,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L67: +L(67): testq $2, N - jle .L68 + jle L(68) movaps -32 * SIZE(Y), %xmm4 movaps -32 * SIZE(X), %xmm9 @@ -2012,9 +2012,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L68: +L(68): testq $1, N - jle .L98 + jle L(98) #ifdef movsd xorps %xmm4, %xmm4 @@ -2029,13 +2029,13 @@ addps %xmm4, %xmm0 mulps %xmm8, %xmm12 addps %xmm12, %xmm1 - jmp .L98 + jmp L(98) ALIGN_3 #else testq $2 * SIZE, Y - je .L50x + je L(50x) #ifdef movsd xorps %xmm0, %xmm0 @@ -2055,10 +2055,10 @@ decq N ALIGN_3 -.L50x: +L(50x): movq N, %rax sarq $4, %rax - jle .L55 + jle L(55) movaps -32 * SIZE(Y), %xmm4 movlps -32 * SIZE(X), %xmm8 @@ -2075,10 +2075,10 @@ movhps -18 * SIZE(X), %xmm11 decq %rax - jle .L52 + jle L(52) ALIGN_3 -.L51: +L(51): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -2171,10 +2171,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L51 + jg L(51) ALIGN_3 -.L52: +L(52): pshufd $0xb1, %xmm4, %xmm12 mulps %xmm8, %xmm4 addps %xmm4, %xmm0 @@ -2239,9 +2239,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L55: +L(55): testq $8, N - jle .L56 + jle L(56) movaps -32 * SIZE(Y), %xmm4 movlps -32 * SIZE(X), %xmm8 @@ -2287,9 +2287,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L56: +L(56): testq $4, N - jle .L57 + jle L(57) movaps -32 * SIZE(Y), %xmm4 movlps -32 * SIZE(X), %xmm8 @@ -2315,9 +2315,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L57: +L(57): testq $2, N - jle .L58 + jle L(58) movaps -32 * SIZE(Y), %xmm4 movlps -32 * SIZE(X), %xmm8 @@ -2334,9 +2334,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L58: +L(58): testq $1, N - jle .L98 + jle L(98) #ifdef movsd xorps %xmm4, %xmm4 @@ -2352,13 +2352,13 @@ addps %xmm4, %xmm0 mulps %xmm8, %xmm12 addps %xmm12, %xmm1 - jmp .L98 + jmp L(98) ALIGN_3 #endif -.L70: +L(70): testq $2 * SIZE, Y - je .L70x + je L(70x) #ifdef movsd xorps %xmm4, %xmm4 @@ -2379,9 +2379,9 @@ decq N ALIGN_3 -.L70x: +L(70x): testq $2 * SIZE, X - jne .L80 + jne L(80) movaps -33 * SIZE(X), %xmm4 addq $3 * SIZE, X @@ -2390,7 +2390,7 @@ movq N, %rax sarq $4, %rax - jle .L75 + jle L(75) movaps -32 * SIZE(X), %xmm5 movaps -32 * SIZE(Y), %xmm9 @@ -2400,10 +2400,10 @@ movaps -24 * SIZE(Y), %xmm11 decq %rax - jle .L72 + jle L(72) ALIGN_3 -.L71: +L(71): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -2504,10 +2504,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L71 + jg L(71) ALIGN_3 -.L72: +L(72): movss %xmm9, %xmm8 pshufd $0x1b, %xmm8, %xmm12 movss %xmm5, %xmm4 @@ -2586,9 +2586,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L75: +L(75): testq $8, N - jle .L76 + jle L(76) movaps -32 * SIZE(X), %xmm5 movaps -32 * SIZE(Y), %xmm9 @@ -2638,9 +2638,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L76: +L(76): testq $4, N - jle .L77 + jle L(77) movaps -32 * SIZE(X), %xmm5 movaps -32 * SIZE(Y), %xmm9 @@ -2670,9 +2670,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L77: +L(77): testq $2, N - jle .L78 + jle L(78) movaps -32 * SIZE(X), %xmm5 movaps -32 * SIZE(Y), %xmm9 @@ -2689,9 +2689,9 @@ movaps %xmm9, %xmm8 ALIGN_3 -.L78: +L(78): testq $1, N - jle .L79 + jle L(79) xorps %xmm5, %xmm5 movss %xmm5, %xmm4 @@ -2707,15 +2707,15 @@ addps %xmm12, %xmm1 ALIGN_3 -.L79: +L(79): shufps $0x39, %xmm0, %xmm0 shufps $0x39, %xmm1, %xmm1 shufps $0x39, %xmm2, %xmm2 shufps $0x39, %xmm3, %xmm3 - jmp .L98 + jmp L(98) ALIGN_3 -.L80: +L(80): movsd -33 * SIZE(X), %xmm4 movhps -31 * SIZE(X), %xmm4 addq $3 * SIZE, X @@ -2724,7 +2724,7 @@ movq N, %rax sarq $4, %rax - jle .L85 + jle L(85) movsd -32 * SIZE(X), %xmm5 movhps -30 * SIZE(X), %xmm5 @@ -2739,10 +2739,10 @@ movaps -24 * SIZE(Y), %xmm11 decq %rax - jle .L82 + jle L(82) ALIGN_3 -.L81: +L(81): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -2851,10 +2851,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L81 + jg L(81) ALIGN_3 -.L82: +L(82): movss %xmm9, %xmm8 pshufd $0x1b, %xmm8, %xmm12 movss %xmm5, %xmm4 @@ -2938,9 +2938,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L85: +L(85): testq $8, N - jle .L86 + jle L(86) movsd -32 * SIZE(X), %xmm5 movhps -30 * SIZE(X), %xmm5 @@ -2994,9 +2994,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L86: +L(86): testq $4, N - jle .L87 + jle L(87) movsd -32 * SIZE(X), %xmm5 movhps -30 * SIZE(X), %xmm5 @@ -3029,9 +3029,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L87: +L(87): testq $2, N - jle .L88 + jle L(88) movsd -32 * SIZE(X), %xmm5 movhps -30 * SIZE(X), %xmm5 @@ -3049,9 +3049,9 @@ movaps %xmm9, %xmm8 ALIGN_3 -.L88: +L(88): testq $1, N - jle .L89 + jle L(89) xorps %xmm5, %xmm5 movss %xmm5, %xmm4 @@ -3067,18 +3067,18 @@ addps %xmm12, %xmm1 ALIGN_3 -.L89: +L(89): shufps $0x39, %xmm0, %xmm0 shufps $0x39, %xmm1, %xmm1 shufps $0x39, %xmm2, %xmm2 shufps $0x39, %xmm3, %xmm3 - jmp .L98 + jmp L(98) ALIGN_3 -.L200: +L(200): movq N, %rax sarq $4, %rax - jle .L205 + jle L(205) movsd (X), %xmm4 addq INCX, X @@ -3117,10 +3117,10 @@ addq INCY, Y decq %rax - jle .L204 + jle L(204) ALIGN_3 -.L203: +L(203): pshufd $0xb1, %xmm8, %xmm12 mulps %xmm4, %xmm8 addps %xmm8, %xmm0 @@ -3235,10 +3235,10 @@ addps %xmm12, %xmm3 decq %rax - jg .L203 + jg L(203) ALIGN_3 -.L204: +L(204): pshufd $0xb1, %xmm8, %xmm12 mulps %xmm4, %xmm8 addps %xmm8, %xmm0 @@ -3320,9 +3320,9 @@ addps %xmm12, %xmm3 ALIGN_3 -.L205: +L(205): testq $8, N - jle .L206 + jle L(206) movsd (X), %xmm4 addq INCX, X @@ -3385,9 +3385,9 @@ addps %xmm12, %xmm3 ALIGN_3 -.L206: +L(206): testq $4, N - jle .L207 + jle L(207) movsd (X), %xmm4 addq INCX, X @@ -3420,9 +3420,9 @@ addps %xmm12, %xmm3 ALIGN_3 -.L207: +L(207): testq $2, N - jle .L208 + jle L(208) movsd (X), %xmm4 addq INCX, X @@ -3440,9 +3440,9 @@ addps %xmm12, %xmm1 ALIGN_3 -.L208: +L(208): testq $1, N - jle .L98 + jle L(98) #ifdef movsd xorps %xmm4, %xmm4 @@ -3460,7 +3460,7 @@ addps %xmm12, %xmm1 ALIGN_3 -.L98: +L(98): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -3474,7 +3474,7 @@ pshufd $1, %xmm1, %xmm3 ALIGN_3 -.L999: +L(999): #ifndef CONJ subss %xmm2, %xmm0 addss %xmm3, %xmm1 diff --git a/kernel/x86_64/zdot_sse2.S b/kernel/x86_64/zdot_sse2.S index 3ab6f45174..2be2fb0f66 100644 --- a/kernel/x86_64/zdot_sse2.S +++ b/kernel/x86_64/zdot_sse2.S @@ -84,25 +84,25 @@ xorps %xmm3, %xmm3 cmpq $0, N - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX - jne .L50 + jne L(50) cmpq $2 * SIZE, INCY - jne .L50 + jne L(50) subq $-16 * SIZE, X subq $-16 * SIZE, Y testq $SIZE, Y - jne .L30 + jne L(30) testq $SIZE, X - jne .L20 + jne L(20) movq N, %rax sarq $3, %rax - jle .L15 + jle L(15) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -114,10 +114,10 @@ movaps -10 * SIZE(Y), %xmm11 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -202,10 +202,10 @@ subq $-16 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): pshufd $0x4e, %xmm8, %xmm12 mulpd %xmm4, %xmm8 addpd %xmm8, %xmm0 @@ -266,9 +266,9 @@ subq $-16 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $4, N - jle .L16 + jle L(16) movaps -16 * SIZE(X), %xmm4 movaps -16 * SIZE(Y), %xmm8 @@ -308,9 +308,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $2, N - jle .L17 + jle L(17) movaps -16 * SIZE(X), %xmm4 movaps -16 * SIZE(Y), %xmm8 @@ -333,9 +333,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $1, N - jle .L98 + jle L(98) movaps -16 * SIZE(X), %xmm4 movaps -16 * SIZE(Y), %xmm8 @@ -345,13 +345,13 @@ addpd %xmm8, %xmm0 mulpd %xmm4, %xmm12 addpd %xmm12, %xmm1 - jmp .L98 + jmp L(98) ALIGN_3 -.L20: +L(20): movq N, %rax sarq $3, %rax - jle .L25 + jle L(25) MOVLPS -16 * SIZE(X), %xmm4 movhps -15 * SIZE(X), %xmm4 @@ -367,10 +367,10 @@ movaps -10 * SIZE(Y), %xmm11 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -463,10 +463,10 @@ subq $-16 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): pshufd $0x4e, %xmm8, %xmm12 mulpd %xmm4, %xmm8 @@ -532,9 +532,9 @@ subq $-16 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $4, N - jle .L26 + jle L(26) MOVLPS -16 * SIZE(X), %xmm4 movhps -15 * SIZE(X), %xmm4 @@ -580,9 +580,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $2, N - jle .L27 + jle L(27) MOVLPS -16 * SIZE(X), %xmm4 movhps -15 * SIZE(X), %xmm4 @@ -608,9 +608,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $1, N - jle .L98 + jle L(98) MOVLPS -16 * SIZE(X), %xmm4 movhps -15 * SIZE(X), %xmm4 @@ -621,16 +621,16 @@ addpd %xmm8, %xmm0 mulpd %xmm4, %xmm12 addpd %xmm12, %xmm1 - jmp .L98 + jmp L(98) ALIGN_3 -.L30: +L(30): testq $SIZE, X - jne .L40 + jne L(40) movq N, %rax sarq $3, %rax - jle .L35 + jle L(35) MOVLPS -16 * SIZE(Y), %xmm4 movhps -15 * SIZE(Y), %xmm4 @@ -646,10 +646,10 @@ movaps -10 * SIZE(X), %xmm11 decq %rax - jle .L32 + jle L(32) ALIGN_3 -.L31: +L(31): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -742,10 +742,10 @@ subq $-16 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L32: +L(32): pshufd $0x4e, %xmm8, %xmm12 mulpd %xmm4, %xmm8 @@ -811,9 +811,9 @@ subq $-16 * SIZE, Y ALIGN_3 -.L35: +L(35): testq $4, N - jle .L36 + jle L(36) MOVLPS -16 * SIZE(Y), %xmm4 movhps -15 * SIZE(Y), %xmm4 @@ -859,9 +859,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L36: +L(36): testq $2, N - jle .L37 + jle L(37) MOVLPS -16 * SIZE(Y), %xmm4 movhps -15 * SIZE(Y), %xmm4 @@ -887,12 +887,12 @@ addq $4 * SIZE, Y ALIGN_3 -.L37: +L(37): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm3, %xmm3 testq $1, N - jle .L98 + jle L(98) MOVLPS -16 * SIZE(Y), %xmm4 movhps -15 * SIZE(Y), %xmm4 @@ -904,10 +904,10 @@ mulpd %xmm4, %xmm12 SHUFPD_1 %xmm12, %xmm12 addpd %xmm12, %xmm1 - jmp .L98 + jmp L(98) ALIGN_3 -.L40: +L(40): movhps -16 * SIZE(X), %xmm4 addq $SIZE, X movhps -16 * SIZE(Y), %xmm8 @@ -915,7 +915,7 @@ movq N, %rax sarq $3, %rax - jle .L45 + jle L(45) movaps -16 * SIZE(X), %xmm5 movaps -16 * SIZE(Y), %xmm9 @@ -924,10 +924,10 @@ movaps -12 * SIZE(X), %xmm7 movaps -12 * SIZE(Y), %xmm11 decq %rax - jle .L42 + jle L(42) ALIGN_3 -.L41: +L(41): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -1028,10 +1028,10 @@ subq $-16 * SIZE, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L42: +L(42): movsd %xmm9, %xmm8 pshufd $0x4e, %xmm8, %xmm12 movsd %xmm5, %xmm4 @@ -1110,9 +1110,9 @@ subq $-16 * SIZE, Y ALIGN_3 -.L45: +L(45): testq $4, N - jle .L46 + jle L(46) movaps -16 * SIZE(X), %xmm5 movaps -16 * SIZE(Y), %xmm9 @@ -1161,9 +1161,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L46: +L(46): testq $2, N - jle .L47 + jle L(47) movaps -16 * SIZE(X), %xmm5 movaps -16 * SIZE(Y), %xmm9 @@ -1194,9 +1194,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L47: +L(47): testq $1, N - jle .L48 + jle L(48) movlps -16 * SIZE(X), %xmm4 movlps -16 * SIZE(Y), %xmm8 @@ -1208,18 +1208,18 @@ addpd %xmm12, %xmm1 ALIGN_3 -.L48: +L(48): SHUFPD_1 %xmm0, %xmm0 SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm2, %xmm2 SHUFPD_1 %xmm3, %xmm3 - jmp .L98 + jmp L(98) ALIGN_3 -.L50: +L(50): movq N, %rax sarq $3, %rax - jle .L55 + jle L(55) MOVLPS 0 * SIZE(X), %xmm4 movhps 1 * SIZE(X), %xmm4 @@ -1250,10 +1250,10 @@ addq INCY, Y decq %rax - jle .L54 + jle L(54) ALIGN_3 -.L53: +L(53): pshufd $0x4e, %xmm8, %xmm12 mulpd %xmm4, %xmm8 addpd %xmm8, %xmm0 @@ -1353,10 +1353,10 @@ addpd %xmm12, %xmm3 decq %rax - jg .L53 + jg L(53) ALIGN_3 -.L54: +L(54): pshufd $0x4e, %xmm8, %xmm12 mulpd %xmm4, %xmm8 addpd %xmm8, %xmm0 @@ -1430,9 +1430,9 @@ addpd %xmm12, %xmm3 ALIGN_3 -.L55: +L(55): testq $4, N - jle .L56 + jle L(56) MOVLPS 0 * SIZE(X), %xmm4 movhps 1 * SIZE(X), %xmm4 @@ -1487,9 +1487,9 @@ addpd %xmm12, %xmm3 ALIGN_3 -.L56: +L(56): testq $2, N - jle .L57 + jle L(57) MOVLPS 0 * SIZE(X), %xmm4 movhps 1 * SIZE(X), %xmm4 @@ -1518,9 +1518,9 @@ addpd %xmm12, %xmm3 ALIGN_3 -.L57: +L(57): testq $1, N - jle .L98 + jle L(98) MOVLPS 0 * SIZE(X), %xmm4 movhps 1 * SIZE(X), %xmm4 @@ -1534,14 +1534,14 @@ addpd %xmm12, %xmm1 ALIGN_3 -.L98: +L(98): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 pshufd $0x4e, %xmm0, %xmm2 pshufd $0x4e, %xmm1, %xmm3 -.L999: +L(999): #ifndef CONJ subsd %xmm2, %xmm0 addsd %xmm3, %xmm1 diff --git a/kernel/x86_64/zgemm3m_kernel_2x8_nehalem.S b/kernel/x86_64/zgemm3m_kernel_2x8_nehalem.S index 0069066ad7..8be6472ff1 100644 --- a/kernel/x86_64/zgemm3m_kernel_2x8_nehalem.S +++ b/kernel/x86_64/zgemm3m_kernel_2x8_nehalem.S @@ -169,10 +169,10 @@ movq N, J sarq $3, J NOBRANCH - jle .L30 + jle L(30) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -189,10 +189,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -253,10 +253,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm12 @@ -379,10 +379,10 @@ subq $-8 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -392,10 +392,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addpd %xmm1, %xmm12 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm13 @@ -431,10 +431,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): addpd %xmm1, %xmm12 addpd %xmm2, %xmm13 addpd %xmm3, %xmm14 @@ -605,13 +605,13 @@ decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M BRANCH - jle .L29 + jle L(29) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -652,10 +652,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -717,10 +717,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -730,10 +730,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -753,10 +753,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): leaq (LDC, LDC, 2), %rax movsd 0 * SIZE(CO1), %xmm0 @@ -832,7 +832,7 @@ movhps %xmm3, 1 * SIZE(CO2, %rax) ALIGN_4 -.L29: +L(29): #if defined(TRMMKERNEL) && !defined(LEFT) addq $8, KK #endif @@ -843,12 +843,12 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L30: +L(30): testq $4, N - jle .L50 + jle L(50) ALIGN_4 #if defined(TRMMKERNEL) && defined(LEFT) @@ -863,10 +863,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L31: +L(31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -913,10 +913,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -988,10 +988,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -1001,10 +1001,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): addpd %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm9 @@ -1026,10 +1026,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): addpd %xmm1, %xmm8 addpd %xmm2, %xmm9 addpd %xmm3, %xmm10 @@ -1119,13 +1119,13 @@ addq $4 * SIZE, CO2 decq I BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L40: +L(40): testq $1, M BRANCH - jle .L49 + jle L(49) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1166,10 +1166,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1207,10 +1207,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -1220,10 +1220,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1237,10 +1237,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1281,7 +1281,7 @@ movhps %xmm3, 1 * SIZE(CO2, LDC) ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif @@ -1291,9 +1291,9 @@ leaq (C, LDC, 4), C ALIGN_4 -.L50: +L(50): testq $2, N - jle .L70 + jle L(70) ALIGN_4 #if defined(TRMMKERNEL) && defined(LEFT) @@ -1308,10 +1308,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1354,10 +1354,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -1397,13 +1397,13 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 ALIGN_3 -.L55: +L(55): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -1413,10 +1413,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addpd %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm9 @@ -1430,10 +1430,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): addpd %xmm1, %xmm8 addpd %xmm2, %xmm9 @@ -1481,13 +1481,13 @@ addq $4 * SIZE, CO2 decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L69 + jle L(69) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1526,10 +1526,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): mulpd %xmm0, %xmm1 movddup -15 * SIZE(AO), %xmm0 addpd %xmm1, %xmm8 @@ -1555,10 +1555,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -1568,10 +1568,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): mulpd %xmm0, %xmm1 movddup -15 * SIZE(AO), %xmm0 addpd %xmm1, %xmm8 @@ -1582,10 +1582,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addpd %xmm9, %xmm8 movsd 0 * SIZE(CO1), %xmm0 @@ -1607,7 +1607,7 @@ movhps %xmm1, 1 * SIZE(CO2) ALIGN_4 -.L69: +L(69): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -1617,9 +1617,9 @@ leaq (C, LDC, 2), C ALIGN_4 -.L70: +L(70): testq $1, N - jle .L999 + jle L(999) ALIGN_4 #if defined(TRMMKERNEL) && defined(LEFT) @@ -1633,10 +1633,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L80 + jle L(80) ALIGN_4 -.L71: +L(71): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1678,10 +1678,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -1709,12 +1709,12 @@ subq $1, %rax BRANCH - jg .L72 + jg L(72) addpd %xmm9, %xmm8 ALIGN_3 -.L75: +L(75): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -1724,10 +1724,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addpd %xmm1, %xmm8 movddup -16 * SIZE(BO), %xmm1 mulpd %xmm0, %xmm1 @@ -1738,10 +1738,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addpd %xmm1, %xmm8 movsd 0 * SIZE(CO1), %xmm0 @@ -1765,13 +1765,13 @@ addq $4 * SIZE, CO1 decq I BRANCH - jg .L71 + jg L(71) ALIGN_4 -.L80: +L(80): testq $1, M BRANCH - jle .L999 + jle L(999) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1819,10 +1819,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): mulpd %xmm0, %xmm1 #ifndef TRMMKERNEL movapd -14 * SIZE(AO), %xmm0 @@ -1858,12 +1858,12 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) addpd %xmm9, %xmm8 ALIGN_3 -.L85: +L(85): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -1873,10 +1873,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): mulsd %xmm0, %xmm1 movsd -15 * SIZE(AO), %xmm0 addsd %xmm1, %xmm8 @@ -1887,10 +1887,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): haddpd %xmm8, %xmm8 movsd 0 * SIZE(CO1), %xmm0 @@ -1904,7 +1904,7 @@ movhps %xmm0, 1 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm3m_kernel_4x2_atom.S b/kernel/x86_64/zgemm3m_kernel_4x2_atom.S index 1049c01312..49b449746c 100644 --- a/kernel/x86_64/zgemm3m_kernel_4x2_atom.S +++ b/kernel/x86_64/zgemm3m_kernel_4x2_atom.S @@ -141,10 +141,10 @@ movq N, J sarq $1, J - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): movq C, CO1 leaq (C, LDC, 1), CO2 leaq (C, LDC, 2), C @@ -157,10 +157,10 @@ movq M, I sarq $2, I - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): movq B, BO prefetcht0 0 * SIZE(BB) @@ -190,10 +190,10 @@ movq K, %rax sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): addsd %xmm2, %xmm13 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps %xmm0, %xmm2 @@ -336,18 +336,18 @@ mulsd %xmm3, %xmm6 movsd 1 * SIZE(BO), %xmm3 - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): movq K, %rax andq $3, %rax BRANCH BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): addsd %xmm2, %xmm13 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -386,10 +386,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): movsd ALPHA_R, %xmm4 addsd %xmm2, %xmm13 movsd ALPHA_I, %xmm5 @@ -464,12 +464,12 @@ addq $8 * SIZE, CO2 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M - jle .L30 + jle L(30) movq B, BO @@ -490,10 +490,10 @@ movq K, %rax sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addsd %xmm2, %xmm9 movaps %xmm0, %xmm2 @@ -571,10 +571,10 @@ addq $8 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): movq K, %rax movsd ALPHA_R, %xmm5 movsd ALPHA_I, %xmm7 @@ -582,10 +582,10 @@ andq $3, %rax BRANCH BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): addsd %xmm2, %xmm9 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -608,10 +608,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): addsd %xmm2, %xmm9 addsd %xmm6, %xmm11 @@ -653,9 +653,9 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_4 movq B, BO @@ -672,10 +672,10 @@ movq K, %rax sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): addsd %xmm5, %xmm8 movsd 2 * SIZE(BO), %xmm5 mulsd %xmm0, %xmm1 @@ -717,10 +717,10 @@ addq $8 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): movq K, %rax addsd %xmm5, %xmm8 @@ -732,10 +732,10 @@ andq $3, %rax BRANCH BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulsd %xmm0, %xmm1 addq $2 * SIZE, BO mulsd %xmm0, %xmm3 @@ -749,10 +749,10 @@ addq $1 * SIZE, AO decq %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): movaps %xmm8, %xmm10 movaps %xmm9, %xmm11 @@ -772,15 +772,15 @@ movsd %xmm11, 1 * SIZE(CO2) ALIGN_4 -.L39: +L(39): movq BO, B decq J # j -- - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $1, N - je .L999 + je L(999) movq C, CO1 addq LDC, C @@ -789,10 +789,10 @@ movq M, I sarq $2, I - jle .L50 + jle L(50) ALIGN_4 -.L41: +L(41): movq B, BO movsd 0 * SIZE(AO), %xmm0 @@ -814,10 +814,10 @@ movq K, %rax sarq $2, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): addsd %xmm9, %xmm8 movsd 4 * SIZE(AO), %xmm9 mulsd %xmm4, %xmm0 @@ -891,10 +891,10 @@ mulsd %xmm5, %xmm15 movsd 1 * SIZE(BO), %xmm5 - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): movq K, %rax movsd ALPHA_R, %xmm6 @@ -908,10 +908,10 @@ andq $3, %rax BRANCH BRANCH - je .L49 + je L(49) ALIGN_4 -.L46: +L(46): mulsd %xmm4, %xmm0 mulsd %xmm4, %xmm1 mulsd %xmm4, %xmm2 @@ -931,10 +931,10 @@ addq $1 * SIZE, BO decq %rax BRANCH - jg .L46 + jg L(46) ALIGN_4 -.L49: +L(49): movaps %xmm8, %xmm9 movaps %xmm10, %xmm11 movaps %xmm12, %xmm13 @@ -970,12 +970,12 @@ addq $8 * SIZE, CO1 decq I # i -- - jg .L41 + jg L(41) ALIGN_4 -.L50: +L(50): testq $2, M - jle .L60 + jle L(60) movq B, BO @@ -991,10 +991,10 @@ movq K, %rax sarq $2, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): addsd %xmm2, %xmm8 movsd 2 * SIZE(AO), %xmm2 mulsd %xmm4, %xmm0 @@ -1035,10 +1035,10 @@ mulsd %xmm5, %xmm3 movsd 1 * SIZE(BO), %xmm5 - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): movq K, %rax movsd ALPHA_R, %xmm6 movsd ALPHA_I, %xmm7 @@ -1049,10 +1049,10 @@ andq $3, %rax BRANCH BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): mulsd %xmm4, %xmm0 mulsd %xmm4, %xmm1 movsd 1 * SIZE(BO), %xmm4 @@ -1066,10 +1066,10 @@ addq $1 * SIZE, BO decq %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): movaps %xmm8, %xmm9 movaps %xmm10, %xmm11 @@ -1091,9 +1091,9 @@ addq $4 * SIZE, CO1 ALIGN_4 -.L60: +L(60): testq $1, M - je .L999 + je L(999) ALIGN_4 movq B, BO @@ -1112,10 +1112,10 @@ movq K, %rax sarq $2, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): addsd %xmm5, %xmm8 movsd 2 * SIZE(BO), %xmm5 mulsd %xmm0, %xmm1 @@ -1140,13 +1140,13 @@ addq $4 * SIZE, BO decq %rax - jne .L62 + jne L(62) addsd %xmm5, %xmm8 addsd %xmm7, %xmm9 ALIGN_4 -.L65: +L(65): movq K, %rax movsd ALPHA_R, %xmm6 movsd ALPHA_I, %xmm7 @@ -1154,10 +1154,10 @@ andq $3, %rax BRANCH BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): movsd 0 * SIZE(AO), %xmm0 movsd 0 * SIZE(BO), %xmm1 @@ -1169,10 +1169,10 @@ decq %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addsd %xmm9, %xmm8 movaps %xmm8, %xmm9 @@ -1186,7 +1186,7 @@ movsd %xmm9, 1 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm3m_kernel_4x4_barcelona.S b/kernel/x86_64/zgemm3m_kernel_4x4_barcelona.S index 76ed76d811..e833ad69ae 100644 --- a/kernel/x86_64/zgemm3m_kernel_4x4_barcelona.S +++ b/kernel/x86_64/zgemm3m_kernel_4x4_barcelona.S @@ -487,10 +487,10 @@ #endif movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #ifdef BUFFERED leaq 16 * SIZE + BUFFER, BO #endif @@ -505,10 +505,10 @@ #ifdef BUFFERED movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_3 -.L02: +L(02): prefetch (RPREFETCHSIZE + 0) * SIZE(B) movaps (B), %xmm0 @@ -545,17 +545,17 @@ subq $-16 * SIZE, B subq $1, %rax - jne .L02 + jne L(02) ALIGN_3 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_3 -.L04: +L(04): movaps (B), %xmm0 movaps %xmm0, -16 * SIZE(BO) @@ -565,20 +565,20 @@ addq $4 * SIZE, B addq $4 * SIZE, BO subq $1, %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): #endif movq A, AO # aoffset = a movq B, BB movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -643,10 +643,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -656,7 +656,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -666,7 +666,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -676,7 +676,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -686,7 +686,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -696,7 +696,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -706,7 +706,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -716,7 +716,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -726,10 +726,10 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) BRANCH - jl .L12 + jl L(12) ALIGN_4 -.L15: +L(15): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -738,7 +738,7 @@ movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_4 @@ -751,14 +751,14 @@ subq $-16 * SIZE, AO ALIGN_4 -.L16: +L(16): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L19 + je L(19) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -766,7 +766,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -793,10 +793,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L17 + jl L(17) ALIGN_4 -.L19: +L(19): movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 movsd 2 * SIZE(CO1), %xmm1 @@ -942,18 +942,18 @@ decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $3, M - je .L39 + je L(39) testq $2, M - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1003,10 +1003,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L26 + je L(26) ALIGN_4 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -1062,10 +1062,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L22 + jl L(22) ALIGN_4 -.L26: +L(26): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1074,7 +1074,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L29 + je L(29) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -1082,7 +1082,7 @@ negq %rax ALIGN_4 -.L27: +L(27): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -1098,10 +1098,10 @@ movddup -11 * SIZE(BO, %rax, 4), %xmm5 addq $SIZE, %rax - jl .L27 + jl L(27) ALIGN_4 -.L29: +L(29): movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 movsd 2 * SIZE(CO1), %xmm1 @@ -1178,9 +1178,9 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1231,10 +1231,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L36 + je L(36) ALIGN_4 -.L32: +L(32): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO, %rax, 4), %xmm0 addpd %xmm1, %xmm8 @@ -1262,10 +1262,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L32 + jl L(32) ALIGN_4 -.L36: +L(36): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1274,7 +1274,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L38 + je L(38) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -1282,7 +1282,7 @@ negq %rax ALIGN_4 -.L37: +L(37): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO, %rax, 4), %xmm0 addpd %xmm1, %xmm8 @@ -1291,10 +1291,10 @@ movddup -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L37 + jl L(37) ALIGN_4 -.L38: +L(38): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1341,7 +1341,7 @@ ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif @@ -1352,18 +1352,18 @@ leaq (C, LDC, 4), C # c += 4 * ldc decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $3, N - je .L999 + je L(999) testq $2, N - je .L80 + je L(80) ALIGN_4 -.L41: +L(41): #ifdef BUFFERED leaq 16 * SIZE + BUFFER, BO #endif @@ -1376,10 +1376,10 @@ #ifdef BUFFERED movq K, %rax sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): prefetchnta (RPREFETCHSIZE + 0) * SIZE(B) movaps (B), %xmm0 @@ -1400,27 +1400,27 @@ subq $-8 * SIZE, B subq $1, %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): movq K, %rax andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movaps (B), %xmm0 movaps %xmm0, -16 * SIZE(BO) addq $2 * SIZE, B addq $2 * SIZE, BO subq $1, %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): #endif movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc @@ -1428,10 +1428,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1484,10 +1484,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L56 + je L(56) ALIGN_4 -.L52: +L(52): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -1539,10 +1539,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L52 + jl L(52) ALIGN_4 -.L56: +L(56): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1551,7 +1551,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L59 + je L(59) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -1559,7 +1559,7 @@ negq %rax ALIGN_4 -.L57: +L(57): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -1574,10 +1574,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L57 + jl L(57) ALIGN_4 -.L59: +L(59): movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 movsd 2 * SIZE(CO1), %xmm1 @@ -1652,15 +1652,15 @@ addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1709,10 +1709,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L66 + je L(66) ALIGN_4 -.L62: +L(62): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -1744,10 +1744,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L62 + jl L(62) ALIGN_4 -.L66: +L(66): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1756,7 +1756,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L69 + je L(69) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -1764,7 +1764,7 @@ negq %rax ALIGN_4 -.L67: +L(67): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -1774,10 +1774,10 @@ movddup -13 * SIZE(BO, %rax, 2), %xmm3 addq $SIZE, %rax - jl .L67 + jl L(67) ALIGN_4 -.L69: +L(69): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1821,12 +1821,12 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1875,10 +1875,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L76 + je L(76) ALIGN_4 -.L72: +L(72): mulpd -16 * SIZE(BO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -12 * SIZE(AO, %rax, 1), %xmm0 @@ -1897,10 +1897,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L72 + jl L(72) ALIGN_4 -.L76: +L(76): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1909,7 +1909,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L78 + je L(78) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -1917,16 +1917,16 @@ negq %rax ALIGN_4 -.L77: +L(77): mulpd -16 * SIZE(BO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L77 + jl L(77) ALIGN_4 -.L78: +L(78): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -1952,7 +1952,7 @@ movhpd %xmm0, 1 * SIZE(CO2) ALIGN_4 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -1964,12 +1964,12 @@ leaq (C, LDC, 2), C ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L81: +L(81): #ifdef BUFFERED leaq 16 * SIZE + BUFFER, BO #endif @@ -1982,10 +1982,10 @@ #ifdef BUFFERED movq K, %rax sarq $3, %rax - jle .L83 + jle L(83) ALIGN_4 -.L82: +L(82): prefetchnta (RPREFETCHSIZE + 0) * SIZE(B) movaps (B), %xmm0 @@ -2006,37 +2006,37 @@ subq $-8 * SIZE, B subq $1, %rax - jne .L82 + jne L(82) ALIGN_4 -.L83: +L(83): movq K, %rax andq $7, %rax BRANCH - jle .L90 + jle L(90) ALIGN_4 -.L84: +L(84): movsd (B), %xmm0 movlpd %xmm0, -16 * SIZE(BO) addq $1 * SIZE, B addq $1 * SIZE, BO decq %rax - jne .L84 + jne L(84) ALIGN_4 -.L90: +L(90): #endif movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2088,10 +2088,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L96 + je L(96) ALIGN_4 -.L92: +L(92): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -2119,10 +2119,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L92 + jl L(92) ALIGN_4 -.L96: +L(96): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2131,7 +2131,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L99 + je L(99) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -2139,7 +2139,7 @@ negq %rax ALIGN_4 -.L97: +L(97): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -2148,10 +2148,10 @@ movddup -15 * SIZE(BO, %rax, 1), %xmm1 addq $SIZE, %rax - jl .L97 + jl L(97) ALIGN_4 -.L99: +L(99): addpd %xmm9, %xmm8 addpd %xmm13, %xmm12 @@ -2193,15 +2193,15 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2250,10 +2250,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L106 + je L(106) ALIGN_4 -.L102: +L(102): mulpd -16 * SIZE(AO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -12 * SIZE(BO, %rax, 1), %xmm0 @@ -2272,10 +2272,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L102 + jl L(102) ALIGN_4 -.L106: +L(106): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2284,7 +2284,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L109 + je L(109) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -2292,16 +2292,16 @@ negq %rax ALIGN_4 -.L107: +L(107): movddup -16 * SIZE(BO, %rax, 1), %xmm0 mulpd -16 * SIZE(AO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 addq $SIZE, %rax - jl .L107 + jl L(107) ALIGN_4 -.L109: +L(109): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -2327,12 +2327,12 @@ addq $4 * SIZE, CO1 ALIGN_4 -.L110: +L(110): testq $1, M - je .L999 + je L(999) ALIGN_4 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2377,10 +2377,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L116 + je L(116) ALIGN_4 -.L112: +L(112): mulpd -16 * SIZE(BO, %rax, 1), %xmm0 addpd %xmm0, %xmm8 movapd -12 * SIZE(AO, %rax, 1), %xmm0 @@ -2391,10 +2391,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L112 + jl L(112) ALIGN_4 -.L116: +L(116): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2403,7 +2403,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L118 + je L(118) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -2411,16 +2411,16 @@ negq %rax ALIGN_4 -.L117: +L(117): mulsd -16 * SIZE(BO, %rax, 1), %xmm0 addsd %xmm0, %xmm8 movsd -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L117 + jl L(117) ALIGN_4 -.L118: +L(118): addpd %xmm9, %xmm8 haddpd %xmm8, %xmm8 @@ -2436,7 +2436,7 @@ movhpd %xmm0, 1 * SIZE(CO1) ALIGN_3 -.L999: +L(999): movq %rbx, %rsp movq (%rsp), %rbx diff --git a/kernel/x86_64/zgemm3m_kernel_4x4_core2.S b/kernel/x86_64/zgemm3m_kernel_4x4_core2.S index a78890dea6..fdef5f9a4f 100644 --- a/kernel/x86_64/zgemm3m_kernel_4x4_core2.S +++ b/kernel/x86_64/zgemm3m_kernel_4x4_core2.S @@ -165,10 +165,10 @@ movq N, J sarq $2, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): /* Copying to Sub Buffer */ leaq 16 * SIZE + BUFFER, BO @@ -180,10 +180,10 @@ movq K, %rax sarq $2, %rax NOBRANCH - jle .L05 + jle L(05) ALIGN_4 -.L02: +L(02): movapd -16 * SIZE(B), %xmm0 prefetchnta (PREFETCH_R + 0) * SIZE(B) movapd -14 * SIZE(B), %xmm1 @@ -240,18 +240,18 @@ subq $-32 * SIZE, BO decq %rax BRANCH - jne .L02 + jne L(02) ALIGN_4 -.L05: +L(05): movq K, %rax andq $3, %rax BRANCH BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L06: +L(06): movapd -16 * SIZE(B), %xmm0 movapd -14 * SIZE(B), %xmm1 @@ -269,10 +269,10 @@ addq $8 * SIZE, BO decq %rax BRANCH - jne .L06 + jne L(06) ALIGN_4 -.L10: +L(10): leaq (PREFETCH_R + 0) * SIZE(B), BB movq C, CO1 # coffset1 = c @@ -282,10 +282,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -344,10 +344,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): PADDING; addpd %xmm2, %xmm10 movaps -16 * SIZE(BO), %xmm2 @@ -476,10 +476,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): prefetcht2 -8 * SIZE(BB) #ifndef TRMMKERNEL @@ -489,10 +489,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): addpd %xmm2, %xmm10 movaps -16 * SIZE(BO), %xmm2 addpd %xmm3, %xmm14 @@ -527,10 +527,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): movapd ALPHA, %xmm7 addpd %xmm2, %xmm10 @@ -682,13 +682,13 @@ addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -731,10 +731,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_4 -.L21: +L(21): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm2, %xmm8 movapd -16 * SIZE(BO), %xmm2 @@ -796,10 +796,10 @@ subq $-32 * SIZE, BO subq $1, %rax BRANCH - jg .L21 + jg L(21) ALIGN_4 -.L25: +L(25): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -809,10 +809,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): addpd %xmm2, %xmm8 movapd -16 * SIZE(BO), %xmm2 mulpd %xmm0, %xmm2 @@ -831,10 +831,10 @@ addq $8 * SIZE, BO subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): addpd %xmm2, %xmm8 addpd %xmm3, %xmm9 addpd %xmm4, %xmm10 @@ -916,10 +916,10 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $1, M BRANCH - jle .L39 + jle L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -963,10 +963,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_4 -.L31: +L(31): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addsd %xmm2, %xmm8 movsd -16 * SIZE(BO), %xmm2 @@ -1028,10 +1028,10 @@ subq $-32 * SIZE, BO subq $1, %rax BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L35: +L(35): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1041,10 +1041,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): addsd %xmm2, %xmm8 movsd -16 * SIZE(BO), %xmm2 mulsd %xmm0, %xmm2 @@ -1063,10 +1063,10 @@ addq $8 * SIZE, BO subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): addsd %xmm2, %xmm8 addsd %xmm3, %xmm9 addsd %xmm4, %xmm10 @@ -1113,7 +1113,7 @@ movhpd %xmm0, 1 * SIZE(CO2, LDC, 2) ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif @@ -1121,16 +1121,16 @@ leaq (C, LDC, 4), C subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $2, N BRANCH - jle .L80 + jle L(80) ALIGN_4 -.L41: +L(41): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -1141,12 +1141,12 @@ movq K, %rax sarq $3, %rax - jle .L43 + jle L(43) addq %rax, %rax ALIGN_4 -.L42: +L(42): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 movddup -14 * SIZE(B), %xmm10 @@ -1169,17 +1169,17 @@ addq $16 * SIZE, BO subq $1, %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): movq K, %rax andq $7, %rax BRANCH - jle .L45 + jle L(45) ALIGN_4 -.L44: +L(44): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 @@ -1189,20 +1189,20 @@ addq $2 * SIZE, B addq $4 * SIZE, BO subq $1, %rax - jne .L44 + jne L(44) ALIGN_4 -.L45: +L(45): movq C, CO1 leaq (C, LDC, 1), CO2 movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1249,10 +1249,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L55 + jle L(55) ALIGN_4 -.L51: +L(51): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm2, %xmm8 @@ -1320,10 +1320,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L51 + jg L(51) ALIGN_4 -.L55: +L(55): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1332,10 +1332,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): addpd %xmm2, %xmm8 movapd -16 * SIZE(BO), %xmm2 addpd %xmm3, %xmm12 @@ -1355,10 +1355,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): addpd %xmm2, %xmm8 addpd %xmm3, %xmm12 addpd %xmm4, %xmm9 @@ -1437,12 +1437,12 @@ addq $8 * SIZE, CO1 addq $8 * SIZE, CO2 subq $1, I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): testq $2, M - jle .L70 + jle L(70) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1486,10 +1486,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L65 + jle L(65) ALIGN_4 -.L61: +L(61): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm2, %xmm8 @@ -1525,10 +1525,10 @@ subq $ -8 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L61 + jg L(61) ALIGN_4 -.L65: +L(65): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1537,10 +1537,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): addpd %xmm2, %xmm8 movapd -16 * SIZE(BO), %xmm2 mulpd %xmm0, %xmm2 @@ -1552,10 +1552,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addpd %xmm2, %xmm8 addpd %xmm3, %xmm9 addpd %xmm4, %xmm10 @@ -1605,9 +1605,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L70: +L(70): testq $1, M - jle .L79 + jle L(79) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1650,10 +1650,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L75 + jle L(75) ALIGN_4 -.L71: +L(71): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addsd %xmm2, %xmm8 @@ -1689,10 +1689,10 @@ subq $ -4 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L71 + jg L(71) ALIGN_4 -.L75: +L(75): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1701,10 +1701,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): addsd %xmm2, %xmm8 movsd -16 * SIZE(BO), %xmm2 mulsd %xmm0, %xmm2 @@ -1716,10 +1716,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addsd %xmm2, %xmm8 addsd %xmm3, %xmm9 addsd %xmm4, %xmm10 @@ -1749,7 +1749,7 @@ movhpd %xmm0, 1 * SIZE(CO2) ALIGN_4 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -1757,13 +1757,13 @@ leaq (C, LDC, 2), C ALIGN_4 -.L80: +L(80): testq $1, N BRANCH - jle .L999 + jle L(999) ALIGN_4 -.L81: +L(81): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -1774,12 +1774,12 @@ movq K, %rax sarq $4, %rax - jle .L83 + jle L(83) addq %rax, %rax ALIGN_4 -.L82: +L(82): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 movddup -14 * SIZE(B), %xmm10 @@ -1801,17 +1801,17 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO subq $1, %rax - jne .L82 + jne L(82) ALIGN_4 -.L83: +L(83): movq K, %rax andq $15, %rax BRANCH - jle .L85 + jle L(85) ALIGN_4 -.L84: +L(84): movddup -16 * SIZE(B), %xmm8 movapd %xmm8, 0 * SIZE(BO) @@ -1819,19 +1819,19 @@ addq $1 * SIZE, B addq $2 * SIZE, BO subq $1, %rax - jne .L84 + jne L(84) ALIGN_4 -.L85: +L(85): movq C, CO1 movq A, AO movq M, I sarq $2, I - jle .L100 + jle L(100) ALIGN_4 -.L90: +L(90): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1876,10 +1876,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L95 + jle L(95) ALIGN_4 -.L91: +L(91): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm0, %xmm8 @@ -1916,10 +1916,10 @@ subq $-16 * SIZE, AO subq $ -8 * SIZE, BO subq $1, %rax - jg .L91 + jg L(91) ALIGN_4 -.L95: +L(95): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1928,10 +1928,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): addpd %xmm0, %xmm8 movapd -16 * SIZE(AO), %xmm0 mulpd %xmm4, %xmm0 @@ -1943,10 +1943,10 @@ addq $4 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): addpd %xmm0, %xmm8 addpd %xmm1, %xmm12 addpd %xmm2, %xmm9 @@ -1993,12 +1993,12 @@ addq $8 * SIZE, CO1 # coffset += 4 subq $1, I - jg .L90 + jg L(90) ALIGN_4 -.L100: +L(100): testq $2, M - jle .L110 + jle L(110) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2043,10 +2043,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L105 + jle L(105) ALIGN_4 -.L101: +L(101): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm0, %xmm8 @@ -2069,10 +2069,10 @@ subq $-8 * SIZE, AO subq $-8 * SIZE, BO subq $1, %rax - jg .L101 + jg L(101) ALIGN_4 -.L105: +L(105): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2081,10 +2081,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L108 + je L(108) ALIGN_4 -.L106: +L(106): addpd %xmm0, %xmm8 movapd -16 * SIZE(AO), %xmm0 mulpd %xmm4, %xmm0 @@ -2093,10 +2093,10 @@ addq $2 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L106 + jg L(106) ALIGN_4 -.L108: +L(108): addpd %xmm0, %xmm8 addpd %xmm1, %xmm9 addpd %xmm2, %xmm10 @@ -2129,9 +2129,9 @@ addq $4 * SIZE, CO1 ALIGN_4 -.L110: +L(110): testq $1, M - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2176,10 +2176,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L115 + jle L(115) ALIGN_4 -.L111: +L(111): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm0, %xmm8 @@ -2202,10 +2202,10 @@ subq $-4 * SIZE, AO subq $-8 * SIZE, BO subq $1, %rax - jg .L111 + jg L(111) ALIGN_4 -.L115: +L(115): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2214,10 +2214,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): addsd %xmm0, %xmm8 movsd -16 * SIZE(AO), %xmm0 mulsd %xmm4, %xmm0 @@ -2226,10 +2226,10 @@ addq $1 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addsd %xmm0, %xmm8 addsd %xmm1, %xmm9 addsd %xmm2, %xmm10 @@ -2251,7 +2251,7 @@ movhpd %xmm0, 1 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/zgemm3m_kernel_4x4_penryn.S b/kernel/x86_64/zgemm3m_kernel_4x4_penryn.S index b000dc5677..39447d91fb 100644 --- a/kernel/x86_64/zgemm3m_kernel_4x4_penryn.S +++ b/kernel/x86_64/zgemm3m_kernel_4x4_penryn.S @@ -181,10 +181,10 @@ movq N, J sarq $2, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -201,10 +201,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -263,10 +263,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm3, %xmm11 movaps -15 * SIZE(BO), %xmm3 @@ -394,10 +394,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): PREFETCHB -8 * SIZE(BB) #ifndef TRMMKERNEL @@ -407,10 +407,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addpd %xmm3, %xmm11 movaps -15 * SIZE(BO), %xmm3 addpd %xmm4, %xmm15 @@ -446,10 +446,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): movups ALPHA_R, %xmm7 addpd %xmm3, %xmm11 @@ -622,13 +622,13 @@ decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -676,10 +676,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_4 -.L22: +L(22): addpd %xmm3, %xmm11 movaps -15 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -745,10 +745,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -756,10 +756,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): addpd %xmm3, %xmm11 movaps -15 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -780,10 +780,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): movups ALPHA_R, %xmm7 addpd %xmm3, %xmm11 @@ -873,10 +873,10 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $1, M BRANCH - jle .L39 + jle L(39) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -919,10 +919,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) shufps $0x44, %xmm0, %xmm0 @@ -969,10 +969,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -980,10 +980,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): shufps $0x44, %xmm0, %xmm0 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm3 @@ -999,10 +999,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): movups ALPHA_R, %xmm7 addpd %xmm10, %xmm8 @@ -1045,7 +1045,7 @@ movhps %xmm1, 1 * SIZE(CO2, LDC, 2) ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif @@ -1056,13 +1056,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $2, N BRANCH - jle .L80 + jle L(80) movq C, CO1 leaq (C, LDC, 1), CO2 @@ -1080,10 +1080,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1129,10 +1129,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_4 -.L52: +L(52): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 @@ -1208,10 +1208,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_4 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1219,10 +1219,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -1245,10 +1245,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): movups ALPHA_R, %xmm7 movaps %xmm8, %xmm0 @@ -1335,13 +1335,13 @@ addq $8 * SIZE, CO2 decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M BRANCH - jle .L70 + jle L(70) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1382,10 +1382,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x4e, %xmm2, %xmm7 @@ -1428,10 +1428,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1439,10 +1439,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm7 @@ -1457,10 +1457,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): movups ALPHA_R, %xmm7 addpd %xmm10, %xmm8 @@ -1510,10 +1510,10 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L70: +L(70): testq $1, M BRANCH - jle .L79 + jle L(79) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1553,10 +1553,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_4 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) shufps $0x44, %xmm0, %xmm0 @@ -1587,10 +1587,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -1598,10 +1598,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): shufps $0x44, %xmm0, %xmm0 mulpd %xmm0, %xmm2 movsd -15 * SIZE(AO), %xmm0 @@ -1613,10 +1613,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): movups ALPHA_R, %xmm7 addpd %xmm9, %xmm8 @@ -1640,7 +1640,7 @@ movhps %xmm1, 1 * SIZE(CO2) ALIGN_4 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -1649,10 +1649,10 @@ movq BO, B ALIGN_4 -.L80: +L(80): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1665,10 +1665,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1710,10 +1710,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_4 -.L92: +L(92): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm2, %xmm3 @@ -1770,10 +1770,10 @@ subq $ -4 * SIZE, BO subq $1, %rax BRANCH - jg .L92 + jg L(92) ALIGN_4 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -1781,10 +1781,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): pshufd $0x44, %xmm2, %xmm3 pshufd $0x44, %xmm2, %xmm4 movsd -16 * SIZE(BO), %xmm2 @@ -1802,10 +1802,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): movups ALPHA_R, %xmm7 movsd 0 * SIZE(CO1), %xmm0 @@ -1847,13 +1847,13 @@ addq $8 * SIZE, CO1 decq I BRANCH - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M BRANCH - jle .L110 + jle L(110) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1892,10 +1892,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_4 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm2, %xmm3 @@ -1930,10 +1930,10 @@ subq $-4 * SIZE, BO subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_4 -.L105: +L(105): #ifndef TRMMKERNEL movq K, %rax #else @@ -1941,10 +1941,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L108 + je L(108) ALIGN_4 -.L106: +L(106): pshufd $0x44, %xmm2, %xmm3 movsd -16 * SIZE(BO), %xmm2 @@ -1957,10 +1957,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_4 -.L108: +L(108): movups ALPHA_R, %xmm7 addpd %xmm9, %xmm8 @@ -1986,10 +1986,10 @@ addq $4 * SIZE, CO1 ALIGN_4 -.L110: +L(110): testq $1, M BRANCH - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2028,10 +2028,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_4 -.L112: +L(112): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulsd %xmm0, %xmm2 @@ -2058,10 +2058,10 @@ subq $-4 * SIZE, BO subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_4 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2069,10 +2069,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd %xmm0, %xmm2 addsd %xmm2, %xmm8 movsd -15 * SIZE(AO), %xmm0 @@ -2083,10 +2083,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): movups ALPHA_R, %xmm7 addpd %xmm9, %xmm8 @@ -2102,7 +2102,7 @@ movhps %xmm0, 1 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm3m_kernel_4x4_sse2.S b/kernel/x86_64/zgemm3m_kernel_4x4_sse2.S index cb90b4c8fc..cd60717fc2 100644 --- a/kernel/x86_64/zgemm3m_kernel_4x4_sse2.S +++ b/kernel/x86_64/zgemm3m_kernel_4x4_sse2.S @@ -407,10 +407,10 @@ #endif movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_3 -.L01: +L(01): /* Copying to Sub Buffer */ leaq 16 * SIZE + BUFFER, BO movq C, CO1 # coffset1 = c @@ -423,14 +423,14 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_3 #define RPREFETCHSIZE (8 * 7 + 4) #define WPREFETCHSIZE (8 * 8 + 4) -.L02: +L(02): PREFETCH (RPREFETCHSIZE + 0) * SIZE(B) movq 0 * SIZE(B), %mm0 @@ -504,17 +504,17 @@ subq $-16 * SIZE, B subq $1, %rax - jne .L02 + jne L(02) ALIGN_3 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_3 -.L04: +L(04): movq 0 * SIZE(B), %mm0 movq 1 * SIZE(B), %mm1 movq 2 * SIZE(B), %mm2 @@ -532,19 +532,19 @@ addq $4 * SIZE, B addq $8 * SIZE, BO subq $1, %rax - jne .L04 + jne L(04) ALIGN_3 -.L10: +L(10): movq A, AO # aoffset = a leaq (RPREFETCHSIZE + 0) * SIZE(B), BB movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_3 -.L11: +L(11): PREFETCH 0 * SIZE(BB) PREFETCH 8 * SIZE(BB) subq $-16 * SIZE, BB @@ -607,10 +607,10 @@ leaq (BO, %rax, 8), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -631,7 +631,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -653,7 +653,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -675,7 +675,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -697,7 +697,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -719,7 +719,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -741,7 +741,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -763,7 +763,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -785,17 +785,17 @@ addq $8 * SIZE, %rax BRANCH - jl .L12 + jl L(12) ALIGN_3 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_3 @@ -814,10 +814,10 @@ #else sarq $2, %rax NOBRANCH - jle .L16 + jle L(16) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -831,10 +831,10 @@ subq $-16 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) #endif -.L16: +L(16): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -843,7 +843,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L19 + je L(19) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -851,7 +851,7 @@ negq %rax ALIGN_3 -.L17: +L(17): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movapd -14 * SIZE(BO, %rax, 8), %xmm1 @@ -878,10 +878,10 @@ movapd -10 * SIZE(AO, %rax, 4), %xmm2 addq $SIZE, %rax - jl .L17 + jl L(17) ALIGN_3 -.L19: +L(19): movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 movsd 2 * SIZE(CO1), %xmm1 @@ -1026,18 +1026,18 @@ addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_3 -.L20: +L(20): testq $3, M - je .L39 + je L(39) testq $2, M - je .L30 + je L(30) ALIGN_3 -.L21: +L(21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1079,10 +1079,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_3 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1192,10 +1192,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_3 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1204,10 +1204,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_3 -.L26: +L(26): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movapd 2 * SIZE(BO), %xmm1 @@ -1224,10 +1224,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_3 -.L29: +L(29): movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 movsd 2 * SIZE(CO1), %xmm1 @@ -1304,12 +1304,12 @@ addq $4 * SIZE, CO2 ALIGN_3 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_3 -.L31: +L(31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1351,10 +1351,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_3 -.L32: +L(32): mulsd %xmm0, %xmm1 addsd %xmm1, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1463,10 +1463,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_3 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1475,10 +1475,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): mulsd %xmm0, %xmm1 addsd %xmm1, %xmm8 movsd 2 * SIZE(BO), %xmm1 @@ -1495,10 +1495,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 @@ -1540,25 +1540,25 @@ movhpd %xmm0, 1 * SIZE(CO2, LDC, 2) ALIGN_3 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C # c += 4 * ldc decq J # j -- - jg .L01 + jg L(01) ALIGN_3 -.L40: +L(40): testq $3, N - je .L999 + je L(999) testq $2, N - je .L80 + je L(80) ALIGN_4 -.L41: +L(41): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -1569,10 +1569,10 @@ movq K, %rax sarq $2, %rax - jle .L43 + jle L(43) ALIGN_3 -.L42: +L(42): PREFETCH 56 * SIZE(B) movq 0 * SIZE(B), %mm0 @@ -1605,17 +1605,17 @@ movq %mm7, -1 * SIZE(BO) decq %rax - jne .L42 + jne L(42) ALIGN_3 -.L43: +L(43): movq K, %rax andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_3 -.L44: +L(44): movq 0 * SIZE(B), %mm0 movq 1 * SIZE(B), %mm1 @@ -1627,20 +1627,20 @@ addq $2 * SIZE, B addq $4 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_3 -.L50: +L(50): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_3 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1687,10 +1687,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_3 -.L52: +L(52): mulpd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(BO), %xmm0 @@ -1802,10 +1802,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_3 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1814,10 +1814,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_3 -.L56: +L(56): movapd 0 * SIZE(BO), %xmm1 mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 @@ -1834,10 +1834,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L59: +L(59): movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 movsd 2 * SIZE(CO1), %xmm1 @@ -1911,15 +1911,15 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L51 + jg L(51) ALIGN_3 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_3 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1961,10 +1961,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_3 -.L62: +L(62): mulpd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(BO), %xmm0 @@ -2026,10 +2026,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_3 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -2038,10 +2038,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_3 -.L66: +L(66): mulpd %xmm0, %xmm1 mulpd 2 * SIZE(BO), %xmm0 addpd %xmm1, %xmm8 @@ -2052,10 +2052,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_3 -.L69: +L(69): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -2099,12 +2099,12 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_3 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_3 -.L71: +L(71): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2146,10 +2146,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_3 -.L72: +L(72): mulsd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulsd 2 * SIZE(BO), %xmm0 @@ -2210,10 +2210,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_3 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2222,10 +2222,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): mulsd %xmm0, %xmm1 mulsd 2 * SIZE(BO), %xmm0 addsd %xmm1, %xmm8 @@ -2236,10 +2236,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_3 -.L78: +L(78): addsd %xmm10, %xmm8 addsd %xmm11, %xmm9 @@ -2264,19 +2264,19 @@ movhpd %xmm0, 1 * SIZE(CO2) ALIGN_3 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif leaq (C, LDC, 2), C ALIGN_3 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L81: +L(81): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -2287,10 +2287,10 @@ movq K, %rax sarq $3, %rax - jle .L83 + jle L(83) ALIGN_3 -.L82: +L(82): PREFETCH 56 * SIZE(B) movq 0 * SIZE(B), %mm0 @@ -2323,17 +2323,17 @@ movq %mm7, -1 * SIZE(BO) decq %rax - jne .L82 + jne L(82) ALIGN_3 -.L83: +L(83): movq K, %rax andq $7, %rax BRANCH - jle .L90 + jle L(90) ALIGN_3 -.L84: +L(84): movq 0 * SIZE(B), %mm0 movq %mm0, 0 * SIZE(BO) @@ -2342,19 +2342,19 @@ addq $1 * SIZE, B addq $2 * SIZE, BO decq %rax - jne .L84 + jne L(84) ALIGN_3 -.L90: +L(90): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_3 -.L91: +L(91): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2398,10 +2398,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_3 -.L92: +L(92): mulpd %xmm1, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd -14 * SIZE(AO), %xmm1 @@ -2458,10 +2458,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_3 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2470,10 +2470,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_3 -.L96: +L(96): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO), %xmm1 addpd %xmm0, %xmm8 @@ -2484,10 +2484,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_3 -.L99: +L(99): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -2528,15 +2528,15 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L91 + jg L(91) ALIGN_3 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_3 -.L101: +L(101): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2575,10 +2575,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L105 + je L(105) ALIGN_3 -.L102: +L(102): mulpd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -14 * SIZE(AO), %xmm0 @@ -2611,10 +2611,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_3 -.L105: +L(105): #ifndef TRMMKERNEL movq K, %rax #else @@ -2623,10 +2623,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_3 -.L106: +L(106): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movapd -14 * SIZE(AO), %xmm0 @@ -2635,10 +2635,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L106 + jg L(106) ALIGN_3 -.L109: +L(109): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -2664,12 +2664,12 @@ addq $4 * SIZE, CO1 ALIGN_3 -.L110: +L(110): testq $1, M - je .L999 + je L(999) ALIGN_3 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2708,10 +2708,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_3 -.L112: +L(112): mulsd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -15 * SIZE(AO), %xmm0 @@ -2743,10 +2743,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_3 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2755,10 +2755,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_3 -.L116: +L(116): mulsd %xmm0, %xmm1 movsd -15 * SIZE(AO), %xmm0 addsd %xmm1, %xmm8 @@ -2767,10 +2767,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_3 -.L118: +L(118): addsd %xmm10, %xmm8 addsd %xmm11, %xmm9 addsd %xmm9, %xmm8 @@ -2787,7 +2787,7 @@ movhpd %xmm0, 1 * SIZE(CO1) ALIGN_3 -.L999: +L(999): movq %rbx, %rsp EMMS diff --git a/kernel/x86_64/zgemm3m_kernel_4x4_sse3.S b/kernel/x86_64/zgemm3m_kernel_4x4_sse3.S index ce46dbdf05..efe0c1294a 100644 --- a/kernel/x86_64/zgemm3m_kernel_4x4_sse3.S +++ b/kernel/x86_64/zgemm3m_kernel_4x4_sse3.S @@ -382,10 +382,10 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -400,10 +400,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): prefetcht0 0 * SIZE(BB) prefetcht0 8 * SIZE(BB) subq $-8 * SIZE, BB @@ -463,9 +463,9 @@ andq $-8, %rax salq $4, %rax NOBRANCH - je .L15 + je L(15) -.L1X: +L(1X): KERNEL1 (16 * 0) KERNEL2 (16 * 0) KERNEL3 (16 * 0) @@ -484,7 +484,7 @@ KERNEL16(16 * 0) cmpq $128 * 1, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 1) KERNEL2 (16 * 1) KERNEL3 (16 * 1) @@ -503,7 +503,7 @@ KERNEL16(16 * 1) cmpq $128 * 2, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 2) KERNEL2 (16 * 2) KERNEL3 (16 * 2) @@ -522,7 +522,7 @@ KERNEL16(16 * 2) cmpq $128 * 3, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 3) KERNEL2 (16 * 3) KERNEL3 (16 * 3) @@ -541,7 +541,7 @@ KERNEL16(16 * 3) cmpq $128 * 4, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 4) KERNEL2 (16 * 4) KERNEL3 (16 * 4) @@ -560,7 +560,7 @@ KERNEL16(16 * 4) cmpq $128 * 5, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 5) KERNEL2 (16 * 5) KERNEL3 (16 * 5) @@ -579,7 +579,7 @@ KERNEL16(16 * 5) cmpq $128 * 6, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 6) KERNEL2 (16 * 6) KERNEL3 (16 * 6) @@ -598,7 +598,7 @@ KERNEL16(16 * 6) cmpq $128 * 7, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 7) KERNEL2 (16 * 7) KERNEL3 (16 * 7) @@ -620,18 +620,18 @@ addq $32 * 8 * SIZE, BO subq $128 * 8, %rax BRANCH - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 2), BO # * 64 #else sarq $3, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -858,11 +858,11 @@ addq $32 * SIZE, AO decq %rax BRANCH - jne .L12 + jne L(12) #endif ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else @@ -873,10 +873,10 @@ andq $7, %rax # if (k & 1) BRANCH BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 addpd %xmm9, %xmm0 @@ -908,10 +908,10 @@ addq $4 * SIZE, BO # boffset1 += 8 decq %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): movsd 0 * SIZE(CO1), %xmm8 movhpd 1 * SIZE(CO1), %xmm8 movsd 2 * SIZE(CO1), %xmm9 @@ -1056,17 +1056,17 @@ addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 - jmp .L20 + jg L(11) + jmp L(20) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1104,10 +1104,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1217,10 +1217,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1230,10 +1230,10 @@ movhpd ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -1251,10 +1251,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): movsd 0 * SIZE(CO1), %xmm8 movhpd 1 * SIZE(CO1), %xmm8 movsd 2 * SIZE(CO1), %xmm9 @@ -1331,12 +1331,12 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_4 -.L31: +L(31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1374,10 +1374,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1439,10 +1439,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1452,10 +1452,10 @@ movhpd ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -1467,10 +1467,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): movsd 0 * SIZE(CO1), %xmm8 movhpd 1 * SIZE(CO1), %xmm8 @@ -1516,7 +1516,7 @@ movhpd %xmm8, 1 * SIZE(CO2, LDC, 2) ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif @@ -1524,12 +1524,12 @@ leaq (C, LDC, 4), C # c += 4 * ldc movq BO, B decq J # j -- - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $2, N - je .L80 + je L(80) ALIGN_4 #if defined(TRMMKERNEL) && defined(LEFT) @@ -1543,10 +1543,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1592,10 +1592,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1714,10 +1714,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1727,10 +1727,10 @@ movhpd ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 addpd %xmm9, %xmm0 @@ -1749,10 +1749,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): movsd 0 * SIZE(CO1), %xmm8 movhpd 1 * SIZE(CO1), %xmm8 movsd 2 * SIZE(CO1), %xmm9 @@ -1827,15 +1827,15 @@ addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1873,10 +1873,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1938,10 +1938,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1951,10 +1951,10 @@ movhpd ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_4 -.L66: +L(66): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -1966,10 +1966,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L69: +L(69): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -2013,12 +2013,12 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2056,10 +2056,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movddup 1 * SIZE(AO), %xmm8 @@ -2091,10 +2091,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2104,10 +2104,10 @@ movhpd ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulpd %xmm8, %xmm9 movddup 1 * SIZE(AO), %xmm8 addpd %xmm9, %xmm0 @@ -2116,10 +2116,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -2146,7 +2146,7 @@ movhpd %xmm8, 1 * SIZE(CO2) ALIGN_4 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -2154,9 +2154,9 @@ movq BO, B ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 #if defined(TRMMKERNEL) && defined(LEFT) @@ -2169,10 +2169,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2216,10 +2216,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(AO), %xmm9 @@ -2274,10 +2274,10 @@ addq $32 * SIZE, AO addq $8 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2287,10 +2287,10 @@ movhpd ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_4 -.L96: +L(96): mulpd %xmm9, %xmm8 mulpd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -2301,10 +2301,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L99: +L(99): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -2345,15 +2345,15 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2391,10 +2391,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L105 + je L(105) ALIGN_4 -.L102: +L(102): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movddup 1 * SIZE(BO), %xmm9 @@ -2426,10 +2426,10 @@ addq $16 * SIZE, AO addq $ 8 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L105: +L(105): #ifndef TRMMKERNEL movq K, %rax #else @@ -2439,10 +2439,10 @@ movhpd ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_4 -.L106: +L(106): mulpd %xmm9, %xmm8 movddup 1 * SIZE(BO), %xmm9 addpd %xmm8, %xmm0 @@ -2451,10 +2451,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L106 + jg L(106) ALIGN_4 -.L109: +L(109): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -2480,12 +2480,12 @@ addq $4 * SIZE, CO1 ALIGN_4 -.L110: +L(110): testq $1, M - je .L999 + je L(999) ALIGN_4 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2528,10 +2528,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulpd %xmm9, %xmm8 movapd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -2550,10 +2550,10 @@ addq $8 * SIZE, AO addq $8 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2563,10 +2563,10 @@ movhpd ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd 0 * SIZE(BO), %xmm9 addsd %xmm9, %xmm0 movsd 1 * SIZE(AO), %xmm9 @@ -2574,10 +2574,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addpd %xmm1, %xmm0 haddpd %xmm0, %xmm0 @@ -2593,7 +2593,7 @@ movhpd %xmm8, 1 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm3m_kernel_4x8_nehalem.S b/kernel/x86_64/zgemm3m_kernel_4x8_nehalem.S index 8da31d22d8..e7ae5a7065 100644 --- a/kernel/x86_64/zgemm3m_kernel_4x8_nehalem.S +++ b/kernel/x86_64/zgemm3m_kernel_4x8_nehalem.S @@ -164,10 +164,10 @@ movq N, J sarq $3, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -184,10 +184,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -248,10 +248,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm12 @@ -374,10 +374,10 @@ subq $-16 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -387,10 +387,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addps %xmm1, %xmm12 movaps -32 * SIZE(BO), %xmm1 addps %xmm2, %xmm13 @@ -426,10 +426,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): addps %xmm1, %xmm12 addps %xmm2, %xmm13 addps %xmm3, %xmm14 @@ -607,13 +607,13 @@ addq $8 * SIZE, CO2 decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -658,10 +658,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -739,10 +739,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -752,10 +752,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -779,10 +779,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): addps %xmm1, %xmm8 addps %xmm2, %xmm9 addps %xmm3, %xmm10 @@ -864,10 +864,10 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L30: +L(30): testq $1, M BRANCH - jle .L39 + jle L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -906,10 +906,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movaps -32 * SIZE(BO), %xmm2 @@ -949,10 +949,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -962,10 +962,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -980,10 +980,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): addps %xmm2, %xmm8 addps %xmm3, %xmm12 @@ -1025,7 +1025,7 @@ movhps %xmm3, (CO2, %rax) ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addq $8, KK #endif @@ -1036,12 +1036,12 @@ subq $1, J BRANCH - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $4, N - jle .L70 + jle L(70) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1055,10 +1055,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L50 + jle L(50) ALIGN_4 -.L41: +L(41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1105,10 +1105,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -1175,10 +1175,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -1188,10 +1188,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): addps %xmm1, %xmm8 movaps -32 * SIZE(BO), %xmm1 addps %xmm2, %xmm9 @@ -1212,10 +1212,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): addps %xmm1, %xmm8 addps %xmm2, %xmm9 addps %xmm3, %xmm10 @@ -1308,13 +1308,13 @@ addq $8 * SIZE, CO2 decq I BRANCH - jg .L41 + jg L(41) ALIGN_4 -.L50: +L(50): testq $2, M BRANCH - jle .L60 + jle L(60) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1354,10 +1354,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1399,10 +1399,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -1412,10 +1412,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1430,10 +1430,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): addps %xmm1, %xmm8 addps %xmm2, %xmm9 @@ -1476,10 +1476,10 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L69 + jle L(69) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1517,10 +1517,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movaps -32 * SIZE(BO), %xmm2 @@ -1548,11 +1548,11 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) addps %xmm9, %xmm8 ALIGN_3 -.L65: +L(65): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -1562,10 +1562,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -1577,10 +1577,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): addps %xmm2, %xmm8 movsd (CO1), %xmm0 @@ -1603,7 +1603,7 @@ movhps %xmm1, (CO2, LDC) ALIGN_4 -.L69: +L(69): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif @@ -1613,9 +1613,9 @@ leaq (C, LDC, 4), C ALIGN_4 -.L70: +L(70): testq $2, N - jle .L100 + jle L(100) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1629,10 +1629,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L80 + jle L(80) ALIGN_4 -.L71: +L(71): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1674,10 +1674,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -1720,10 +1720,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_3 -.L75: +L(75): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -1733,10 +1733,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addps %xmm1, %xmm8 pshufd $0x00, %xmm3, %xmm1 mulps %xmm0, %xmm1 @@ -1751,10 +1751,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_3 -.L78: +L(78): addps %xmm1, %xmm8 addps %xmm2, %xmm9 @@ -1797,13 +1797,13 @@ addq $8 * SIZE, CO2 decq I BRANCH - jg .L71 + jg L(71) ALIGN_4 -.L80: +L(80): testq $2, M BRANCH - jle .L90 + jle L(90) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1843,10 +1843,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): addps %xmm1, %xmm8 movsd -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -1876,10 +1876,10 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) ALIGN_3 -.L85: +L(85): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -1889,10 +1889,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): addps %xmm1, %xmm8 movsd -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -1904,10 +1904,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_3 -.L88: +L(88): addps %xmm1, %xmm8 movsd 0 * SIZE(CO1), %xmm0 @@ -1933,10 +1933,10 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L90: +L(90): testq $1, M BRANCH - jle .L99 + jle L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1974,10 +1974,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_3 -.L92: +L(92): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movsd -32 * SIZE(BO), %xmm2 @@ -2005,11 +2005,11 @@ subq $1, %rax BRANCH - jg .L92 + jg L(92) addps %xmm9, %xmm8 ALIGN_3 -.L95: +L(95): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -2019,10 +2019,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_3 -.L96: +L(96): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -2034,10 +2034,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_3 -.L98: +L(98): addps %xmm2, %xmm8 movsd (CO1), %xmm0 @@ -2052,7 +2052,7 @@ movhps %xmm0, (CO2) ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -2062,9 +2062,9 @@ leaq (C, LDC, 2), C ALIGN_4 -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -2077,10 +2077,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L110 + jle L(110) ALIGN_4 -.L101: +L(101): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2118,10 +2118,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_3 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -2152,10 +2152,10 @@ subq $ -4 * SIZE, BO subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_3 -.L105: +L(105): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -2165,10 +2165,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L108 + je L(108) ALIGN_3 -.L106: +L(106): addps %xmm1, %xmm8 pshufd $0x00, %xmm3, %xmm1 movss -31 * SIZE(BO), %xmm3 @@ -2180,10 +2180,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_3 -.L108: +L(108): addps %xmm1, %xmm8 movsd 0 * SIZE(CO1), %xmm0 @@ -2207,13 +2207,13 @@ addq $8 * SIZE, CO1 decq I BRANCH - jg .L101 + jg L(101) ALIGN_4 -.L110: +L(110): testq $2, M BRANCH - jle .L120 + jle L(120) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2250,10 +2250,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_3 -.L112: +L(112): addps %xmm1, %xmm8 movss -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -2283,10 +2283,10 @@ subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_3 -.L115: +L(115): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -2296,10 +2296,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_3 -.L116: +L(116): addps %xmm1, %xmm8 movss -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -2311,10 +2311,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_3 -.L118: +L(118): addps %xmm1, %xmm8 movsd 0 * SIZE(CO1), %xmm0 @@ -2331,10 +2331,10 @@ addq $4 * SIZE, CO1 ALIGN_4 -.L120: +L(120): testq $1, M BRANCH - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2371,10 +2371,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L125 + jle L(125) ALIGN_3 -.L122: +L(122): addss %xmm2, %xmm8 movss -32 * SIZE(BO), %xmm2 mulss %xmm0, %xmm2 @@ -2400,10 +2400,10 @@ subq $1, %rax BRANCH - jg .L122 + jg L(122) ALIGN_3 -.L125: +L(125): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -2413,10 +2413,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_3 -.L126: +L(126): addss %xmm2, %xmm8 movss -32 * SIZE(BO), %xmm2 mulss %xmm0, %xmm2 @@ -2427,10 +2427,10 @@ subq $1, %rax BRANCH - jg .L126 + jg L(126) ALIGN_3 -.L128: +L(128): addps %xmm2, %xmm8 movsd (CO1), %xmm0 @@ -2443,7 +2443,7 @@ movlps %xmm0, (CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm3m_kernel_8x4_barcelona.S b/kernel/x86_64/zgemm3m_kernel_8x4_barcelona.S index 3dbc0dd8c8..b185c0ad92 100644 --- a/kernel/x86_64/zgemm3m_kernel_8x4_barcelona.S +++ b/kernel/x86_64/zgemm3m_kernel_8x4_barcelona.S @@ -479,9 +479,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L50 + jle L(50) -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -492,10 +492,10 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): prefetch (RPREFETCHSIZE + 0) * SIZE(B) movaps 0 * SIZE(B), %xmm3 @@ -553,17 +553,17 @@ addq $64 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movaps 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -579,10 +579,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): movq C, CO1 leaq (C, LDC, 1), CO2 movq A, AO @@ -591,10 +591,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -655,10 +655,10 @@ leaq (BO, %rax, 8), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_3 -.L12: +L(12): KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -668,7 +668,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -678,7 +678,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -688,7 +688,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -698,7 +698,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -708,7 +708,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -718,7 +718,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -728,7 +728,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -738,10 +738,10 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) BRANCH - jl .L12 + jl L(12) ALIGN_4 -.L15: +L(15): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -750,7 +750,7 @@ movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_3 @@ -763,14 +763,14 @@ addq $64 * SIZE, BO ALIGN_3 -.L16: +L(16): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L18 + je L(18) leaq (, %rax, 8), %rax leaq (AO, %rax, 4), AO @@ -778,7 +778,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulps %xmm1, %xmm0 mulps -28 * SIZE(AO, %rax, 4), %xmm1 addps %xmm0, %xmm8 @@ -805,10 +805,10 @@ movaps %xmm0, %xmm2 addq $SIZE * 2, %rax - jl .L17 + jl L(17) ALIGN_4 -.L18: +L(18): movups 0 * SIZE(CO1), %xmm0 movups 4 * SIZE(CO1), %xmm1 movups 8 * SIZE(CO1), %xmm2 @@ -928,12 +928,12 @@ addq $16 * SIZE, CO1 # coffset += 4 addq $16 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -977,10 +977,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1090,10 +1090,10 @@ addq $ 32 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1102,10 +1102,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1122,10 +1122,10 @@ addq $ 4 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): movups 0 * SIZE(CO1), %xmm8 movups 4 * SIZE(CO1), %xmm9 @@ -1198,9 +1198,9 @@ addq $8 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $2, M - je .L40 + je L(40) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1244,10 +1244,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1364,10 +1364,10 @@ addq $ 16 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1376,10 +1376,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movsd 4 * SIZE(BO), %xmm9 @@ -1397,10 +1397,10 @@ addq $ 2 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): movups 0 * SIZE(CO1), %xmm8 pshufd $0x50, %xmm0, %xmm4 @@ -1441,9 +1441,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L40: +L(40): testq $1, M - je .L49 + je L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1488,10 +1488,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): mulss %xmm8, %xmm9 addss %xmm9, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1608,10 +1608,10 @@ addq $ 8 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1620,10 +1620,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -1641,10 +1641,10 @@ addq $ 1 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): movsd 0 * SIZE(CO1), %xmm8 pshufd $0x50, %xmm0, %xmm4 @@ -1678,19 +1678,19 @@ movlps %xmm4, 0 * SIZE(CO2, LDC, 2) ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C # c += 4 * ldc decq J # j -- - jg .L01 + jg L(01) -.L50: +L(50): testq $2, N - je .L100 + je L(100) -.L51: +L(51): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1701,10 +1701,10 @@ movq K, %rax sarq $2, %rax - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): prefetch (RPREFETCHSIZE + 0) * SIZE(B) movaps 0 * SIZE(B), %xmm3 @@ -1737,17 +1737,17 @@ addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): movq K, %rax andq $3, %rax BRANCH - jle .L60 + jle L(60) ALIGN_4 -.L54: +L(54): movsd 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -1764,20 +1764,20 @@ addq $ 2 * SIZE, B addq $ 8 * SIZE, BO decq %rax - jne .L54 + jne L(54) ALIGN_4 -.L60: +L(60): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L70 + jle L(70) ALIGN_4 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1825,10 +1825,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulps 4 * SIZE(BO), %xmm8 @@ -1941,10 +1941,10 @@ addq $64 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1953,10 +1953,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -1973,10 +1973,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): movups 0 * SIZE(CO1), %xmm8 movups 4 * SIZE(CO1), %xmm9 movups 8 * SIZE(CO1), %xmm10 @@ -2038,12 +2038,12 @@ addq $16 * SIZE, CO1 # coffset += 4 addq $16 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L61 + jg L(61) ALIGN_4 -.L70: +L(70): testq $4, M - je .L80 + je L(80) #if !defined(TRMMKERNEL) || \ @@ -2088,10 +2088,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulps 4 * SIZE(BO), %xmm8 @@ -2152,10 +2152,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2164,10 +2164,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -2178,10 +2178,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -2221,9 +2221,9 @@ addq $8 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L80: +L(80): testq $2, M - je .L90 + je L(90) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2267,10 +2267,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L85 + je L(85) ALIGN_4 -.L82: +L(82): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2339,10 +2339,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L85: +L(85): #ifndef TRMMKERNEL movq K, %rax #else @@ -2351,10 +2351,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movsd 4 * SIZE(BO), %xmm9 @@ -2366,10 +2366,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -2397,9 +2397,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L90: +L(90): testq $1, M - je .L99 + je L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2443,10 +2443,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2515,10 +2515,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2527,10 +2527,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -2542,10 +2542,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): addss %xmm2, %xmm0 addss %xmm3, %xmm1 @@ -2568,7 +2568,7 @@ movlps %xmm2, 0 * SIZE(CO2) ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -2576,11 +2576,11 @@ ALIGN_4 -.L100: +L(100): testq $1, N - je .L999 + je L(999) -.L101: +L(101): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -2591,11 +2591,11 @@ movq K, %rax sarq $3, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): prefetch (RPREFETCHSIZE + 0) * SIZE(B) movups 0 * SIZE(B), %xmm3 @@ -2628,17 +2628,17 @@ addq $32 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): movq K, %rax andq $7, %rax BRANCH - jle .L110 + jle L(110) ALIGN_4 -.L104: +L(104): movss 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -2648,19 +2648,19 @@ addq $ 1 * SIZE, B addq $ 4 * SIZE, BO decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L110: +L(110): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L120 + jle L(120) ALIGN_4 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2707,10 +2707,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulps %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulps -28 * SIZE(AO), %xmm9 @@ -2774,10 +2774,10 @@ addq $64 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2786,10 +2786,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulps %xmm9, %xmm8 mulps -28 * SIZE(AO), %xmm9 addps %xmm8, %xmm0 @@ -2800,10 +2800,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): movups 0 * SIZE(CO1), %xmm8 movups 4 * SIZE(CO1), %xmm9 movups 8 * SIZE(CO1), %xmm10 @@ -2835,12 +2835,12 @@ addq $16 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L111 + jg L(111) ALIGN_4 -.L120: +L(120): testq $4, M - je .L130 + je L(130) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2882,10 +2882,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L125 + je L(125) ALIGN_4 -.L122: +L(122): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -28 * SIZE(AO), %xmm8 @@ -2919,10 +2919,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L122 + jne L(122) ALIGN_4 -.L125: +L(125): #ifndef TRMMKERNEL movq K, %rax #else @@ -2931,10 +2931,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): mulps %xmm8, %xmm9 movaps -28 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -2943,10 +2943,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): addps %xmm1, %xmm0 addps %xmm3, %xmm2 addps %xmm2, %xmm0 @@ -2970,9 +2970,9 @@ addq $8 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L130: +L(130): testq $2, M - je .L140 + je L(140) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3014,10 +3014,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L135 + je L(135) ALIGN_4 -.L132: +L(132): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -30 * SIZE(AO), %xmm8 @@ -3061,10 +3061,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L132 + jne L(132) ALIGN_4 -.L135: +L(135): #ifndef TRMMKERNEL movq K, %rax #else @@ -3073,10 +3073,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): mulps %xmm8, %xmm9 movsd -30 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -3085,10 +3085,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): addps %xmm1, %xmm0 movups 0 * SIZE(CO1), %xmm8 @@ -3103,9 +3103,9 @@ addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L140: +L(140): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3147,10 +3147,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L145 + je L(145) ALIGN_4 -.L142: +L(142): mulss %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movss -31 * SIZE(AO), %xmm8 @@ -3182,10 +3182,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L142 + jne L(142) ALIGN_4 -.L145: +L(145): #ifndef TRMMKERNEL movq K, %rax #else @@ -3194,10 +3194,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): mulss %xmm8, %xmm9 movss -31 * SIZE(AO), %xmm8 addss %xmm9, %xmm0 @@ -3206,10 +3206,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): addss %xmm1, %xmm0 addss %xmm3, %xmm2 addss %xmm2, %xmm0 @@ -3223,7 +3223,7 @@ movlps %xmm2, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/zgemm3m_kernel_8x4_core2.S b/kernel/x86_64/zgemm3m_kernel_8x4_core2.S index 0b97d85a9a..65618b4c96 100644 --- a/kernel/x86_64/zgemm3m_kernel_8x4_core2.S +++ b/kernel/x86_64/zgemm3m_kernel_8x4_core2.S @@ -166,10 +166,10 @@ movq N, J sarq $2, J - jle .L50 + jle L(50) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -180,10 +180,10 @@ movq K, %rax sarq $2, %rax - jle .L05 + jle L(05) ALIGN_4 -.L02: +L(02): prefetcht0 (PREFETCH_R + 0) * SIZE(B) movaps -32 * SIZE(B), %xmm3 movaps -28 * SIZE(B), %xmm7 @@ -233,17 +233,17 @@ subq $-16 * SIZE, B subq $-64 * SIZE, BO subq $1, %rax - jne .L02 + jne L(02) ALIGN_4 -.L05: +L(05): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L06: +L(06): movaps -32 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -259,10 +259,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO subq $1, %rax - jne .L06 + jne L(06) ALIGN_4 -.L10: +L(10): movq B, BB movq C, CO1 # coffset1 = c @@ -271,10 +271,10 @@ movq M, I sarq $3, I - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -331,10 +331,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): addps %xmm2, %xmm10 movaps -32 * SIZE(BO), %xmm2 addps %xmm3, %xmm14 @@ -463,10 +463,10 @@ subq $-64 * SIZE, BO subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else @@ -474,10 +474,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): addps %xmm2, %xmm10 movaps -32 * SIZE(BO), %xmm2 addps %xmm3, %xmm14 @@ -512,10 +512,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): movaps ALPHA, %xmm7 addps %xmm2, %xmm10 @@ -658,12 +658,12 @@ addq $16 * SIZE, CO1 addq $16 * SIZE, CO2 subq $1, I - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M - jle .L30 + jle L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -699,10 +699,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L25 + jle L(25) ALIGN_4 -.L21: +L(21): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -772,10 +772,10 @@ subq $-16 * SIZE, AO subq $-64 * SIZE, BO subq $1, %rax - jg .L21 + jg L(21) ALIGN_4 -.L25: +L(25): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -784,10 +784,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): movaps -32 * SIZE(AO), %xmm0 movaps -32 * SIZE(BO), %xmm2 movaps -28 * SIZE(BO), %xmm3 @@ -807,10 +807,10 @@ addq $ 4 * SIZE, AO addq $16 * SIZE, BO subq $1, %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 movsd 4 * SIZE(CO1), %xmm1 @@ -887,9 +887,9 @@ addq $8 * SIZE, CO2 ALIGN_4 -.L30: +L(30): testq $2, M - jle .L40 + jle L(40) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -930,10 +930,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L35 + jle L(35) ALIGN_4 -.L31: +L(31): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -32 * SIZE(AO), %xmm0 @@ -1003,10 +1003,10 @@ subq $ -8 * SIZE, AO subq $-64 * SIZE, BO subq $1, %rax - jg .L31 + jg L(31) ALIGN_4 -.L35: +L(35): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1015,10 +1015,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): movsd -32 * SIZE(AO), %xmm0 movsd -32 * SIZE(BO), %xmm2 movsd -28 * SIZE(BO), %xmm3 @@ -1038,10 +1038,10 @@ addq $ 2 * SIZE, AO addq $16 * SIZE, BO subq $1, %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -1086,9 +1086,9 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L40: +L(40): testq $1, M - jle .L49 + jle L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1130,10 +1130,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L45 + jle L(45) ALIGN_4 -.L41: +L(41): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movss -32 * SIZE(AO), %xmm0 @@ -1203,10 +1203,10 @@ subq $ -4 * SIZE, AO subq $-64 * SIZE, BO subq $1, %rax - jg .L41 + jg L(41) ALIGN_4 -.L45: +L(45): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1215,10 +1215,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): movss -32 * SIZE(AO), %xmm0 movss -32 * SIZE(BO), %xmm2 movss -28 * SIZE(BO), %xmm3 @@ -1238,10 +1238,10 @@ addq $ 1 * SIZE, AO addq $16 * SIZE, BO subq $1, %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): movsd 0 * SIZE(CO1), %xmm0 pshufd $0x50, %xmm8, %xmm4 @@ -1275,22 +1275,22 @@ movlps %xmm4, 0 * SIZE(CO2, LDC, 2) ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C subq $1, J - jg .L01 + jg L(01) ALIGN_4 -.L50: +L(50): testq $2, N - jle .L100 + jle L(100) ALIGN_4 -.L51: +L(51): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -1301,12 +1301,12 @@ movq K, %rax sarq $3, %rax - jle .L53 + jle L(53) addq %rax, %rax ALIGN_4 -.L52: +L(52): movaps -32 * SIZE(B), %xmm3 movaps -28 * SIZE(B), %xmm7 @@ -1336,17 +1336,17 @@ addq $32 * SIZE, BO subq $1, %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): movq K, %rax andq $7, %rax BRANCH - jle .L55 + jle L(55) ALIGN_4 -.L54: +L(54): movss -32 * SIZE(B), %xmm8 movss -31 * SIZE(B), %xmm9 @@ -1359,20 +1359,20 @@ addq $2 * SIZE, B addq $8 * SIZE, BO subq $1, %rax - jne .L54 + jne L(54) ALIGN_4 -.L55: +L(55): movq C, CO1 leaq (C, LDC, 1), CO2 movq A, AO # aoffset = a movq M, I sarq $3, I - jle .L70 + jle L(70) ALIGN_4 -.L60: +L(60): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1410,10 +1410,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L65 + jle L(65) ALIGN_4 -.L61: +L(61): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -1491,10 +1491,10 @@ subq $-32 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jg .L61 + jg L(61) ALIGN_4 -.L65: +L(65): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1503,10 +1503,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): movaps -32 * SIZE(AO), %xmm0 movaps -28 * SIZE(AO), %xmm1 @@ -1528,10 +1528,10 @@ addq $8 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 movsd 4 * SIZE(CO1), %xmm1 @@ -1601,12 +1601,12 @@ addq $16 * SIZE, CO1 addq $16 * SIZE, CO2 subq $1, I - jg .L60 + jg L(60) ALIGN_4 -.L70: +L(70): testq $4, M - jle .L80 + jle L(80) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1642,10 +1642,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L75 + jle L(75) ALIGN_4 -.L71: +L(71): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -1685,10 +1685,10 @@ subq $-16 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jg .L71 + jg L(71) ALIGN_4 -.L75: +L(75): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1697,10 +1697,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): movaps -32 * SIZE(AO), %xmm0 movaps -32 * SIZE(BO), %xmm2 movaps -28 * SIZE(BO), %xmm3 @@ -1714,10 +1714,10 @@ addq $4 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm10, %xmm8 addps %xmm11, %xmm9 @@ -1761,9 +1761,9 @@ addq $8 * SIZE, CO2 ALIGN_4 -.L80: +L(80): testq $2, M - jle .L90 + jle L(90) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1799,10 +1799,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L85 + jle L(85) ALIGN_4 -.L81: +L(81): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -32 * SIZE(AO), %xmm0 @@ -1842,10 +1842,10 @@ subq $ -8 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jg .L81 + jg L(81) ALIGN_4 -.L85: +L(85): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1854,10 +1854,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): movsd -32 * SIZE(AO), %xmm0 movsd -32 * SIZE(BO), %xmm2 movsd -28 * SIZE(BO), %xmm3 @@ -1871,10 +1871,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): addps %xmm10, %xmm8 addps %xmm11, %xmm9 @@ -1904,9 +1904,9 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L90: +L(90): testq $1, M - jle .L99 + jle L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1942,10 +1942,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L95 + jle L(95) ALIGN_4 -.L91: +L(91): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1986,10 +1986,10 @@ subq $ -4 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jg .L91 + jg L(91) ALIGN_4 -.L95: +L(95): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1998,10 +1998,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): movss -32 * SIZE(AO), %xmm0 movss -32 * SIZE(BO), %xmm2 movss -28 * SIZE(BO), %xmm3 @@ -2015,10 +2015,10 @@ addq $1 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): addss %xmm10, %xmm8 addss %xmm11, %xmm9 @@ -2041,7 +2041,7 @@ movlps %xmm4, 0 * SIZE(CO2) ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -2051,12 +2051,12 @@ -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) ALIGN_4 -.L101: +L(101): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -2067,12 +2067,12 @@ movq K, %rax sarq $4, %rax - jle .L103 + jle L(103) addq %rax, %rax ALIGN_4 -.L102: +L(102): movss -32 * SIZE(B), %xmm0 movss -31 * SIZE(B), %xmm1 movss -30 * SIZE(B), %xmm2 @@ -2103,17 +2103,17 @@ addq $ 8 * SIZE, B subq $-32 * SIZE, BO subq $1, %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): movq K, %rax andq $15, %rax BRANCH - jle .L105 + jle L(105) ALIGN_4 -.L104: +L(104): movss -32 * SIZE(B), %xmm8 shufps $0, %xmm8, %xmm8 @@ -2123,19 +2123,19 @@ addq $1 * SIZE, B addq $4 * SIZE, BO subq $1, %rax - jne .L104 + jne L(104) ALIGN_4 -.L105: +L(105): movq C, CO1 movq A, AO movq M, I sarq $3, I - jle .L120 + jle L(120) ALIGN_4 -.L110: +L(110): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2172,10 +2172,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L115 + jle L(115) ALIGN_4 -.L111: +L(111): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -2221,10 +2221,10 @@ subq $-32 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L111 + jg L(111) ALIGN_4 -.L115: +L(115): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2233,10 +2233,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): movaps -32 * SIZE(AO), %xmm0 movaps -28 * SIZE(AO), %xmm1 @@ -2251,10 +2251,10 @@ addq $8 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addps %xmm9, %xmm8 addps %xmm13, %xmm12 @@ -2293,12 +2293,12 @@ addq $16 * SIZE, CO1 subq $1, I - jg .L110 + jg L(110) ALIGN_4 -.L120: +L(120): testq $4, M - jle .L130 + jle L(130) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2334,10 +2334,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L125 + jle L(125) ALIGN_4 -.L121: +L(121): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -2365,10 +2365,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L121 + jg L(121) ALIGN_4 -.L125: +L(125): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2377,10 +2377,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): movaps -32 * SIZE(AO), %xmm0 movaps -32 * SIZE(BO), %xmm2 @@ -2390,10 +2390,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): addps %xmm10, %xmm8 addps %xmm11, %xmm9 @@ -2420,9 +2420,9 @@ addq $8 * SIZE, CO1 ALIGN_4 -.L130: +L(130): testq $2, M - jle .L140 + jle L(140) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2458,10 +2458,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L135 + jle L(135) ALIGN_4 -.L131: +L(131): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -32 * SIZE(AO), %xmm0 @@ -2487,10 +2487,10 @@ subq $ -8 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L131 + jg L(131) ALIGN_4 -.L135: +L(135): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2499,10 +2499,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): movsd -32 * SIZE(AO), %xmm0 movsd -32 * SIZE(BO), %xmm2 @@ -2512,10 +2512,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): addps %xmm10, %xmm8 addps %xmm11, %xmm9 addps %xmm9, %xmm8 @@ -2534,9 +2534,9 @@ addq $4 * SIZE, CO1 ALIGN_4 -.L140: +L(140): testq $1, M - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2572,10 +2572,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L145 + jle L(145) ALIGN_4 -.L141: +L(141): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movss -32 * SIZE(AO), %xmm0 @@ -2601,10 +2601,10 @@ subq $ -4 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L141 + jg L(141) ALIGN_4 -.L145: +L(145): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2613,10 +2613,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): movss -32 * SIZE(AO), %xmm0 movss -32 * SIZE(BO), %xmm2 @@ -2626,10 +2626,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): addss %xmm10, %xmm8 addss %xmm11, %xmm9 addss %xmm9, %xmm8 @@ -2644,7 +2644,7 @@ movlps %xmm4, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/zgemm3m_kernel_8x4_penryn.S b/kernel/x86_64/zgemm3m_kernel_8x4_penryn.S index 1255c2cd5c..13e07bbfdc 100644 --- a/kernel/x86_64/zgemm3m_kernel_8x4_penryn.S +++ b/kernel/x86_64/zgemm3m_kernel_8x4_penryn.S @@ -163,10 +163,10 @@ movq N, J sarq $2, J NOBRANCH - jle .L50 + jle L(50) ALIGN_4 -.L10: +L(10): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -183,10 +183,10 @@ movq M, I sarq $3, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -246,10 +246,10 @@ #endif sarq $3, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH -32 * SIZE(PREA) addps %xmm6, %xmm10 addps %xmm3, %xmm14 @@ -500,10 +500,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): prefetcht0 -16 * SIZE(BB) #ifndef TRMMKERNEL @@ -513,10 +513,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addps %xmm6, %xmm10 addps %xmm3, %xmm14 movaps %xmm2, %xmm3 @@ -552,10 +552,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): movups ALPHA_R, %xmm7 addps %xmm6, %xmm10 @@ -725,13 +725,13 @@ addq $16 * SIZE, CO2 decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M BRANCH - jle .L30 + jle L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -777,10 +777,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): addps %xmm6, %xmm10 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x39, %xmm2, %xmm7 @@ -847,10 +847,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -858,10 +858,10 @@ #endif andq $3, %rax BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): addps %xmm6, %xmm10 pshufd $0x39, %xmm2, %xmm7 mulps %xmm0, %xmm2 @@ -882,10 +882,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): movups ALPHA_R, %xmm7 addps %xmm6, %xmm10 @@ -980,10 +980,10 @@ addq $8 * SIZE, CO2 ALIGN_4 -.L30: +L(30): testq $2, M BRANCH - jle .L40 + jle L(40) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1028,10 +1028,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm0, %xmm1 @@ -1077,10 +1077,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1088,10 +1088,10 @@ #endif andq $3, %rax BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): pshufd $0x44, %xmm0, %xmm1 movsd -30 * SIZE(AO), %xmm0 addps %xmm3, %xmm8 @@ -1107,10 +1107,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): movups ALPHA_R, %xmm7 addps %xmm10, %xmm8 @@ -1159,10 +1159,10 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L40: +L(40): testq $1, M BRANCH - jle .L49 + jle L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1200,10 +1200,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x00, %xmm0, %xmm1 @@ -1235,10 +1235,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1246,10 +1246,10 @@ #endif andq $3, %rax BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 mulps %xmm1, %xmm2 @@ -1261,10 +1261,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): movups ALPHA_R, %xmm7 addps %xmm9, %xmm8 @@ -1288,7 +1288,7 @@ movhps %xmm8, 0 * SIZE(CO2, LDC, 2) ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif @@ -1299,12 +1299,12 @@ subq $1, J BRANCH - jg .L10 + jg L(10) ALIGN_4 -.L50: +L(50): testq $2, N - jle .L90 + jle L(90) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1322,10 +1322,10 @@ movq M, I sarq $3, I NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1377,10 +1377,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm3, %xmm8 @@ -1456,10 +1456,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1467,10 +1467,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addps %xmm3, %xmm8 pshufd $0x00, %xmm2, %xmm3 mulps %xmm0, %xmm3 @@ -1493,10 +1493,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): movups ALPHA_R, %xmm7 addps %xmm3, %xmm8 @@ -1574,13 +1574,13 @@ addq $16 * SIZE, CO2 decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $4, M BRANCH - jle .L70 + jle L(70) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1623,10 +1623,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm3, %xmm8 @@ -1668,10 +1668,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1679,10 +1679,10 @@ #endif andq $3, %rax BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): addps %xmm3, %xmm8 pshufd $0x00, %xmm2, %xmm3 mulps %xmm0, %xmm3 @@ -1697,10 +1697,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): movups ALPHA_R, %xmm7 addps %xmm10, %xmm8 @@ -1750,10 +1750,10 @@ addq $8 * SIZE, CO2 ALIGN_4 -.L70: +L(70): testq $2, M BRANCH - jle .L80 + jle L(80) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1792,10 +1792,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm3, %xmm8 @@ -1827,10 +1827,10 @@ subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_3 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -1838,10 +1838,10 @@ #endif andq $3, %rax BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addps %xmm3, %xmm8 pshufd $0x44, %xmm0, %xmm1 movsd -30 * SIZE(AO), %xmm0 @@ -1854,10 +1854,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_3 -.L78: +L(78): movups ALPHA_R, %xmm7 addps %xmm9, %xmm8 @@ -1885,10 +1885,10 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L80: +L(80): testq $1, M BRANCH - jle .L89 + jle L(89) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1926,10 +1926,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x00, %xmm0, %xmm1 @@ -1961,10 +1961,10 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) ALIGN_3 -.L85: +L(85): #ifndef TRMMKERNEL movq K, %rax #else @@ -1972,10 +1972,10 @@ #endif andq $3, %rax BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 mulps %xmm1, %xmm2 @@ -1987,10 +1987,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_3 -.L88: +L(88): movups ALPHA_R, %xmm7 addps %xmm9, %xmm8 @@ -2007,7 +2007,7 @@ movhps %xmm4, 0 * SIZE(CO2) ALIGN_4 -.L89: +L(89): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -2017,9 +2017,9 @@ leaq (C, LDC, 2), C ALIGN_4 -.L90: +L(90): testq $1, N - jle .L999 + jle L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -2032,10 +2032,10 @@ movq M, I sarq $3, I NOBRANCH - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2076,10 +2076,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_3 -.L92: +L(92): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x00, %xmm2, %xmm3 @@ -2123,10 +2123,10 @@ subq $1, %rax BRANCH - jg .L92 + jg L(92) ALIGN_3 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2134,10 +2134,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_3 -.L96: +L(96): pshufd $0x00, %xmm2, %xmm3 movss -31 * SIZE(BO), %xmm2 mulps %xmm3, %xmm0 @@ -2152,10 +2152,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_3 -.L98: +L(98): movups ALPHA_R, %xmm7 addps %xmm10, %xmm8 @@ -2197,13 +2197,13 @@ addq $16 * SIZE, CO1 decq I BRANCH - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $4, M BRANCH - jle .L110 + jle L(110) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2241,10 +2241,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_3 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x00, %xmm2, %xmm3 @@ -2274,10 +2274,10 @@ subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_3 -.L105: +L(105): #ifndef TRMMKERNEL movq K, %rax #else @@ -2285,10 +2285,10 @@ #endif andq $3, %rax BRANCH - je .L108 + je L(108) ALIGN_3 -.L106: +L(106): pshufd $0x00, %xmm2, %xmm3 movss -31 * SIZE(BO), %xmm2 mulps %xmm0, %xmm3 @@ -2300,10 +2300,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_3 -.L108: +L(108): movups ALPHA_R, %xmm7 addps %xmm9, %xmm8 @@ -2330,10 +2330,10 @@ addq $8 * SIZE, CO1 ALIGN_4 -.L110: +L(110): testq $2, M BRANCH - jle .L120 + jle L(120) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2372,10 +2372,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_3 -.L112: +L(112): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x00, %xmm2, %xmm3 @@ -2405,10 +2405,10 @@ subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_3 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2416,10 +2416,10 @@ #endif andq $3, %rax BRANCH - je .L118 + je L(118) ALIGN_3 -.L116: +L(116): pshufd $0x00, %xmm2, %xmm3 movss -31 * SIZE(BO), %xmm2 mulps %xmm0, %xmm3 @@ -2431,10 +2431,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_3 -.L118: +L(118): movups ALPHA_R, %xmm7 movsd 0 * SIZE(CO1), %xmm0 @@ -2451,10 +2451,10 @@ addq $4 * SIZE, CO1 ALIGN_4 -.L120: +L(120): testq $1, M BRANCH - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2492,10 +2492,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L125 + jle L(125) ALIGN_3 -.L122: +L(122): mulss %xmm0, %xmm2 movss -31 * SIZE(AO), %xmm0 addss %xmm2, %xmm8 @@ -2521,10 +2521,10 @@ subq $1, %rax BRANCH - jg .L122 + jg L(122) ALIGN_3 -.L125: +L(125): #ifndef TRMMKERNEL movq K, %rax #else @@ -2532,10 +2532,10 @@ #endif andq $3, %rax BRANCH - je .L128 + je L(128) ALIGN_3 -.L126: +L(126): mulss %xmm0, %xmm2 movss -31 * SIZE(AO), %xmm0 addss %xmm2, %xmm8 @@ -2546,10 +2546,10 @@ subq $1, %rax BRANCH - jg .L126 + jg L(126) ALIGN_3 -.L128: +L(128): movups ALPHA_R, %xmm7 addss %xmm9, %xmm8 @@ -2564,7 +2564,7 @@ movlps %xmm4, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm3m_kernel_8x4_sse.S b/kernel/x86_64/zgemm3m_kernel_8x4_sse.S index a3c69163e4..d1b9e81818 100644 --- a/kernel/x86_64/zgemm3m_kernel_8x4_sse.S +++ b/kernel/x86_64/zgemm3m_kernel_8x4_sse.S @@ -417,9 +417,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L50 + jle L(50) -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -430,12 +430,12 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): PREFETCH (RPREFETCHSIZE + 0) * SIZE(B) movd 0 * SIZE(B), %mm0 @@ -482,17 +482,17 @@ addq $32 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movd 0 * SIZE(B), %mm0 movd 1 * SIZE(B), %mm1 movd 2 * SIZE(B), %mm2 @@ -515,10 +515,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a @@ -527,10 +527,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): PREFETCH 0 * SIZE(BB) subq $-16 * SIZE, BB @@ -592,10 +592,10 @@ leaq (BO, %rax, 8), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -616,7 +616,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -637,7 +637,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -658,7 +658,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -679,7 +679,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -700,7 +700,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -721,7 +721,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -742,7 +742,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -763,17 +763,17 @@ addq $16 * SIZE, %rax BRANCH - jl .L12 + jl L(12) ALIGN_3 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_3 @@ -792,10 +792,10 @@ #else sarq $2, %rax NOBRANCH - jle .L16 + jle L(16) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -809,10 +809,10 @@ subq $-32 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) #endif -.L16: +L(16): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -821,7 +821,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L18 + je L(18) leaq (, %rax, 8), %rax leaq (AO, %rax, 4), AO @@ -829,7 +829,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulps %xmm0, %xmm1 addps %xmm1, %xmm8 movaps -28 * SIZE(BO, %rax, 8), %xmm1 @@ -856,10 +856,10 @@ movaps -20 * SIZE(AO, %rax, 4), %xmm2 addq $SIZE * 2, %rax - jl .L17 + jl L(17) ALIGN_4 -.L18: +L(18): movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 movsd 4 * SIZE(CO1), %xmm1 @@ -995,12 +995,12 @@ addq $16 * SIZE, CO1 # coffset += 4 addq $16 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1044,10 +1044,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) @@ -1161,10 +1161,10 @@ addq $ 32 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1173,10 +1173,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1193,10 +1193,10 @@ addq $ 4 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 movsd 4 * SIZE(CO1), %xmm9 @@ -1277,9 +1277,9 @@ addq $8 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $2, M - je .L40 + je L(40) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1323,10 +1323,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) @@ -1445,10 +1445,10 @@ addq $ 16 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1457,10 +1457,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1478,10 +1478,10 @@ addq $ 2 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -1526,9 +1526,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L40: +L(40): testq $1, M - je .L49 + je L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1573,10 +1573,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): mulss %xmm8, %xmm9 addss %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) @@ -1695,10 +1695,10 @@ addq $ 8 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1707,10 +1707,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -1728,10 +1728,10 @@ addq $ 1 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): movsd 0 * SIZE(CO1), %xmm8 pshufd $0x50, %xmm0, %xmm4 @@ -1765,19 +1765,19 @@ movlps %xmm4, 0 * SIZE(CO2, LDC, 2) ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C # c += 4 * ldc decq J # j -- - jg .L01 + jg L(01) -.L50: +L(50): testq $2, N - je .L100 + je L(100) -.L51: +L(51): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1788,10 +1788,10 @@ movq K, %rax sarq $2, %rax - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): #if defined(PENTIUM4) || defined(GENERIC) movss 0 * SIZE(B), %xmm0 movss 1 * SIZE(B), %xmm1 @@ -1869,17 +1869,17 @@ #endif decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): movq K, %rax andq $3, %rax BRANCH - jle .L60 + jle L(60) ALIGN_4 -.L54: +L(54): #if defined(PENTIUM4) || defined(GENERIC) movss 0 * SIZE(B), %xmm0 movss 1 * SIZE(B), %xmm1 @@ -1907,20 +1907,20 @@ addq $ 2 * SIZE, B addq $ 8 * SIZE, BO decq %rax - jne .L54 + jne L(54) ALIGN_4 -.L60: +L(60): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L70 + jle L(70) ALIGN_4 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1968,10 +1968,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2091,10 +2091,10 @@ addq $64 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -2103,10 +2103,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -2123,10 +2123,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 movsd 4 * SIZE(CO1), %xmm9 @@ -2196,12 +2196,12 @@ addq $16 * SIZE, CO1 # coffset += 4 addq $16 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L61 + jg L(61) ALIGN_4 -.L70: +L(70): testq $4, M - je .L80 + je L(80) #if !defined(TRMMKERNEL) || \ @@ -2246,10 +2246,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2313,10 +2313,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2325,10 +2325,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -2339,10 +2339,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -2386,9 +2386,9 @@ addq $8 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L80: +L(80): testq $2, M - je .L90 + je L(90) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2432,10 +2432,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L85 + je L(85) ALIGN_4 -.L82: +L(82): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) @@ -2506,10 +2506,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L85: +L(85): #ifndef TRMMKERNEL movq K, %rax #else @@ -2518,10 +2518,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -2533,10 +2533,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -2564,9 +2564,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L90: +L(90): testq $1, M - je .L99 + je L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2610,10 +2610,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) @@ -2684,10 +2684,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2696,10 +2696,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -2711,10 +2711,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): addss %xmm2, %xmm0 addss %xmm3, %xmm1 @@ -2735,7 +2735,7 @@ movlps %xmm2, 0 * SIZE(CO2) ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -2743,11 +2743,11 @@ ALIGN_4 -.L100: +L(100): testq $1, N - je .L999 + je L(999) -.L101: +L(101): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -2758,11 +2758,11 @@ movq K, %rax sarq $3, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): #if defined(PENTIUM4) || defined(GENERIC) movss 0 * SIZE(B), %xmm0 movss 1 * SIZE(B), %xmm1 @@ -2840,17 +2840,17 @@ #endif decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): movq K, %rax andq $7, %rax BRANCH - jle .L110 + jle L(110) ALIGN_4 -.L104: +L(104): #if defined(PENTIUM4) || defined(GENERIC) movss 0 * SIZE(B), %xmm0 shufps $0, %xmm0, %xmm0 @@ -2867,19 +2867,19 @@ addq $ 1 * SIZE, B addq $ 4 * SIZE, BO decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L110: +L(110): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L120 + jle L(120) ALIGN_4 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2926,10 +2926,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulps %xmm9, %xmm8 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3002,10 +3002,10 @@ addq $64 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -3014,10 +3014,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulps %xmm9, %xmm8 mulps -28 * SIZE(AO), %xmm9 addps %xmm8, %xmm0 @@ -3028,10 +3028,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 movsd 4 * SIZE(CO1), %xmm9 @@ -3067,12 +3067,12 @@ addq $16 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L111 + jg L(111) ALIGN_4 -.L120: +L(120): testq $4, M - je .L130 + je L(130) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3114,10 +3114,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L125 + je L(125) ALIGN_4 -.L122: +L(122): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3155,10 +3155,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L122 + jne L(122) ALIGN_4 -.L125: +L(125): #ifndef TRMMKERNEL movq K, %rax #else @@ -3167,10 +3167,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): mulps %xmm8, %xmm9 movaps -28 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -3179,10 +3179,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): addps %xmm1, %xmm0 addps %xmm3, %xmm2 addps %xmm2, %xmm0 @@ -3209,9 +3209,9 @@ addq $8 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L130: +L(130): testq $2, M - je .L140 + je L(140) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3253,10 +3253,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L135 + je L(135) ALIGN_4 -.L132: +L(132): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3302,10 +3302,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L132 + jne L(132) ALIGN_4 -.L135: +L(135): #ifndef TRMMKERNEL movq K, %rax #else @@ -3314,10 +3314,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): mulps %xmm8, %xmm9 movsd -30 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -3326,10 +3326,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): addps %xmm1, %xmm0 movsd 0 * SIZE(CO1), %xmm8 @@ -3345,9 +3345,9 @@ addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L140: +L(140): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3389,10 +3389,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L145 + je L(145) ALIGN_4 -.L142: +L(142): mulss %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3426,10 +3426,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L142 + jne L(142) ALIGN_4 -.L145: +L(145): #ifndef TRMMKERNEL movq K, %rax #else @@ -3438,10 +3438,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): mulss %xmm8, %xmm9 movss -31 * SIZE(AO), %xmm8 addss %xmm9, %xmm0 @@ -3450,10 +3450,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): addss %xmm1, %xmm0 addss %xmm3, %xmm2 addss %xmm2, %xmm0 @@ -3467,7 +3467,7 @@ movlps %xmm2, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp EMMS movq 0(%rsp), %rbx diff --git a/kernel/x86_64/zgemm3m_kernel_8x4_sse3.S b/kernel/x86_64/zgemm3m_kernel_8x4_sse3.S index adf0a53316..a2fc2a3e10 100644 --- a/kernel/x86_64/zgemm3m_kernel_8x4_sse3.S +++ b/kernel/x86_64/zgemm3m_kernel_8x4_sse3.S @@ -400,9 +400,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L50 + jle L(50) -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -413,10 +413,10 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): movddup 0 * SIZE(B), %xmm0 movddup 2 * SIZE(B), %xmm1 movddup 4 * SIZE(B), %xmm2 @@ -442,17 +442,17 @@ addq $32 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movddup 0 * SIZE(B), %xmm0 movddup 2 * SIZE(B), %xmm1 @@ -462,10 +462,10 @@ addq $4 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a @@ -474,10 +474,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): prefetcht0 0 * SIZE(BB) prefetcht0 8 * SIZE(BB) subq $-16 * SIZE, BB @@ -538,9 +538,9 @@ #if 1 andq $-8, %rax salq $4, %rax - je .L15 + je L(15) -.L1X: +L(1X): KERNEL1 (64 * 0) KERNEL2 (64 * 0) KERNEL3 (64 * 0) @@ -559,7 +559,7 @@ KERNEL16(64 * 0) cmpq $128 * 1, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 1) KERNEL2 (64 * 1) KERNEL3 (64 * 1) @@ -578,7 +578,7 @@ KERNEL16(64 * 1) cmpq $128 * 2, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 2) KERNEL2 (64 * 2) KERNEL3 (64 * 2) @@ -597,7 +597,7 @@ KERNEL16(64 * 2) cmpq $128 * 3, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 3) KERNEL2 (64 * 3) KERNEL3 (64 * 3) @@ -616,7 +616,7 @@ KERNEL16(64 * 3) cmpq $128 * 4, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 4) KERNEL2 (64 * 4) KERNEL3 (64 * 4) @@ -635,7 +635,7 @@ KERNEL16(64 * 4) cmpq $128 * 5, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 5) KERNEL2 (64 * 5) KERNEL3 (64 * 5) @@ -654,7 +654,7 @@ KERNEL16(64 * 5) cmpq $128 * 6, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 6) KERNEL2 (64 * 6) KERNEL3 (64 * 6) @@ -673,7 +673,7 @@ KERNEL16(64 * 6) cmpq $128 * 7, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 7) KERNEL2 (64 * 7) KERNEL3 (64 * 7) @@ -694,17 +694,17 @@ addq $64 * 8 * SIZE, AO addq $64 * 8 * SIZE, BO subq $128 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO leaq (BO, %rax, 2), BO #else sarq $3, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): KERNEL1 (64 * 0) KERNEL2 (64 * 0) KERNEL3 (64 * 0) @@ -725,11 +725,11 @@ addq $64 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L12 + jne L(12) #endif ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else @@ -738,10 +738,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -772,10 +772,10 @@ addq $8 * SIZE, AO addq $8 * SIZE, BO decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 movsd 4 * SIZE(CO1), %xmm9 @@ -926,12 +926,12 @@ addq $16 * SIZE, CO1 # coffset += 4 addq $16 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -975,10 +975,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm9, %xmm0 @@ -1088,10 +1088,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1100,10 +1100,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -1121,10 +1121,10 @@ addq $4 * SIZE, AO addq $8 * SIZE, BO decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 movsd 4 * SIZE(CO1), %xmm9 @@ -1205,9 +1205,9 @@ addq $8 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $2, M - je .L40 + je L(40) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1248,10 +1248,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): shufps $0x50, %xmm9, %xmm9 mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1329,10 +1329,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1341,10 +1341,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): shufps $0x50, %xmm9, %xmm9 mulps %xmm8, %xmm9 addps %xmm9, %xmm0 @@ -1358,10 +1358,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -1407,9 +1407,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L40: +L(40): testq $1, M - je .L49 + je L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1448,10 +1448,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): shufps $0, %xmm8, %xmm8 movhps 4 * SIZE(BO), %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1505,10 +1505,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1517,10 +1517,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): shufps $0, %xmm8, %xmm8 movhps 4 * SIZE(BO), %xmm9 mulps %xmm8, %xmm9 @@ -1531,10 +1531,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): addps %xmm1, %xmm0 movsd 0 * SIZE(CO1), %xmm8 @@ -1557,19 +1557,19 @@ movhps %xmm0, 0 * SIZE(CO2, LDC, 2) ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C # c += 4 * ldc decq J # j -- - jg .L01 + jg L(01) -.L50: +L(50): testq $2, N - je .L100 + je L(100) -.L51: +L(51): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1580,10 +1580,10 @@ movq K, %rax sarq $3, %rax - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): movddup 0 * SIZE(B), %xmm0 movddup 2 * SIZE(B), %xmm1 movddup 4 * SIZE(B), %xmm2 @@ -1609,37 +1609,37 @@ addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): movq K, %rax andq $7, %rax BRANCH - jle .L60 + jle L(60) ALIGN_4 -.L54: +L(54): movddup 0 * SIZE(B), %xmm0 movaps %xmm0, 0 * SIZE(BO) addq $ 2 * SIZE, B addq $ 4 * SIZE, BO decq %rax - jne .L54 + jne L(54) ALIGN_4 -.L60: +L(60): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L70 + jle L(70) ALIGN_4 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1684,10 +1684,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm9, %xmm0 @@ -1806,10 +1806,10 @@ addq $64 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1818,10 +1818,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -1840,10 +1840,10 @@ addq $8 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 movsd 4 * SIZE(CO1), %xmm9 @@ -1913,12 +1913,12 @@ addq $16 * SIZE, CO1 # coffset += 4 addq $16 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L61 + jg L(61) ALIGN_4 -.L70: +L(70): testq $4, M - je .L80 + je L(80) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1959,10 +1959,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm9, %xmm0 @@ -2024,10 +2024,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2036,10 +2036,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -2051,10 +2051,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -2100,9 +2100,9 @@ addq $8 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L80: +L(80): testq $2, M - je .L90 + je L(90) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2141,10 +2141,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L85 + je L(85) ALIGN_4 -.L82: +L(82): shufps $0x50, %xmm9, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulps %xmm8, %xmm9 @@ -2190,10 +2190,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L85: +L(85): #ifndef TRMMKERNEL movq K, %rax #else @@ -2202,10 +2202,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): shufps $0x50, %xmm9, %xmm9 mulps %xmm8, %xmm9 movddup 2 * SIZE(AO), %xmm8 @@ -2215,10 +2215,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): addps %xmm1, %xmm0 movsd 0 * SIZE(CO1), %xmm8 @@ -2244,9 +2244,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L90: +L(90): testq $1, M - je .L99 + je L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2285,10 +2285,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): shufps $0, %xmm8, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulps %xmm8, %xmm9 @@ -2334,10 +2334,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2346,10 +2346,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): shufps $0, %xmm8, %xmm8 mulps %xmm8, %xmm9 movss 1 * SIZE(AO), %xmm8 @@ -2359,10 +2359,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): addps %xmm1, %xmm0 movsd 0 * SIZE(CO1), %xmm8 @@ -2378,18 +2378,18 @@ movhps %xmm12, 0 * SIZE(CO2) ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif leaq (C, LDC, 2), C # c += 4 * ldc ALIGN_4 -.L100: +L(100): testq $1, N - je .L999 + je L(999) -.L101: +L(101): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -2400,11 +2400,11 @@ movq K, %rax sarq $3, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movss 0 * SIZE(B), %xmm0 movss 1 * SIZE(B), %xmm1 movss 2 * SIZE(B), %xmm2 @@ -2435,17 +2435,17 @@ addq $16 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): movq K, %rax andq $7, %rax BRANCH - jle .L110 + jle L(110) ALIGN_4 -.L104: +L(104): movss 0 * SIZE(B), %xmm0 movss %xmm0, 0 * SIZE(BO) movss %xmm0, 1 * SIZE(BO) @@ -2453,19 +2453,19 @@ addq $ 1 * SIZE, B addq $ 2 * SIZE, BO decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L110: +L(110): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L120 + jle L(120) ALIGN_4 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2509,10 +2509,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulps %xmm8, %xmm9 movaps 4 * SIZE(AO), %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2583,10 +2583,10 @@ addq $64 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2595,10 +2595,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulps %xmm8, %xmm9 movaps 4 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -2611,10 +2611,10 @@ addq $8 * SIZE, AO addq $2 * SIZE, BO decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addps %xmm1, %xmm0 addps %xmm5, %xmm4 @@ -2653,12 +2653,12 @@ addq $16 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L111 + jg L(111) ALIGN_4 -.L120: +L(120): testq $4, M - je .L130 + je L(130) #if !defined(TRMMKERNEL) || \ @@ -2704,10 +2704,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L125 + je L(125) ALIGN_4 -.L122: +L(122): mulps %xmm8, %xmm9 movaps 4 * SIZE(AO), %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2745,10 +2745,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L122 + jne L(122) ALIGN_4 -.L125: +L(125): #ifndef TRMMKERNEL movq K, %rax #else @@ -2757,10 +2757,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): mulps %xmm8, %xmm9 movaps 4 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -2769,10 +2769,10 @@ addq $4 * SIZE, AO addq $2 * SIZE, BO decq %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): addps %xmm1, %xmm0 movsd 0 * SIZE(CO1), %xmm8 @@ -2797,9 +2797,9 @@ addq $8 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L130: +L(130): testq $2, M - je .L140 + je L(140) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2840,10 +2840,10 @@ movq %rax, KKK #endif sarq $4, %rax - je .L135 + je L(135) ALIGN_4 -.L132: +L(132): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps 4 * SIZE(AO), %xmm8 @@ -2875,10 +2875,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L132 + jne L(132) ALIGN_4 -.L135: +L(135): #ifndef TRMMKERNEL movq K, %rax #else @@ -2887,10 +2887,10 @@ movaps ALPHA, %xmm15 andq $15, %rax # if (k & 1) BRANCH - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): movsd 0 * SIZE(AO), %xmm8 movsd 0 * SIZE(BO), %xmm9 @@ -2900,10 +2900,10 @@ addq $2 * SIZE, AO addq $2 * SIZE, BO decq %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): addps %xmm1, %xmm0 addps %xmm3, %xmm2 addps %xmm2, %xmm0 @@ -2925,9 +2925,9 @@ addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L140: +L(140): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2968,10 +2968,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L145 + je L(145) ALIGN_4 -.L142: +L(142): mulss %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movss 1 * SIZE(AO), %xmm8 @@ -3003,10 +3003,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L142 + jne L(142) ALIGN_4 -.L145: +L(145): #ifndef TRMMKERNEL movq K, %rax #else @@ -3015,10 +3015,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): movss 0 * SIZE(AO), %xmm8 movss 0 * SIZE(BO), %xmm9 mulps %xmm8, %xmm9 @@ -3027,10 +3027,10 @@ addq $1 * SIZE, AO addq $2 * SIZE, BO decq %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): addss %xmm1, %xmm0 addss %xmm3, %xmm2 addss %xmm2, %xmm0 @@ -3045,7 +3045,7 @@ movlps %xmm12, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/zgemm_beta.S b/kernel/x86_64/zgemm_beta.S index 1612d92f16..3f5a49d054 100644 --- a/kernel/x86_64/zgemm_beta.S +++ b/kernel/x86_64/zgemm_beta.S @@ -96,35 +96,35 @@ movq STACK_LDC, LDC testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) salq $ZBASE_SHIFT, LDC #ifdef DOUBLE ucomisd %xmm15, %xmm0 - jne .L71 + jne L(71) ucomisd %xmm15, %xmm1 - jne .L71 + jne L(71) #else ucomiss %xmm15, %xmm0 - jne .L71 + jne L(71) ucomiss %xmm15, %xmm1 - jne .L71 + jne L(71) #endif ALIGN_2 -.L53: +L(53): movq C, C1 # c_offset1 = c_offset addq LDC, C # c_offset += ldc movq M, I sarq $2, I - jle .L56 + jle L(56) ALIGN_2 -.L57: +L(57): #ifdef OPTERON prefetchw 64 * SIZE(C1) #endif @@ -139,39 +139,39 @@ MOVSD %xmm0, 7 * SIZE(C1) addq $8 * SIZE, C1 # c_offset1 += 8 decq I # i-- - jg .L57 + jg L(57) ALIGN_2 -.L56: +L(56): movq M, I andq $3, I - jle .L62 + jle L(62) ALIGN_2 -.L63: +L(63): MOVSD %xmm0, 0 * SIZE(C1) MOVSD %xmm0, 1 * SIZE(C1) addq $2 * SIZE,C1 decq I - jg .L63 + jg L(63) ALIGN_2 -.L62: +L(62): decq N # j -- - jg .L53 - jmp .L999 + jg L(53) + jmp L(999) ALIGN_3 -.L71: +L(71): movq C, C1 addq LDC, C # c_offset += ldc movq M, I sarq $1, I - jle .L84 + jle L(84) ALIGN_3 -.L85: +L(85): #ifdef OPTERON prefetchw 16 * SIZE(C1) #endif @@ -207,15 +207,15 @@ MOVSD %xmm8, 3 * SIZE(C1) addq $4 * SIZE, C1 decq I - jg .L85 + jg L(85) ALIGN_3 -.L84: +L(84): testq $1, M - jle .L74 + jle L(74) ALIGN_3 -.L75: +L(75): prefetchnta 80 * SIZE(C1) MOVSD 0 * SIZE(C1), %xmm2 @@ -234,12 +234,12 @@ MOVSD %xmm4, 1 * SIZE(C1) ALIGN_2 -.L74: +L(74): decq N - jg .L71 + jg L(71) ALIGN_2 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 movups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/zgemm_kernel_1x4_nehalem.S b/kernel/x86_64/zgemm_kernel_1x4_nehalem.S index 0d6acf3309..b8a618ea3d 100644 --- a/kernel/x86_64/zgemm_kernel_1x4_nehalem.S +++ b/kernel/x86_64/zgemm_kernel_1x4_nehalem.S @@ -178,15 +178,15 @@ movq %r11, KK #endif testq M, M - jle .L999 + jle L(999) movq N, J sarq $2, J NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -203,7 +203,7 @@ movq M, I ALIGN_4 -.L11: +L(11): prefetcht2 -16 * SIZE(BB) subq $-8 * SIZE, BB @@ -260,10 +260,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm12 @@ -389,10 +389,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): movddup ALPHA_R, %xmm6 movddup ALPHA_I, %xmm7 @@ -403,10 +403,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm1, %xmm12 movaps -16 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm13 @@ -442,10 +442,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): ADD1 %xmm1, %xmm12 ADD2 %xmm2, %xmm13 ADD1 %xmm3, %xmm14 @@ -505,7 +505,7 @@ testq $15, CO1 NOBRANCH - jne .L18x + jne L(18x) #ifndef TRMMKERNEL movaps (CO1), %xmm0 @@ -541,7 +541,7 @@ addq $2 * SIZE, CO2 decq I BRANCH - jg .L11 + jg L(11) #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK @@ -552,11 +552,11 @@ subq $1, J BRANCH - jg .L01 - jmp .L20 + jg L(01) + jmp L(20) ALIGN_4 -.L18x: +L(18x): #ifndef TRMMKERNEL movups (CO1), %xmm0 movups (CO1, LDC), %xmm1 @@ -591,7 +591,7 @@ addq $2 * SIZE, CO2 decq I BRANCH - jg .L11 + jg L(11) #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK @@ -602,13 +602,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L20: +L(20): testq $2, N BRANCH - jle .L30 + jle L(30) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -622,7 +622,7 @@ movq M, I ALIGN_4 -.L21: +L(21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -667,10 +667,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -742,10 +742,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -753,10 +753,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): ADD1 %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -778,10 +778,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): ADD1 %xmm1, %xmm8 ADD2 %xmm2, %xmm9 ADD1 %xmm3, %xmm10 @@ -857,7 +857,7 @@ addq $2 * SIZE, CO2 decq I BRANCH - jg .L21 + jg L(21) #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK @@ -867,10 +867,10 @@ movq BO, B ALIGN_4 -.L30: +L(30): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -883,7 +883,7 @@ movq M, I ALIGN_4 -.L31: +L(31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -925,10 +925,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -968,13 +968,13 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 ALIGN_3 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -982,10 +982,10 @@ #endif andq $3, %rax BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): ADD1 %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -999,10 +999,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): ADD1 %xmm1, %xmm8 ADD2 %xmm2, %xmm9 @@ -1061,10 +1061,10 @@ addq $2 * SIZE, CO1 decq I BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm_kernel_2x1_atom.S b/kernel/x86_64/zgemm_kernel_2x1_atom.S index d9f320a8a1..a570279569 100644 --- a/kernel/x86_64/zgemm_kernel_2x1_atom.S +++ b/kernel/x86_64/zgemm_kernel_2x1_atom.S @@ -172,10 +172,10 @@ movq N, J testq N, N - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -192,10 +192,10 @@ movq M, I sarq $1, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -249,10 +249,10 @@ #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): ADDSD2 %xmm2, %xmm13 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps %xmm0, %xmm2 @@ -395,10 +395,10 @@ mulsd %xmm3, %xmm6 movsd 1 * SIZE(BO), %xmm3 - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else @@ -407,10 +407,10 @@ andq $3, %rax BRANCH BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): ADDSD2 %xmm2, %xmm13 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -449,10 +449,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): movsd ALPHA_R, %xmm0 movsd ALPHA_I, %xmm1 @@ -513,12 +513,12 @@ addq $4 * SIZE, CO1 decq I - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): testq $1, M - jle .L99 + jle L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -565,10 +565,10 @@ #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADDSD2 %xmm2, %xmm9 movaps %xmm0, %xmm2 @@ -646,10 +646,10 @@ addq $8 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -658,10 +658,10 @@ andq $3, %rax BRANCH BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): ADDSD2 %xmm2, %xmm9 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -684,10 +684,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): movsd ALPHA_R, %xmm0 movsd ALPHA_I, %xmm1 @@ -730,17 +730,17 @@ #endif ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addq $1, KK #endif movq BO, B decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm_kernel_2x2_barcelona.S b/kernel/x86_64/zgemm_kernel_2x2_barcelona.S index 70e8f6039c..7b47a84a24 100644 --- a/kernel/x86_64/zgemm_kernel_2x2_barcelona.S +++ b/kernel/x86_64/zgemm_kernel_2x2_barcelona.S @@ -485,10 +485,10 @@ movq N, J sarq $1, J - jle .L100 + jle L(100) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -505,10 +505,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L30 + jle L(30) ALIGN_4 -.L10: +L(10): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -562,10 +562,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -575,7 +575,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -585,7 +585,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -595,7 +595,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -605,7 +605,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -615,7 +615,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -625,7 +625,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -635,7 +635,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -644,17 +644,17 @@ KERNEL6(16 * 0) KERNEL7(16 * 0) KERNEL8(16 * 0) - jl .L12 + jl L(12) ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_4 @@ -667,7 +667,7 @@ subq $-16 * SIZE, AO ALIGN_4 -.L16: +L(16): movddup ALPHA_R, %xmm6 movddup ALPHA_I, %xmm7 @@ -677,7 +677,7 @@ movq KKK, %rax #endif andq $3, %rax - je .L19 + je L(19) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -685,7 +685,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 ADD1 %xmm0, %xmm8 @@ -712,10 +712,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L17 + jl L(17) ALIGN_4 -.L19: +L(19): prefetch -8 * SIZE(BB) subq $-16 * SIZE, BB @@ -807,12 +807,12 @@ addq $4 * SIZE, CO1 # coffset += 4 addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L30: +L(30): testq $1, M - jle .L99 + jle L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -857,10 +857,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L46 + je L(46) ALIGN_4 -.L42: +L(42): mulpd %xmm0, %xmm1 ADD1 %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -916,10 +916,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L42 + jl L(42) ALIGN_4 -.L46: +L(46): movddup ALPHA_R, %xmm6 movddup ALPHA_I, %xmm7 @@ -929,7 +929,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L49 + je L(49) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -937,7 +937,7 @@ negq %rax ALIGN_4 -.L47: +L(47): mulpd %xmm0, %xmm1 ADD1 %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -953,10 +953,10 @@ movddup -11 * SIZE(BO, %rax, 4), %xmm5 addq $SIZE, %rax - jl .L47 + jl L(47) ALIGN_4 -.L49: +L(49): #ifndef TRMMKERNEL movupd (CO1), %xmm0 movupd (CO2), %xmm1 @@ -1014,7 +1014,7 @@ #endif ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -1023,13 +1023,13 @@ leaq (C, LDC, 2), C # c += 2 * ldc decq J # j -- - jg .L01 + jg L(01) -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) -.L101: +L(101): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1040,10 +1040,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L130 + jle L(130) ALIGN_4 -.L110: +L(110): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1089,10 +1089,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L116 + je L(116) ALIGN_4 -.L112: +L(112): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 ADD1 %xmm0, %xmm8 @@ -1144,10 +1144,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L112 + jl L(112) ALIGN_4 -.L116: +L(116): movddup ALPHA_R, %xmm6 movddup ALPHA_I, %xmm7 @@ -1157,7 +1157,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L119 + je L(119) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -1165,7 +1165,7 @@ negq %rax ALIGN_4 -.L117: +L(117): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 ADD1 %xmm0, %xmm8 @@ -1180,10 +1180,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L117 + jl L(117) ALIGN_4 -.L119: +L(119): #ifndef TRMMKERNEL movupd (CO1), %xmm0 movupd 2 * SIZE(CO1), %xmm2 @@ -1243,12 +1243,12 @@ addq $4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L130: +L(130): testq $1, M - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1292,10 +1292,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L146 + je L(146) ALIGN_4 -.L142: +L(142): mulpd %xmm0, %xmm1 ADD1 %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -1327,10 +1327,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L142 + jl L(142) ALIGN_4 -.L146: +L(146): movddup ALPHA_R, %xmm6 movddup ALPHA_I, %xmm7 @@ -1340,7 +1340,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L148 + je L(148) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -1348,7 +1348,7 @@ negq %rax ALIGN_4 -.L147: +L(147): mulpd %xmm0, %xmm1 ADD1 %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -1358,10 +1358,10 @@ movddup -13 * SIZE(BO, %rax, 2), %xmm3 addq $SIZE, %rax - jl .L147 + jl L(147) ALIGN_4 -.L148: +L(148): #ifndef TRMMKERNEL movupd (CO1), %xmm0 #endif @@ -1394,7 +1394,7 @@ movhpd %xmm8, 1 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq (%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm_kernel_2x2_bulldozer.S b/kernel/x86_64/zgemm_kernel_2x2_bulldozer.S index 6c8b4c8722..aaaf4d8d44 100644 --- a/kernel/x86_64/zgemm_kernel_2x2_bulldozer.S +++ b/kernel/x86_64/zgemm_kernel_2x2_bulldozer.S @@ -451,13 +451,13 @@ STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -485,23 +485,23 @@ #endif #endif -.L2_0: +L(2_0): movq Ndiv6, J cmpq $0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovups (BO1), %xmm0 vmovups 2 * SIZE(BO1), %xmm1 @@ -510,13 +510,13 @@ addq $4*SIZE,BO1 addq $4*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -530,11 +530,11 @@ movq M, I sarq $1, I // i = (m >> 1) - je .L2_40 + je L(2_40) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -572,7 +572,7 @@ andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -583,7 +583,7 @@ negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -599,7 +599,7 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -615,12 +615,12 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -628,7 +628,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -640,14 +640,14 @@ negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL2x2_SUB(xxx) - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vmovddup ALPHA_R, %xmm0 vmovddup ALPHA_I, %xmm1 @@ -743,20 +743,20 @@ addq $4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -794,7 +794,7 @@ andq $-8, %rax // K = K - ( K % 8 ) - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -805,7 +805,7 @@ negq %rax ALIGN_4 -.L2_42: +L(2_42): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x2_1(xxx) @@ -821,7 +821,7 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x2_1(xxx) @@ -837,12 +837,12 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -850,7 +850,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -862,14 +862,14 @@ negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vmovddup ALPHA_R, %xmm0 vmovddup ALPHA_I, %xmm1 @@ -945,17 +945,17 @@ -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -963,30 +963,30 @@ movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -1000,11 +1000,11 @@ movq M, I sarq $1, I // i = (m >> 1) - je .L1_40 + je L(1_40) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1042,7 +1042,7 @@ andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1053,7 +1053,7 @@ negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -1067,7 +1067,7 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_16 + je L(1_16) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -1081,12 +1081,12 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -1094,7 +1094,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1106,14 +1106,14 @@ negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL2x1_SUB(xxx) - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vmovddup ALPHA_R, %xmm0 vmovddup ALPHA_I, %xmm1 @@ -1186,20 +1186,20 @@ addq $4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1237,7 +1237,7 @@ andq $-8, %rax // K = K - ( K % 8 ) - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1248,7 +1248,7 @@ negq %rax ALIGN_4 -.L1_42: +L(1_42): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x1_1(xxx) @@ -1262,7 +1262,7 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x1_1(xxx) @@ -1276,12 +1276,12 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -1289,7 +1289,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1301,14 +1301,14 @@ negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vmovddup ALPHA_R, %xmm0 vmovddup ALPHA_I, %xmm1 @@ -1372,7 +1372,7 @@ -.L999: +L(999): vzeroupper movq SP, %rsp diff --git a/kernel/x86_64/zgemm_kernel_2x2_core2.S b/kernel/x86_64/zgemm_kernel_2x2_core2.S index b74e2fe91b..03110c4de1 100644 --- a/kernel/x86_64/zgemm_kernel_2x2_core2.S +++ b/kernel/x86_64/zgemm_kernel_2x2_core2.S @@ -183,10 +183,10 @@ movq N, J sarq $1, J # j = (n >> 2) NOBRANCH - jle .L100 + jle L(100) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -200,10 +200,10 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_3 -.L02: +L(02): prefetcht0 (PREFETCH_R + 0) * SIZE(B) prefetcht0 (PREFETCH_R + 8) * SIZE(B) @@ -261,17 +261,17 @@ subq $-32 * SIZE, BO subq $-16 * SIZE, B decq %rax - jne .L02 + jne L(02) ALIGN_3 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L05 + jle L(05) ALIGN_3 -.L04: +L(04): movapd -14 * SIZE(B), %xmm1 movddup %xmm0, %xmm8 @@ -290,10 +290,10 @@ addq $ 8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_3 -.L05: +L(05): leaq (PREFETCH_R + 0) * SIZE(B), BB movq C, CO1 # coffset1 = c @@ -302,10 +302,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L30 + jle L(30) ALIGN_4 -.L10: +L(10): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -363,10 +363,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): PADDING; ADD1 %xmm2, %xmm10 movaps -15 * SIZE(BO), %xmm2 @@ -496,10 +496,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): prefetcht2 -8 * SIZE(BB) #ifndef TRMMKERNEL @@ -510,10 +510,10 @@ andq $3, %rax BRANCH BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): ADD1 %xmm2, %xmm10 movaps -15 * SIZE(BO), %xmm2 ADD1 %xmm3, %xmm14 @@ -548,10 +548,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L19: +L(19): movapd ALPHA_R, %xmm6 ADD1 %xmm2, %xmm10 ADD1 %xmm3, %xmm14 @@ -653,14 +653,14 @@ addq $4 * SIZE, CO1 # coffset += 4 addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L30: +L(30): testq $1, M - jle .L99 + jle L(99) -.L40: +L(40): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -695,9 +695,9 @@ movq %rax, KKK #endif sarq $2, %rax - je .L42 + je L(42) -.L41: +L(41): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -767,9 +767,9 @@ subq $ -8 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L41 + jne L(41) -.L42: +L(42): #ifndef TRMMKERNEL movq K, %rax #else @@ -777,9 +777,9 @@ #endif andq $3, %rax # if (k & 1) BRANCH - jle .L44 + jle L(44) -.L43: +L(43): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -799,10 +799,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): movapd ALPHA_R, %xmm6 movapd ALPHA_I, %xmm7 @@ -869,20 +869,20 @@ #endif ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif leaq (C, LDC, 2), C # c += 2 * ldc decq J # j -- - jg .L01 + jg L(01) -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) -.L101: +L(101): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -893,10 +893,10 @@ movq K, %rax sarq $2, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 movddup -14 * SIZE(B), %xmm10 @@ -918,17 +918,17 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): movq K, %rax andq $3, %rax BRANCH - jle .L105 + jle L(105) ALIGN_4 -.L104: +L(104): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 @@ -938,19 +938,19 @@ addq $4 * SIZE, BO addq $2 * SIZE, B decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L105: +L(105): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $1, I # i = (m >> 2) - jle .L130 + jle L(130) ALIGN_4 -.L110: +L(110): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -986,9 +986,9 @@ movq %rax, KKK #endif sarq $2, %rax - je .L112 + je L(112) -.L111: +L(111): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1066,10 +1066,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #ifndef TRMMKERNEL movq K, %rax #else @@ -1077,9 +1077,9 @@ #endif andq $3, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): movapd -16 * SIZE(AO), %xmm0 movapd -14 * SIZE(AO), %xmm1 @@ -1101,10 +1101,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): movapd ALPHA_R, %xmm6 movapd ALPHA_I, %xmm7 @@ -1171,15 +1171,15 @@ addq $4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L130: +L(130): testq $1, M - jle .L999 + jle L(999) ALIGN_4 -.L140: +L(140): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1214,9 +1214,9 @@ movq %rax, KKK #endif sarq $2, %rax - je .L142 + je L(142) -.L141: +L(141): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1256,9 +1256,9 @@ subq $ -8 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L141 + jne L(141) -.L142: +L(142): #ifndef TRMMKERNEL movq K, %rax #else @@ -1266,9 +1266,9 @@ #endif andq $3, %rax # if (k & 1) BRANCH - jle .L144 + jle L(144) -.L143: +L(143): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -1282,10 +1282,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L143 + jg L(143) ALIGN_4 -.L144: +L(144): movapd ALPHA_R, %xmm6 movapd ALPHA_I, %xmm7 @@ -1322,7 +1322,7 @@ movhpd %xmm8, 1 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/zgemm_kernel_2x2_penryn.S b/kernel/x86_64/zgemm_kernel_2x2_penryn.S index 24e1afdeaf..74bc014e60 100644 --- a/kernel/x86_64/zgemm_kernel_2x2_penryn.S +++ b/kernel/x86_64/zgemm_kernel_2x2_penryn.S @@ -204,10 +204,10 @@ movq N, J sarq $1, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -224,10 +224,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -282,10 +282,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm3, %xmm12 movaps -15 * SIZE(BO), %xmm3 @@ -413,10 +413,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): PREFETCHB -8 * SIZE(BB) #ifdef DUNNINGTON PREFETCHB 0 * SIZE(BB) @@ -430,10 +430,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm3, %xmm12 movaps -15 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 @@ -469,10 +469,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #ifndef DUNNINGTON subq $-16 * SIZE, BB #else @@ -579,13 +579,13 @@ addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M BRANCH - jle .L39 + jle L(39) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -631,10 +631,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_4 -.L22: +L(22): ADD1 %xmm3, %xmm12 movaps -15 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -700,10 +700,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -711,10 +711,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): ADD1 %xmm3, %xmm12 movaps -15 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -735,10 +735,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): ADD1 %xmm3, %xmm12 pcmpeqb %xmm0, %xmm0 ADD2 %xmm5, %xmm13 @@ -811,7 +811,7 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -821,13 +821,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $1, N BRANCH - jle .L999 + jle L(999) movq C, CO1 leaq (C, LDC, 1), CO2 @@ -841,10 +841,10 @@ movq M, I sarq $1, I # i = (m >> 2) NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -886,10 +886,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_4 -.L52: +L(52): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -964,10 +964,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_4 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -975,10 +975,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -1001,10 +1001,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): pcmpeqb %xmm0, %xmm0 movddup ALPHA_R, %xmm2 psllq $63, %xmm0 @@ -1074,13 +1074,13 @@ addq $4 * SIZE, CO2 decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L79 + jle L(79) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1121,10 +1121,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x4e, %xmm2, %xmm7 @@ -1167,10 +1167,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1178,10 +1178,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm7 @@ -1196,10 +1196,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1259,7 +1259,7 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addq $1, KK #endif @@ -1268,7 +1268,7 @@ movq BO, B ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm_kernel_2x2_piledriver.S b/kernel/x86_64/zgemm_kernel_2x2_piledriver.S index bffe5439db..16b7af9be6 100644 --- a/kernel/x86_64/zgemm_kernel_2x2_piledriver.S +++ b/kernel/x86_64/zgemm_kernel_2x2_piledriver.S @@ -476,13 +476,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -510,23 +510,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif #endif -.L2_0: +L(2_0): movq Ndiv6, J cmpq $0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovups (BO1), %xmm0 vmovups 2 * SIZE(BO1), %xmm1 @@ -535,13 +535,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $4*SIZE,BO1 addq $4*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -555,11 +555,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $1, I // i = (m >> 1) - je .L2_40 + je L(2_40) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -597,7 +597,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -608,7 +608,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -624,7 +624,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -640,12 +640,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -653,7 +653,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -665,14 +665,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL2x2_SUB(xxx) - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vmovddup ALPHA_R, %xmm0 vmovddup ALPHA_I, %xmm1 @@ -768,20 +768,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -819,7 +819,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax // K = K - ( K % 8 ) - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -830,7 +830,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_42: +L(2_42): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x2_1(xxx) @@ -846,7 +846,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x2_1(xxx) @@ -862,12 +862,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -875,7 +875,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -887,14 +887,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vmovddup ALPHA_R, %xmm0 vmovddup ALPHA_I, %xmm1 @@ -970,17 +970,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -988,30 +988,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -1025,11 +1025,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $1, I // i = (m >> 1) - je .L1_40 + je L(1_40) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1067,7 +1067,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1078,7 +1078,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -1092,7 +1092,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_16 + je L(1_16) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -1106,12 +1106,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -1119,7 +1119,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1131,14 +1131,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL2x1_SUB(xxx) - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vmovddup ALPHA_R, %xmm0 vmovddup ALPHA_I, %xmm1 @@ -1211,20 +1211,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1262,7 +1262,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax // K = K - ( K % 8 ) - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1273,7 +1273,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_42: +L(1_42): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x1_1(xxx) @@ -1287,7 +1287,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x1_1(xxx) @@ -1301,12 +1301,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -1314,7 +1314,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1326,14 +1326,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vmovddup ALPHA_R, %xmm0 vmovddup ALPHA_I, %xmm1 @@ -1397,7 +1397,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L999: +L(999): vzeroupper movq SP, %rsp diff --git a/kernel/x86_64/zgemm_kernel_2x2_sse2.S b/kernel/x86_64/zgemm_kernel_2x2_sse2.S index ae7bb8faff..fcc6903009 100644 --- a/kernel/x86_64/zgemm_kernel_2x2_sse2.S +++ b/kernel/x86_64/zgemm_kernel_2x2_sse2.S @@ -417,10 +417,10 @@ movq N, J sarq $1, J # j = (n >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L01: +L(01): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc @@ -433,10 +433,10 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): PREFETCH (RPREFETCHSIZE + 0) * SIZE(B) movq 0 * SIZE(B), %mm0 @@ -508,17 +508,17 @@ addq $ 32 * SIZE, BO subq $-16 * SIZE, B decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L05 + jle L(05) ALIGN_4 -.L04: +L(04): movq 0 * SIZE(B), %mm0 movq %mm0, -16 * SIZE(BO) movq %mm0, -15 * SIZE(BO) @@ -537,20 +537,20 @@ addq $ 8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L05: +L(05): movq A, AO # aoffset = a leaq (RPREFETCHSIZE + 0) * SIZE(B), BB movq M, I sarq $1, I # i = (m >> 2) - jle .L30 + jle L(30) ALIGN_4 -.L10: +L(10): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -608,10 +608,10 @@ leaq (BO, %rax, 8), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -632,7 +632,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -653,7 +653,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -674,7 +674,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -695,7 +695,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -716,7 +716,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -737,7 +737,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -758,7 +758,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -779,10 +779,10 @@ addq $8 * SIZE, %rax BRANCH - jl .L12 + jl L(12) ALIGN_3 -.L15: +L(15): PREFETCH 8 * SIZE(BB) subq $-16 * SIZE, BB @@ -792,7 +792,7 @@ movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_3 @@ -811,10 +811,10 @@ #else sarq $2, %rax NOBRANCH - jle .L16 + jle L(16) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -828,10 +828,10 @@ subq $-16 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) #endif -.L16: +L(16): movapd POSINV, %xmm5 movapd ALPHA_R, %xmm6 movapd ALPHA_I, %xmm7 @@ -842,7 +842,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L19 + je L(19) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -850,7 +850,7 @@ negq %rax ALIGN_3 -.L17: +L(17): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movapd -14 * SIZE(BO, %rax, 8), %xmm1 @@ -877,10 +877,10 @@ movapd -10 * SIZE(AO, %rax, 4), %xmm2 addq $SIZE, %rax - jl .L17 + jl L(17) ALIGN_3 -.L19: +L(19): #ifndef TRMMKERNEL movlpd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 @@ -977,12 +977,12 @@ addq $4 * SIZE, CO1 # coffset += 4 addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L30: +L(30): testq $1, M - jle .L99 + jle L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1022,10 +1022,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L44 + je L(44) ALIGN_4 -.L41: +L(41): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movapd -14 * SIZE(BO), %xmm1 @@ -1128,10 +1128,10 @@ subq $-16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L41 + jne L(41) ALIGN_4 -.L44: +L(44): #ifndef TRMMKERNEL movq K, %rax #else @@ -1139,7 +1139,7 @@ #endif andq $4, %rax BRANCH - jle .L45 + jle L(45) mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 @@ -1194,7 +1194,7 @@ addq $32 * SIZE, BO ALIGN_4 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1205,10 +1205,10 @@ movapd ALPHA_I, %xmm7 andq $3, %rax # if (k & 1) BRANCH - jle .L47 + jle L(47) ALIGN_4 -.L46: +L(46): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movapd -14 * SIZE(BO), %xmm1 @@ -1226,10 +1226,10 @@ addq $8 * SIZE, BO decq %rax - jg .L46 + jg L(46) ALIGN_4 -.L47: +L(47): #ifndef TRMMKERNEL movlpd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 @@ -1293,20 +1293,20 @@ #endif ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif leaq (C, LDC, 2), C # c += 2 * ldc decq J # j -- - jg .L01 + jg L(01) -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) -.L101: +L(101): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1317,10 +1317,10 @@ movq K, %rax sarq $2, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movlpd 0 * SIZE(B), %xmm8 movlpd 1 * SIZE(B), %xmm9 movlpd 2 * SIZE(B), %xmm10 @@ -1350,17 +1350,17 @@ subq $-16 * SIZE, BO addq $ 8 * SIZE, B decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): movq K, %rax andq $3, %rax BRANCH - jle .L105 + jle L(105) ALIGN_4 -.L104: +L(104): movlpd 0 * SIZE(B), %xmm8 movlpd 1 * SIZE(B), %xmm9 @@ -1372,19 +1372,19 @@ addq $4 * SIZE, BO addq $2 * SIZE, B decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L105: +L(105): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $1, I # i = (m >> 2) - jle .L130 + jle L(130) ALIGN_4 -.L110: +L(110): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1424,9 +1424,9 @@ movq %rax, KKK #endif sarq $2, %rax - je .L112 + je L(112) -.L111: +L(111): mulpd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd -14 * SIZE(BO), %xmm0 @@ -1481,10 +1481,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO decq %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #ifndef TRMMKERNEL movq K, %rax #else @@ -1495,9 +1495,9 @@ movapd ALPHA_I, %xmm7 andq $3, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO), %xmm0 addpd %xmm1, %xmm8 @@ -1514,10 +1514,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): #ifndef TRMMKERNEL movlpd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 @@ -1582,12 +1582,12 @@ addq $4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L130: +L(130): testq $1, M - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1628,10 +1628,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L144 + je L(144) ALIGN_4 -.L141: +L(141): mulpd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd -14 * SIZE(BO), %xmm0 @@ -1686,11 +1686,11 @@ subq $-16 * SIZE, AO subq $-32 * SIZE, BO decq %rax - jne .L141 + jne L(141) ALIGN_4 -.L144: +L(144): #ifndef TRMMKERNEL movq K, %rax #else @@ -1698,7 +1698,7 @@ #endif andq $4, %rax # if (k & 1) BRANCH - jle .L145 + jle L(145) mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO), %xmm0 @@ -1728,7 +1728,7 @@ subq $-16 * SIZE, BO ALIGN_4 -.L145: +L(145): movapd POSINV, %xmm5 movapd ALPHA_R, %xmm6 movapd ALPHA_I, %xmm7 @@ -1740,10 +1740,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - jle .L148 + jle L(148) ALIGN_4 -.L146: +L(146): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO), %xmm0 addpd %xmm1, %xmm8 @@ -1754,10 +1754,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1797,7 +1797,7 @@ movhpd %xmm8, 1 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp EMMS diff --git a/kernel/x86_64/zgemm_kernel_2x2_sse3.S b/kernel/x86_64/zgemm_kernel_2x2_sse3.S index b78f1a902b..e6a8f51ff6 100644 --- a/kernel/x86_64/zgemm_kernel_2x2_sse3.S +++ b/kernel/x86_64/zgemm_kernel_2x2_sse3.S @@ -409,10 +409,10 @@ salq $ZBASE_SHIFT, LDC movq N, J sarq $1, J # j = (n >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -428,10 +428,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L30 + jle L(30) ALIGN_4 -.L10: +L(10): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -486,9 +486,9 @@ andq $-8, %rax salq $4, %rax - je .L12 + je L(12) -.L1X: +L(1X): KERNEL1 (16 * 0) KERNEL2 (16 * 0) KERNEL3 (16 * 0) @@ -507,7 +507,7 @@ KERNEL16(16 * 0) cmpq $128 * 1, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 1) KERNEL2 (16 * 1) KERNEL3 (16 * 1) @@ -526,7 +526,7 @@ KERNEL16(16 * 1) cmpq $128 * 2, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 2) KERNEL2 (16 * 2) KERNEL3 (16 * 2) @@ -545,7 +545,7 @@ KERNEL16(16 * 2) cmpq $128 * 3, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 3) KERNEL2 (16 * 3) KERNEL3 (16 * 3) @@ -564,7 +564,7 @@ KERNEL16(16 * 3) cmpq $128 * 4, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 4) KERNEL2 (16 * 4) KERNEL3 (16 * 4) @@ -583,7 +583,7 @@ KERNEL16(16 * 4) cmpq $128 * 5, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 5) KERNEL2 (16 * 5) KERNEL3 (16 * 5) @@ -602,7 +602,7 @@ KERNEL16(16 * 5) cmpq $128 * 6, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 6) KERNEL2 (16 * 6) KERNEL3 (16 * 6) @@ -621,7 +621,7 @@ KERNEL16(16 * 6) cmpq $128 * 7, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 7) KERNEL2 (16 * 7) KERNEL3 (16 * 7) @@ -642,14 +642,14 @@ addq $32 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $128 * 8, %rax - jg .L1X + jg L(1X) -.L11: +L(11): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 2), BO # * 64 ALIGN_4 -.L12: +L(12): #ifndef TRMMKERNEL movq K, %rax #else @@ -659,10 +659,10 @@ movddup ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L14 + je L(14) ALIGN_4 -.L13: +L(13): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 ADD1 %xmm9, %xmm0 @@ -693,10 +693,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm3, %xmm3 SHUFPD_1 %xmm5, %xmm5 @@ -791,14 +791,14 @@ addq $4 * SIZE, CO1 # coffset += 4 addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L30: +L(30): testq $1, M - jle .L99 + jle L(99) -.L40: +L(40): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -836,9 +836,9 @@ movq %rax, KKK #endif sarq $3, %rax - je .L42 + je L(42) -.L41: +L(41): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -948,9 +948,9 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L41 + jne L(41) -.L42: +L(42): #ifndef TRMMKERNEL movq K, %rax #else @@ -960,9 +960,9 @@ movddup ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - jle .L44 + jle L(44) -.L43: +L(43): mulpd %xmm8, %xmm9 ADD1 %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -980,10 +980,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm3, %xmm3 @@ -1047,7 +1047,7 @@ #endif ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -1055,13 +1055,13 @@ leaq (C, LDC, 2), C # c += 2 * ldc movq BO, B decq J # j -- - jg .L01 + jg L(01) -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) -.L101: +L(101): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1072,10 +1072,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L130 + jle L(130) ALIGN_4 -.L110: +L(110): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1115,9 +1115,9 @@ movq %rax, KKK #endif sarq $3, %rax - je .L112 + je L(112) -.L111: +L(111): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -1236,10 +1236,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #ifndef TRMMKERNEL movq K, %rax #else @@ -1249,9 +1249,9 @@ movddup ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 ADD1 %xmm9, %xmm0 @@ -1270,10 +1270,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm5, %xmm5 @@ -1337,15 +1337,15 @@ addq $4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L130: +L(130): testq $1, M - jle .L999 + jle L(999) ALIGN_4 -.L140: +L(140): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1383,9 +1383,9 @@ movq %rax, KKK #endif sarq $3, %rax - je .L142 + je L(142) -.L141: +L(141): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -1447,9 +1447,9 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L141 + jne L(141) -.L142: +L(142): #ifndef TRMMKERNEL movq K, %rax #else @@ -1459,9 +1459,9 @@ movddup ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - jle .L144 + jle L(144) -.L143: +L(143): mulpd %xmm8, %xmm9 ADD1 %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -1473,10 +1473,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L143 + jg L(143) ALIGN_4 -.L144: +L(144): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -1510,7 +1510,7 @@ movhpd %xmm0, 1 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm_kernel_2x4_nehalem.S b/kernel/x86_64/zgemm_kernel_2x4_nehalem.S index ce60123884..b57f256b94 100644 --- a/kernel/x86_64/zgemm_kernel_2x4_nehalem.S +++ b/kernel/x86_64/zgemm_kernel_2x4_nehalem.S @@ -184,10 +184,10 @@ movq N, J sarq $2, J NOBRANCH - jle .L30 + jle L(30) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -204,10 +204,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): prefetcht2 -32 * SIZE(BB) subq $-16 * SIZE, BB @@ -263,10 +263,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm12 @@ -389,10 +389,10 @@ subq $-16 * SIZE, AO subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else @@ -400,10 +400,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm1, %xmm12 movaps -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm13 @@ -439,10 +439,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): ADD1 %xmm1, %xmm12 ADD2 %xmm2, %xmm13 ADD1 %xmm3, %xmm14 @@ -546,13 +546,13 @@ addq $4 * SIZE, CO2 decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M BRANCH - jle .L29 + jle L(29) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -597,10 +597,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -670,10 +670,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -681,10 +681,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -706,10 +706,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): ADD1 %xmm1, %xmm8 ADD2 %xmm2, %xmm9 ADD1 %xmm3, %xmm10 @@ -785,7 +785,7 @@ #endif ALIGN_4 -.L29: +L(29): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif @@ -795,13 +795,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L30: +L(30): testq $2, N BRANCH - jle .L50 + jle L(50) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -815,10 +815,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L31: +L(31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -864,10 +864,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -931,10 +931,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -942,10 +942,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): ADD1 %xmm1, %xmm8 movaps -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -965,10 +965,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): ADD1 %xmm1, %xmm8 ADD2 %xmm2, %xmm9 ADD1 %xmm3, %xmm10 @@ -1047,13 +1047,13 @@ addq $4 * SIZE, CO2 decq I # i -- BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L40: +L(40): testq $1, M BRANCH - jle .L49 + jle L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1098,10 +1098,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1143,10 +1143,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1154,10 +1154,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1172,10 +1172,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): ADD1 %xmm1, %xmm8 ADD2 %xmm2, %xmm9 @@ -1232,7 +1232,7 @@ #endif ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -1241,10 +1241,10 @@ movq BO, B ALIGN_4 -.L50: +L(50): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1257,10 +1257,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1305,10 +1305,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -1348,10 +1348,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1359,10 +1359,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): ADD1 %xmm1, %xmm8 movddup -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -1376,10 +1376,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): ADD1 %xmm1, %xmm8 ADD2 %xmm2, %xmm9 @@ -1435,13 +1435,13 @@ addq $4 * SIZE, CO1 decq I # i -- BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1481,10 +1481,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1526,10 +1526,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1537,10 +1537,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1555,10 +1555,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): ADD1 %xmm1, %xmm8 ADD2 %xmm2, %xmm9 @@ -1599,7 +1599,7 @@ movsd %xmm8, (CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm_kernel_4x2_barcelona.S b/kernel/x86_64/zgemm_kernel_4x2_barcelona.S index 06d0bbe14b..2d66045ebd 100644 --- a/kernel/x86_64/zgemm_kernel_4x2_barcelona.S +++ b/kernel/x86_64/zgemm_kernel_4x2_barcelona.S @@ -502,10 +502,10 @@ salq $ZBASE_SHIFT, LDC movq N, J sarq $1, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -517,12 +517,12 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): prefetch (RPREFETCHSIZE + 0) * SIZE(B) movaps 0 * SIZE(B), %xmm3 @@ -568,17 +568,17 @@ addq $32 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movaps 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -603,10 +603,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a @@ -615,10 +615,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -677,10 +677,10 @@ leaq (BO, %rax, 8), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_3 -.L12: +L(12): KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -690,7 +690,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -700,7 +700,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -710,7 +710,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -720,7 +720,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -730,7 +730,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -740,7 +740,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -750,7 +750,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -760,17 +760,17 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) BRANCH - jl .L12 + jl L(12) ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_3 @@ -783,7 +783,7 @@ addq $32 * SIZE, AO ALIGN_3 -.L16: +L(16): #ifndef TRMMKERNEL movq K, %rax #else @@ -793,7 +793,7 @@ movaps ALPHA_I, %xmm7 andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) leaq (, %rax, 8), %rax leaq (AO, %rax, 4), AO @@ -801,7 +801,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulps %xmm1, %xmm0 mulps -28 * SIZE(AO, %rax, 4), %xmm1 addps %xmm0, %xmm8 @@ -828,10 +828,10 @@ movaps %xmm0, %xmm2 addq $SIZE * 2, %rax - jl .L17 + jl L(17) ALIGN_4 -.L18: +L(18): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -920,12 +920,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -971,10 +971,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -1085,10 +1085,10 @@ subq $-128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1098,10 +1098,10 @@ movaps ALPHA_I, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm0, %xmm1 addps %xmm1, %xmm8 movaps -28 * SIZE(BO), %xmm1 @@ -1118,10 +1118,10 @@ subq $- 4 * SIZE, AO subq $-16 * SIZE, BO decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -1184,9 +1184,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1230,10 +1230,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -1351,10 +1351,10 @@ subq $-128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1364,10 +1364,10 @@ movaps ALPHA_I, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm0, %xmm1 addps %xmm1, %xmm8 movsd -28 * SIZE(BO), %xmm1 @@ -1385,10 +1385,10 @@ subq $ -2 * SIZE, AO subq $-16 * SIZE, BO decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movsd 0 * SIZE(CO2), %xmm1 @@ -1442,22 +1442,22 @@ #endif ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif leaq (C, LDC, 2), C # c += 2 * ldc decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L41: +L(41): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1469,10 +1469,10 @@ movq K, %rax sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): prefetch (RPREFETCHSIZE + 0) * SIZE(B) movaps 0 * SIZE(B), %xmm3 @@ -1518,17 +1518,17 @@ addq $32 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): movq K, %rax andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movsd 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -1547,19 +1547,19 @@ addq $2 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1610,10 +1610,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulps %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulps -28 * SIZE(BO), %xmm0 @@ -1729,10 +1729,10 @@ subq $-64 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1742,10 +1742,10 @@ movaps ALPHA_I, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): mulps %xmm0, %xmm1 mulps -28 * SIZE(BO), %xmm0 addps %xmm1, %xmm8 @@ -1762,10 +1762,10 @@ addq $ 8 * SIZE, AO addq $ 8 * SIZE, BO decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -1824,12 +1824,12 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1873,10 +1873,10 @@ #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulps -28 * SIZE(BO), %xmm0 @@ -1936,10 +1936,10 @@ subq $-64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1949,10 +1949,10 @@ movaps ALPHA_I, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm0, %xmm1 mulps -28 * SIZE(BO), %xmm0 addps %xmm1, %xmm8 @@ -1963,10 +1963,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -2016,9 +2016,9 @@ addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L70: +L(70): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2061,10 +2061,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -2134,10 +2134,10 @@ subq $-64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2147,10 +2147,10 @@ movaps ALPHA_I, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm0, %xmm1 addps %xmm1, %xmm8 movsd -28 * SIZE(BO), %xmm1 @@ -2162,10 +2162,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 #endif @@ -2196,7 +2196,7 @@ movsd %xmm8, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/zgemm_kernel_4x2_core2.S b/kernel/x86_64/zgemm_kernel_4x2_core2.S index bb4584a6b7..ac60c97949 100644 --- a/kernel/x86_64/zgemm_kernel_4x2_core2.S +++ b/kernel/x86_64/zgemm_kernel_4x2_core2.S @@ -180,10 +180,10 @@ salq $ZBASE_SHIFT, LDC movq N, J sarq $1, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -196,10 +196,10 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): prefetcht0 (PREFETCH_R + 0) * SIZE(B) movaps -28 * SIZE(B), %xmm7 movaps -24 * SIZE(B), %xmm11 @@ -252,17 +252,17 @@ subq $-16 * SIZE, B subq $-64 * SIZE, BO subq $1, %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): pshufd $0x00, %xmm3, %xmm0 movaps %xmm0, -32 * SIZE(BO) pshufd $0x55, %xmm3, %xmm1 @@ -277,10 +277,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO subq $1, %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): leaq (PREFETCH_R + 0) * SIZE(B), BB movq C, CO1 @@ -289,10 +289,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -348,10 +348,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): addps %xmm2, %xmm10 movaps -32 * SIZE(BO), %xmm2 addps %xmm3, %xmm14 @@ -480,10 +480,10 @@ subq $-64 * SIZE, BO subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): prefetcht2 -16 * SIZE(BB) #ifndef TRMMKERNEL @@ -493,10 +493,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): addps %xmm2, %xmm10 movaps -32 * SIZE(BO), %xmm2 addps %xmm3, %xmm14 @@ -530,10 +530,10 @@ movaps -28 * SIZE(AO), %xmm1 subq $1, %rax - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): movaps ALPHA_R, %xmm6 movaps ALPHA_I, %xmm7 @@ -646,12 +646,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -687,10 +687,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -760,10 +760,10 @@ subq $-16 * SIZE, AO subq $-64 * SIZE, BO subq $1, %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -771,10 +771,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): movaps -32 * SIZE(AO), %xmm0 movaps -32 * SIZE(BO), %xmm2 movaps -28 * SIZE(BO), %xmm3 @@ -794,10 +794,10 @@ addq $ 4 * SIZE, AO addq $16 * SIZE, BO subq $1, %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): movaps ALPHA_R, %xmm6 movaps ALPHA_I, %xmm7 @@ -869,9 +869,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -907,10 +907,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -32 * SIZE(AO), %xmm0 @@ -980,10 +980,10 @@ subq $ -8 * SIZE, AO subq $-64 * SIZE, BO subq $1, %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -991,10 +991,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): movsd -32 * SIZE(AO), %xmm0 movsd -32 * SIZE(BO), %xmm2 movsd -28 * SIZE(BO), %xmm3 @@ -1014,10 +1014,10 @@ addq $ 2 * SIZE, AO addq $16 * SIZE, BO subq $1, %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): movaps ALPHA_R, %xmm6 movaps ALPHA_I, %xmm7 @@ -1082,22 +1082,22 @@ #endif ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif leaq (C, LDC, 2), C # c += 2 * ldc decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L41: +L(41): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1108,10 +1108,10 @@ movq K, %rax sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): movss -32 * SIZE(B), %xmm8 movss -31 * SIZE(B), %xmm9 movss -30 * SIZE(B), %xmm10 @@ -1143,17 +1143,17 @@ addq $32 * SIZE, BO subq $1, %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): movq K, %rax andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movss -32 * SIZE(B), %xmm8 movss -31 * SIZE(B), %xmm9 @@ -1166,19 +1166,19 @@ addq $2 * SIZE, B addq $8 * SIZE, BO subq $1, %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1214,10 +1214,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -1295,10 +1295,10 @@ subq $-32 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1306,10 +1306,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): movaps -32 * SIZE(AO), %xmm0 movaps -28 * SIZE(AO), %xmm1 @@ -1331,10 +1331,10 @@ addq $8 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): movaps ALPHA_R, %xmm6 movaps ALPHA_I, %xmm7 @@ -1404,12 +1404,12 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1445,10 +1445,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -1488,10 +1488,10 @@ subq $-16 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1499,10 +1499,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): movaps -32 * SIZE(AO), %xmm0 movaps -32 * SIZE(BO), %xmm2 movaps -28 * SIZE(BO), %xmm3 @@ -1516,10 +1516,10 @@ addq $4 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): movaps ALPHA_R, %xmm6 movaps ALPHA_I, %xmm7 @@ -1568,9 +1568,9 @@ addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L70: +L(70): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1606,10 +1606,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -32 * SIZE(AO), %xmm0 @@ -1649,10 +1649,10 @@ subq $ -8 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -1660,10 +1660,10 @@ #endif andq $3, %rax BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): movsd -32 * SIZE(AO), %xmm0 movsd -32 * SIZE(BO), %xmm2 movsd -28 * SIZE(BO), %xmm3 @@ -1677,10 +1677,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): movaps ALPHA_R, %xmm6 movaps ALPHA_I, %xmm7 @@ -1713,7 +1713,7 @@ movsd %xmm8, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/zgemm_kernel_4x2_haswell.S b/kernel/x86_64/zgemm_kernel_4x2_haswell.S index 29729b1017..a8712730cf 100644 --- a/kernel/x86_64/zgemm_kernel_4x2_haswell.S +++ b/kernel/x86_64/zgemm_kernel_4x2_haswell.S @@ -1107,13 +1107,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $ 0, OLD_M - je .L999 + je L(999) cmpq $ 0, OLD_N - je .L999 + je L(999) cmpq $ 0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -1134,16 +1134,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /************************************************************************************************/ -.L6_00_0: +L(6_00_0): movq Ndiv6, J cmpq $ 0, J - je .L2_00_0 + je L(2_00_0) ALIGN_4 -.L6_00_01: +L(6_00_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO @@ -1154,7 +1154,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax ALIGN_4 -.L6_00_02b: +L(6_00_02b): vmovups (BO1), %xmm0 vmovups 2 * SIZE(BO1), %xmm1 @@ -1166,13 +1166,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4*SIZE,BO2 addq $ 6*SIZE,BO decq %rax - jnz .L6_00_02b + jnz L(6_00_02b) -.L6_00_02c: +L(6_00_02c): -.L6_00_10: +L(6_00_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc leaq (C, LDC, 1), C // c += 1 * ldc @@ -1181,13 +1181,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 2, I // i = (m >> 2) - je .L6_2_10 + je L(6_2_10) ALIGN_4 /******************************************************************************************************************/ -.L6_4_11: +L(6_4_11): leaq BUFFER1, BO // first buffer to BO @@ -1196,10 +1196,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L6_4_16 + je L(6_4_16) ALIGN_4 -.L6_4_12: +L(6_4_12): KERNEL4x3_SUB KERNEL4x3_SUB @@ -1211,7 +1211,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_SUB KERNEL4x3_SUB - je .L6_4_16 + je L(6_4_16) KERNEL4x3_SUB KERNEL4x3_SUB @@ -1223,33 +1223,33 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_SUB KERNEL4x3_SUB - je .L6_4_16 + je L(6_4_16) - jmp .L6_4_12 + jmp L(6_4_12) ALIGN_4 -.L6_4_16: +L(6_4_16): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L6_4_19 + je L(6_4_19) ALIGN_4 -.L6_4_17: +L(6_4_17): KERNEL4x3_SUB - jnz .L6_4_17 + jnz L(6_4_17) ALIGN_4 -.L6_4_19: +L(6_4_19): SAVE4x3 addq $ 8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L6_4_11 + jg L(6_4_11) ALIGN_4 @@ -1259,11 +1259,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************************************************/ -.L6_2_10: +L(6_2_10): testq $ 2, M - jz .L6_2_40 // to next 2 lines of N + jz L(6_2_40) // to next 2 lines of N -.L6_2_11: +L(6_2_11): leaq BUFFER1, BO // first buffer to BO @@ -1272,10 +1272,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L6_2_16 + je L(6_2_16) ALIGN_4 -.L6_2_12: +L(6_2_12): KERNEL2x3_SUB KERNEL2x3_SUB @@ -1287,7 +1287,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_SUB KERNEL2x3_SUB - je .L6_2_16 + je L(6_2_16) KERNEL2x3_SUB KERNEL2x3_SUB @@ -1299,27 +1299,27 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_SUB KERNEL2x3_SUB - je .L6_2_16 + je L(6_2_16) - jmp .L6_2_12 + jmp L(6_2_12) ALIGN_4 -.L6_2_16: +L(6_2_16): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L6_2_19 + je L(6_2_19) ALIGN_4 -.L6_2_17: +L(6_2_17): KERNEL2x3_SUB - jnz .L6_2_17 + jnz L(6_2_17) ALIGN_4 -.L6_2_19: +L(6_2_19): SAVE2x3 @@ -1330,13 +1330,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /************************************************************************** * Rest of M ***************************************************************************/ -.L6_2_40: +L(6_2_40): testq $ 1, M - jz .L6_2_60 // to next 2 lines of N + jz L(6_2_60) // to next 2 lines of N ALIGN_4 -.L6_2_41: +L(6_2_41): leaq BUFFER1, BO // first buffer to BO @@ -1345,11 +1345,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L6_2_46 + je L(6_2_46) ALIGN_4 -.L6_2_42: +L(6_2_42): KERNEL1x3_SUB KERNEL1x3_SUB @@ -1361,7 +1361,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_SUB KERNEL1x3_SUB - je .L6_2_46 + je L(6_2_46) KERNEL1x3_SUB KERNEL1x3_SUB @@ -1373,40 +1373,40 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_SUB KERNEL1x3_SUB - je .L6_2_46 + je L(6_2_46) - jmp .L6_2_42 + jmp L(6_2_42) ALIGN_4 -.L6_2_46: +L(6_2_46): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L6_2_49 + je L(6_2_49) ALIGN_4 -.L6_2_47: +L(6_2_47): KERNEL1x3_SUB - jnz .L6_2_47 + jnz L(6_2_47) ALIGN_4 -.L6_2_49: +L(6_2_49): SAVE1x3 addq $ 2 * SIZE, CO1 # coffset += 2 decq I # i -- - jg .L6_2_41 + jg L(6_2_41) ALIGN_4 -.L6_2_60: +L(6_2_60): /************************************************************************************************/ @@ -1414,7 +1414,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /************************************************************************************************/ -.L7_00_01: +L(7_00_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO @@ -1424,7 +1424,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax ALIGN_4 -.L7_00_02b: +L(7_00_02b): vmovups 2 * SIZE(BO1), %xmm0 vmovups (BO2), %xmm1 @@ -1436,14 +1436,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4*SIZE,BO2 addq $ 6*SIZE,BO decq %rax - jnz .L7_00_02b + jnz L(7_00_02b) -.L7_00_02c: +L(7_00_02c): movq BO2, B // next offset of B -.L7_00_10: +L(7_00_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc leaq (C, LDC, 1), C // c += 1 * ldc @@ -1452,13 +1452,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 2, I // i = (m >> 2) - je .L7_2_10 + je L(7_2_10) ALIGN_4 /******************************************************************************************************************/ -.L7_4_11: +L(7_4_11): leaq BUFFER1, BO // first buffer to BO @@ -1467,10 +1467,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L7_4_16 + je L(7_4_16) ALIGN_4 -.L7_4_12: +L(7_4_12): KERNEL4x3_SUB KERNEL4x3_SUB @@ -1482,7 +1482,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_SUB KERNEL4x3_SUB - je .L7_4_16 + je L(7_4_16) KERNEL4x3_SUB KERNEL4x3_SUB @@ -1494,34 +1494,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_SUB KERNEL4x3_SUB - je .L7_4_16 + je L(7_4_16) - jmp .L7_4_12 + jmp L(7_4_12) ALIGN_4 -.L7_4_16: +L(7_4_16): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L7_4_19 + je L(7_4_19) ALIGN_4 -.L7_4_17: +L(7_4_17): KERNEL4x3_SUB - jnz .L7_4_17 + jnz L(7_4_17) ALIGN_4 -.L7_4_19: +L(7_4_19): SAVE4x3 addq $ 8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L7_4_11 + jg L(7_4_11) ALIGN_4 @@ -1531,11 +1531,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************************************************/ -.L7_2_10: +L(7_2_10): testq $ 2, M - jz .L7_2_40 // to next 2 lines of N + jz L(7_2_40) // to next 2 lines of N -.L7_2_11: +L(7_2_11): leaq BUFFER1, BO // first buffer to BO @@ -1544,10 +1544,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L7_2_16 + je L(7_2_16) ALIGN_4 -.L7_2_12: +L(7_2_12): KERNEL2x3_SUB KERNEL2x3_SUB @@ -1559,7 +1559,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_SUB KERNEL2x3_SUB - je .L7_2_16 + je L(7_2_16) KERNEL2x3_SUB KERNEL2x3_SUB @@ -1571,28 +1571,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_SUB KERNEL2x3_SUB - je .L7_2_16 + je L(7_2_16) - jmp .L7_2_12 + jmp L(7_2_12) ALIGN_4 -.L7_2_16: +L(7_2_16): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L7_2_19 + je L(7_2_19) ALIGN_4 -.L7_2_17: +L(7_2_17): KERNEL2x3_SUB - jnz .L7_2_17 + jnz L(7_2_17) ALIGN_4 -.L7_2_19: +L(7_2_19): SAVE2x3 @@ -1603,13 +1603,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /************************************************************************** * Rest of M ***************************************************************************/ -.L7_2_40: +L(7_2_40): testq $ 1, M - jz .L7_2_60 // to next 2 lines of N + jz L(7_2_60) // to next 2 lines of N ALIGN_4 -.L7_2_41: +L(7_2_41): leaq BUFFER1, BO // first buffer to BO @@ -1618,11 +1618,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L7_2_46 + je L(7_2_46) ALIGN_4 -.L7_2_42: +L(7_2_42): KERNEL1x3_SUB KERNEL1x3_SUB @@ -1634,7 +1634,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_SUB KERNEL1x3_SUB - je .L7_2_46 + je L(7_2_46) KERNEL1x3_SUB KERNEL1x3_SUB @@ -1646,66 +1646,66 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_SUB KERNEL1x3_SUB - je .L7_2_46 + je L(7_2_46) - jmp .L7_2_42 + jmp L(7_2_42) ALIGN_4 -.L7_2_46: +L(7_2_46): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L7_2_49 + je L(7_2_49) ALIGN_4 -.L7_2_47: +L(7_2_47): KERNEL1x3_SUB - jnz .L7_2_47 + jnz L(7_2_47) ALIGN_4 -.L7_2_49: +L(7_2_49): SAVE1x3 addq $ 2 * SIZE, CO1 # coffset += 2 decq I # i -- - jg .L7_2_41 + jg L(7_2_41) ALIGN_4 -.L7_2_60: +L(7_2_60): decq J // j -- - jg .L6_00_01 // next 6 lines of N + jg L(6_00_01) // next 6 lines of N /************************************************************************************************/ /************************************************************************************************/ -.L2_00_0: +L(2_00_0): movq Nmod6, J sarq $1, J // j = j / 2 cmpq $ 0, J - je .L1_2_0 + je L(1_2_0) ALIGN_4 -.L2_00_01: +L(2_00_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_00_02b: +L(2_00_02b): vmovups (BO1), %xmm0 vmovups 2 * SIZE(BO1), %xmm1 @@ -1714,14 +1714,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4*SIZE,BO1 addq $ 4*SIZE,BO decq %rax - jnz .L2_00_02b + jnz L(2_00_02b) -.L2_00_02c: +L(2_00_02c): movq BO1, B // next offset of B -.L2_00_10: +L(2_00_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -1735,13 +1735,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 2, I // i = (m >> 2) - je .L2_2_10 + je L(2_2_10) ALIGN_4 /******************************************************************************************************************/ -.L2_4_11: +L(2_4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1779,7 +1779,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_4_16 + je L(2_4_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1790,7 +1790,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_12: +L(2_4_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI ,SIZE) @@ -1814,7 +1814,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL4x2_SUB - je .L2_4_16 + je L(2_4_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI ,SIZE) @@ -1838,12 +1838,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL4x2_SUB - je .L2_4_16 + je L(2_4_16) - jmp .L2_4_12 + jmp L(2_4_12) ALIGN_4 -.L2_4_16: +L(2_4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -1851,7 +1851,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_4_19 + je L(2_4_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1863,15 +1863,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_17: +L(2_4_17): KERNEL4x2_SUB - jl .L2_4_17 + jl L(2_4_17) ALIGN_4 -.L2_4_19: +L(2_4_19): SAVE4x2 @@ -1893,7 +1893,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L2_4_11 + jg L(2_4_11) ALIGN_4 @@ -1903,11 +1903,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************************************************/ -.L2_2_10: +L(2_2_10): testq $ 2, M - jz .L2_2_40 // to next 2 lines of N + jz L(2_2_40) // to next 2 lines of N -.L2_2_11: +L(2_2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1945,7 +1945,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_2_16 + je L(2_2_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1956,7 +1956,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_2_12: +L(2_2_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -1976,7 +1976,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_2_16 + je L(2_2_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -1996,12 +1996,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_2_16 + je L(2_2_16) - jmp .L2_2_12 + jmp L(2_2_12) ALIGN_4 -.L2_2_16: +L(2_2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -2009,7 +2009,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_2_19 + je L(2_2_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2021,15 +2021,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_2_17: +L(2_2_17): KERNEL2x2_SUB - jl .L2_2_17 + jl L(2_2_17) ALIGN_4 -.L2_2_19: +L(2_2_19): SAVE2x2 @@ -2056,13 +2056,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /************************************************************************** * Rest of M ***************************************************************************/ -.L2_2_40: +L(2_2_40): testq $ 1, M - jz .L2_2_60 // to next 2 lines of N + jz L(2_2_60) // to next 2 lines of N ALIGN_4 -.L2_2_41: +L(2_2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2100,7 +2100,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_2_46 + je L(2_2_46) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2111,7 +2111,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_2_42: +L(2_2_42): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2129,7 +2129,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_2_46 + je L(2_2_46) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2147,12 +2147,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_2_46 + je L(2_2_46) - jmp .L2_2_42 + jmp L(2_2_42) ALIGN_4 -.L2_2_46: +L(2_2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -2160,7 +2160,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_2_49 + je L(2_2_49) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2172,15 +2172,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_2_47: +L(2_2_47): KERNEL1x2_SUB - jl .L2_2_47 + jl L(2_2_47) ALIGN_4 -.L2_2_49: +L(2_2_49): SAVE1x2 @@ -2202,23 +2202,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 2 * SIZE, CO1 # coffset += 2 decq I # i -- - jg .L2_2_41 + jg L(2_2_41) ALIGN_4 -.L2_2_60: +L(2_2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $ 2, KK #endif decq J // j -- - jg .L2_00_01 // next 2 lines of N + jg L(2_00_01) // next 2 lines of N -.L1_2_0: +L(1_2_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -2226,30 +2226,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $ 1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_00_01: +L(1_00_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_00_02b: +L(1_00_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $ 2*SIZE,BO1 addq $ 2*SIZE,BO decq %rax - jnz .L1_00_02b + jnz L(1_00_02b) -.L1_00_02c: +L(1_00_02c): movq BO1, B // next offset of B -.L1_00_10: +L(1_00_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -2263,14 +2263,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 2, I // i = (m >> 2) - je .L1_2_10 + je L(1_2_10) ALIGN_4 /*******************************************************************************************************/ -.L1_4_11: +L(1_4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2308,7 +2308,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_4_16 + je L(1_4_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -2319,7 +2319,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_12: +L(1_4_12): KERNEL4x1_SUB KERNEL4x1_SUB @@ -2331,7 +2331,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_4_16 + je L(1_4_16) KERNEL4x1_SUB KERNEL4x1_SUB @@ -2343,12 +2343,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_4_16 + je L(1_4_16) - jmp .L1_4_12 + jmp L(1_4_12) ALIGN_4 -.L1_4_16: +L(1_4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -2356,7 +2356,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_4_19 + je L(1_4_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -2368,15 +2368,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_17: +L(1_4_17): KERNEL4x1_SUB - jl .L1_4_17 + jl L(1_4_17) ALIGN_4 -.L1_4_19: +L(1_4_19): SAVE4x1 @@ -2399,19 +2399,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L1_4_11 + jg L(1_4_11) ALIGN_4 /*******************************************************************************************************/ -.L1_2_10: +L(1_2_10): testq $ 2, M - jz .L1_2_40 + jz L(1_2_40) -.L1_2_11: +L(1_2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2449,7 +2449,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_2_16 + je L(1_2_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -2460,7 +2460,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_2_12: +L(1_2_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2477,7 +2477,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_2_16 + je L(1_2_16) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_SUB @@ -2493,12 +2493,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_2_16 + je L(1_2_16) - jmp .L1_2_12 + jmp L(1_2_12) ALIGN_4 -.L1_2_16: +L(1_2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -2506,7 +2506,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_2_19 + je L(1_2_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -2518,15 +2518,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_2_17: +L(1_2_17): KERNEL2x1_SUB - jl .L1_2_17 + jl L(1_2_17) ALIGN_4 -.L1_2_19: +L(1_2_19): SAVE2x1 @@ -2554,13 +2554,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /************************************************************************** * Rest of M ***************************************************************************/ -.L1_2_40: +L(1_2_40): testq $ 1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_2_41: +L(1_2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2598,7 +2598,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_2_46 + je L(1_2_46) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -2609,7 +2609,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_2_42: +L(1_2_42): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2625,7 +2625,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_2_46 + je L(1_2_46) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2641,12 +2641,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_2_46 + je L(1_2_46) - jmp .L1_2_42 + jmp L(1_2_42) ALIGN_4 -.L1_2_46: +L(1_2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -2654,7 +2654,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_2_49 + je L(1_2_49) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -2666,15 +2666,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_2_47: +L(1_2_47): KERNEL1x1_SUB - jl .L1_2_47 + jl L(1_2_47) ALIGN_4 -.L1_2_49: +L(1_2_49): SAVE1x1 @@ -2696,7 +2696,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 2 * SIZE, CO1 # coffset += 2 decq I # i -- - jg .L1_2_41 + jg L(1_2_41) ALIGN_4 @@ -2704,7 +2704,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L999: +L(999): vzeroupper movq SP, %rsp @@ -2796,13 +2796,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $ 0, OLD_M - je .L999 + je L(999) cmpq $ 0, OLD_N - je .L999 + je L(999) cmpq $ 0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -2830,23 +2830,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif #endif -.L2_00_0: +L(2_00_0): movq Ndiv6, J cmpq $ 0, J - je .L1_2_0 + je L(1_2_0) ALIGN_4 -.L2_00_01: +L(2_00_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_00_02b: +L(2_00_02b): vmovups (BO1), %xmm0 vmovups 2 * SIZE(BO1), %xmm1 @@ -2855,14 +2855,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4*SIZE,BO1 addq $ 4*SIZE,BO decq %rax - jnz .L2_00_02b + jnz L(2_00_02b) -.L2_00_02c: +L(2_00_02c): movq BO1, B // next offset of B -.L2_00_10: +L(2_00_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -2876,13 +2876,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 2, I // i = (m >> 2) - je .L2_2_10 + je L(2_2_10) ALIGN_4 /******************************************************************************************************************/ -.L2_4_11: +L(2_4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2920,7 +2920,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_4_16 + je L(2_4_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2931,7 +2931,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_12: +L(2_4_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI ,SIZE) @@ -2955,7 +2955,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL4x2_SUB - je .L2_4_16 + je L(2_4_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI ,SIZE) @@ -2979,12 +2979,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL4x2_SUB - je .L2_4_16 + je L(2_4_16) - jmp .L2_4_12 + jmp L(2_4_12) ALIGN_4 -.L2_4_16: +L(2_4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -2992,7 +2992,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_4_19 + je L(2_4_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -3004,15 +3004,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_17: +L(2_4_17): KERNEL4x2_SUB - jl .L2_4_17 + jl L(2_4_17) ALIGN_4 -.L2_4_19: +L(2_4_19): SAVE4x2 @@ -3034,7 +3034,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L2_4_11 + jg L(2_4_11) ALIGN_4 @@ -3044,11 +3044,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************************************************/ -.L2_2_10: +L(2_2_10): testq $ 2, M - jz .L2_2_40 // to next 2 lines of N + jz L(2_2_40) // to next 2 lines of N -.L2_2_11: +L(2_2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3086,7 +3086,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_2_16 + je L(2_2_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -3097,7 +3097,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_2_12: +L(2_2_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3117,7 +3117,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_2_16 + je L(2_2_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3137,12 +3137,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_2_16 + je L(2_2_16) - jmp .L2_2_12 + jmp L(2_2_12) ALIGN_4 -.L2_2_16: +L(2_2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3150,7 +3150,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_2_19 + je L(2_2_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -3162,15 +3162,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_2_17: +L(2_2_17): KERNEL2x2_SUB - jl .L2_2_17 + jl L(2_2_17) ALIGN_4 -.L2_2_19: +L(2_2_19): SAVE2x2 @@ -3197,13 +3197,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /************************************************************************** * Rest of M ***************************************************************************/ -.L2_2_40: +L(2_2_40): testq $ 1, M - jz .L2_2_60 // to next 2 lines of N + jz L(2_2_60) // to next 2 lines of N ALIGN_4 -.L2_2_41: +L(2_2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3241,7 +3241,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_2_46 + je L(2_2_46) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -3252,7 +3252,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_2_42: +L(2_2_42): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3270,7 +3270,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_2_46 + je L(2_2_46) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3288,12 +3288,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_2_46 + je L(2_2_46) - jmp .L2_2_42 + jmp L(2_2_42) ALIGN_4 -.L2_2_46: +L(2_2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -3301,7 +3301,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_2_49 + je L(2_2_49) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -3313,15 +3313,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_2_47: +L(2_2_47): KERNEL1x2_SUB - jl .L2_2_47 + jl L(2_2_47) ALIGN_4 -.L2_2_49: +L(2_2_49): SAVE1x2 @@ -3343,23 +3343,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 2 * SIZE, CO1 # coffset += 2 decq I # i -- - jg .L2_2_41 + jg L(2_2_41) ALIGN_4 -.L2_2_60: +L(2_2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $ 2, KK #endif decq J // j -- - jg .L2_00_01 // next 2 lines of N + jg L(2_00_01) // next 2 lines of N -.L1_2_0: +L(1_2_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -3367,30 +3367,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $ 1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_00_01: +L(1_00_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_00_02b: +L(1_00_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $ 2*SIZE,BO1 addq $ 2*SIZE,BO decq %rax - jnz .L1_00_02b + jnz L(1_00_02b) -.L1_00_02c: +L(1_00_02c): movq BO1, B // next offset of B -.L1_00_10: +L(1_00_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -3404,14 +3404,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 2, I // i = (m >> 2) - je .L1_2_10 + je L(1_2_10) ALIGN_4 /*******************************************************************************************************/ -.L1_4_11: +L(1_4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3449,7 +3449,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_4_16 + je L(1_4_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -3460,7 +3460,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_12: +L(1_4_12): KERNEL4x1_SUB KERNEL4x1_SUB @@ -3472,7 +3472,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_4_16 + je L(1_4_16) KERNEL4x1_SUB KERNEL4x1_SUB @@ -3484,12 +3484,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_4_16 + je L(1_4_16) - jmp .L1_4_12 + jmp L(1_4_12) ALIGN_4 -.L1_4_16: +L(1_4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3497,7 +3497,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_4_19 + je L(1_4_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -3509,15 +3509,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_17: +L(1_4_17): KERNEL4x1_SUB - jl .L1_4_17 + jl L(1_4_17) ALIGN_4 -.L1_4_19: +L(1_4_19): SAVE4x1 @@ -3540,19 +3540,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L1_4_11 + jg L(1_4_11) ALIGN_4 /*******************************************************************************************************/ -.L1_2_10: +L(1_2_10): testq $ 2, M - jz .L1_2_40 + jz L(1_2_40) -.L1_2_11: +L(1_2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3590,7 +3590,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_2_16 + je L(1_2_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -3601,7 +3601,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_2_12: +L(1_2_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3618,7 +3618,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_2_16 + je L(1_2_16) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_SUB @@ -3634,12 +3634,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_2_16 + je L(1_2_16) - jmp .L1_2_12 + jmp L(1_2_12) ALIGN_4 -.L1_2_16: +L(1_2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3647,7 +3647,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_2_19 + je L(1_2_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -3659,15 +3659,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_2_17: +L(1_2_17): KERNEL2x1_SUB - jl .L1_2_17 + jl L(1_2_17) ALIGN_4 -.L1_2_19: +L(1_2_19): SAVE2x1 @@ -3695,13 +3695,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /************************************************************************** * Rest of M ***************************************************************************/ -.L1_2_40: +L(1_2_40): testq $ 1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_2_41: +L(1_2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3739,7 +3739,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_2_46 + je L(1_2_46) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -3750,7 +3750,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_2_42: +L(1_2_42): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3766,7 +3766,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_2_46 + je L(1_2_46) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3782,12 +3782,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_2_46 + je L(1_2_46) - jmp .L1_2_42 + jmp L(1_2_42) ALIGN_4 -.L1_2_46: +L(1_2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -3795,7 +3795,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_2_49 + je L(1_2_49) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -3807,15 +3807,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_2_47: +L(1_2_47): KERNEL1x1_SUB - jl .L1_2_47 + jl L(1_2_47) ALIGN_4 -.L1_2_49: +L(1_2_49): SAVE1x1 @@ -3837,7 +3837,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 2 * SIZE, CO1 # coffset += 2 decq I # i -- - jg .L1_2_41 + jg L(1_2_41) ALIGN_4 @@ -3845,7 +3845,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L999: +L(999): vzeroupper movq SP, %rsp diff --git a/kernel/x86_64/zgemm_kernel_4x2_penryn.S b/kernel/x86_64/zgemm_kernel_4x2_penryn.S index 9aa852ae6e..0dfa421287 100644 --- a/kernel/x86_64/zgemm_kernel_4x2_penryn.S +++ b/kernel/x86_64/zgemm_kernel_4x2_penryn.S @@ -185,10 +185,10 @@ movq N, J sarq $1, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -205,10 +205,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -266,10 +266,10 @@ #endif sarq $3, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH -32 * SIZE(PREA) ADD1 %xmm6, %xmm10 ADD1 %xmm3, %xmm14 @@ -520,10 +520,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): prefetcht0 -16 * SIZE(BB) #ifndef TRMMKERNEL @@ -533,10 +533,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm6, %xmm10 ADD1 %xmm3, %xmm14 movaps %xmm2, %xmm3 @@ -572,10 +572,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): ADD1 %xmm6, %xmm10 ADD1 %xmm3, %xmm14 ADD2 %xmm4, %xmm11 @@ -686,13 +686,13 @@ addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -734,10 +734,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): ADD1 %xmm6, %xmm10 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0xb1, %xmm2, %xmm7 @@ -804,10 +804,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -815,10 +815,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): ADD1 %xmm6, %xmm10 pshufd $0xb1, %xmm2, %xmm7 mulps %xmm0, %xmm2 @@ -839,10 +839,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): ADD1 %xmm6, %xmm10 ADD2 %xmm4, %xmm11 @@ -919,10 +919,10 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L30: +L(30): testq $1, M BRANCH - jle .L39 + jle L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -964,10 +964,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): ADD1 %xmm6, %xmm10 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0xb1, %xmm2, %xmm7 @@ -1034,10 +1034,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1045,10 +1045,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): ADD1 %xmm6, %xmm10 pshufd $0xb1, %xmm2, %xmm7 mulps %xmm0, %xmm2 @@ -1069,10 +1069,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): ADD1 %xmm6, %xmm10 ADD2 %xmm4, %xmm11 @@ -1142,7 +1142,7 @@ #endif ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -1152,13 +1152,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1171,10 +1171,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L50 + jle L(50) ALIGN_4 -.L41: +L(41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1220,10 +1220,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm3, %xmm8 @@ -1299,10 +1299,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1310,10 +1310,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): addps %xmm3, %xmm8 pshufd $0x00, %xmm2, %xmm3 mulps %xmm0, %xmm3 @@ -1336,10 +1336,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): addps %xmm3, %xmm8 addps %xmm4, %xmm12 addps %xmm5, %xmm9 @@ -1421,13 +1421,13 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- BRANCH - jg .L41 + jg L(41) ALIGN_4 -.L50: +L(50): testq $2, M BRANCH - jle .L60 + jle L(60) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1467,10 +1467,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm3, %xmm8 @@ -1512,10 +1512,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1523,10 +1523,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addps %xmm3, %xmm8 pshufd $0x00, %xmm2, %xmm3 mulps %xmm0, %xmm3 @@ -1541,10 +1541,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): addps %xmm3, %xmm8 movddup ALPHA_R, %xmm2 addps %xmm4, %xmm9 @@ -1605,10 +1605,10 @@ addq $4 * SIZE, CO1 ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1648,10 +1648,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm3, %xmm8 @@ -1693,10 +1693,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1704,10 +1704,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): addps %xmm3, %xmm8 pshufd $0x00, %xmm2, %xmm3 mulps %xmm0, %xmm3 @@ -1722,10 +1722,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): addps %xmm3, %xmm8 movddup ALPHA_R, %xmm2 addps %xmm4, %xmm9 @@ -1765,7 +1765,7 @@ movsd %xmm8, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm_kernel_4x2_sse.S b/kernel/x86_64/zgemm_kernel_4x2_sse.S index 5841f8b9e0..abb2b2f4e5 100644 --- a/kernel/x86_64/zgemm_kernel_4x2_sse.S +++ b/kernel/x86_64/zgemm_kernel_4x2_sse.S @@ -437,10 +437,10 @@ salq $ZBASE_SHIFT, LDC movq N, J sarq $1, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -452,12 +452,12 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): PREFETCH (RPREFETCHSIZE + 0) * SIZE(B) movss 0 * SIZE(B), %xmm8 @@ -507,17 +507,17 @@ addq $32 * SIZE, BO addq $ 8 * SIZE, B decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movss 0 * SIZE(B), %xmm8 movss 1 * SIZE(B), %xmm9 movss 2 * SIZE(B), %xmm10 @@ -545,10 +545,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a @@ -557,10 +557,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -620,10 +620,10 @@ leaq (BO, %rax, 8), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -644,7 +644,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -665,7 +665,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -686,7 +686,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -707,7 +707,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -728,7 +728,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -749,7 +749,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -770,7 +770,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -791,17 +791,17 @@ addq $16 * SIZE, %rax BRANCH - jl .L12 + jl L(12) ALIGN_3 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_3 @@ -820,10 +820,10 @@ #else sarq $2, %rax NOBRANCH - jle .L16 + jle L(16) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -837,11 +837,11 @@ subq $-32 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) #endif -.L16: +L(16): #ifndef TRMMKERNEL movq K, %rax #else @@ -851,7 +851,7 @@ movaps ALPHA_I, %xmm7 andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) leaq (, %rax, 8), %rax leaq (AO, %rax, 4), AO @@ -859,7 +859,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulps %xmm0, %xmm1 addps %xmm1, %xmm8 movaps -28 * SIZE(BO, %rax, 8), %xmm1 @@ -886,10 +886,10 @@ movaps -20 * SIZE(AO, %rax, 4), %xmm2 addq $SIZE * 2, %rax - jl .L17 + jl L(17) ALIGN_4 -.L18: +L(18): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -978,12 +978,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1029,10 +1029,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm0, %xmm1 #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1147,10 +1147,10 @@ subq $-128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1160,10 +1160,10 @@ movaps ALPHA_I, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm0, %xmm1 addps %xmm1, %xmm8 movaps -28 * SIZE(BO), %xmm1 @@ -1180,10 +1180,10 @@ subq $- 4 * SIZE, AO subq $-16 * SIZE, BO decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -1246,9 +1246,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1292,10 +1292,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm0, %xmm1 #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1415,10 +1415,10 @@ subq $-128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1428,10 +1428,10 @@ movaps ALPHA_I, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm0, %xmm1 addps %xmm1, %xmm8 movaps -28 * SIZE(BO), %xmm1 @@ -1449,10 +1449,10 @@ subq $ -2 * SIZE, AO subq $-16 * SIZE, BO decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #ifndef TRMMKERNEL #ifdef movsd xorps %xmm0, %xmm0 @@ -1512,22 +1512,22 @@ #endif ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif leaq (C, LDC, 2), C # c += 2 * ldc decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L41: +L(41): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1539,10 +1539,10 @@ movq K, %rax sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): movss 0 * SIZE(B), %xmm8 movss 1 * SIZE(B), %xmm9 movss 2 * SIZE(B), %xmm10 @@ -1595,17 +1595,17 @@ addq $32 * SIZE, BO addq $ 8 * SIZE, B decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): movq K, %rax andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movss 0 * SIZE(B), %xmm8 movss 1 * SIZE(B), %xmm9 @@ -1625,19 +1625,19 @@ addq $2 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1688,10 +1688,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulps %xmm0, %xmm1 #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1815,10 +1815,10 @@ subq $-64 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1828,10 +1828,10 @@ movaps ALPHA_I, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): mulps %xmm0, %xmm1 mulps -28 * SIZE(BO), %xmm0 addps %xmm1, %xmm8 @@ -1848,10 +1848,10 @@ addq $ 8 * SIZE, AO addq $ 8 * SIZE, BO decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -1910,12 +1910,12 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1959,10 +1959,10 @@ #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm0, %xmm1 #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2026,10 +2026,10 @@ subq $-64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -2039,10 +2039,10 @@ movaps ALPHA_I, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm0, %xmm1 mulps -28 * SIZE(BO), %xmm0 addps %xmm1, %xmm8 @@ -2053,10 +2053,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -2106,9 +2106,9 @@ addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L70: +L(70): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2151,10 +2151,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm0, %xmm1 #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2226,10 +2226,10 @@ subq $-64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2239,10 +2239,10 @@ movaps ALPHA_I, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm0, %xmm1 addps %xmm1, %xmm8 movaps -28 * SIZE(BO), %xmm1 @@ -2254,10 +2254,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #ifndef TRMMKERNEL #ifdef movsd xorps %xmm0, %xmm0 @@ -2291,7 +2291,7 @@ movlps %xmm8, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/zgemm_kernel_4x2_sse3.S b/kernel/x86_64/zgemm_kernel_4x2_sse3.S index 4e5504c426..6a1d0cbac3 100644 --- a/kernel/x86_64/zgemm_kernel_4x2_sse3.S +++ b/kernel/x86_64/zgemm_kernel_4x2_sse3.S @@ -409,10 +409,10 @@ salq $ZBASE_SHIFT, LDC movq N, J sarq $1, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -423,10 +423,10 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): movddup 0 * SIZE(B), %xmm0 movddup 2 * SIZE(B), %xmm1 movddup 4 * SIZE(B), %xmm2 @@ -451,17 +451,17 @@ addq $16 * SIZE, B addq $32 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movddup 0 * SIZE(B), %xmm0 movddup 2 * SIZE(B), %xmm1 @@ -471,10 +471,10 @@ addq $4 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a @@ -483,10 +483,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): prefetcht0 0 * SIZE(BB) subq $-8 * SIZE, BB @@ -542,9 +542,9 @@ #if 1 andq $-8, %rax salq $4, %rax - je .L15 + je L(15) -.L1X: +L(1X): KERNEL1 (32 * 0) KERNEL2 (32 * 0) KERNEL3 (32 * 0) @@ -562,7 +562,7 @@ KERNEL15(32 * 0) KERNEL16(32 * 0) cmpq $128 * 1, %rax - jle .L12 + jle L(12) KERNEL1 (32 * 1) KERNEL2 (32 * 1) KERNEL3 (32 * 1) @@ -580,7 +580,7 @@ KERNEL15(32 * 1) KERNEL16(32 * 1) cmpq $128 * 2, %rax - jle .L12 + jle L(12) KERNEL1 (32 * 2) KERNEL2 (32 * 2) KERNEL3 (32 * 2) @@ -598,7 +598,7 @@ KERNEL15(32 * 2) KERNEL16(32 * 2) cmpq $128 * 3, %rax - jle .L12 + jle L(12) KERNEL1 (32 * 3) KERNEL2 (32 * 3) KERNEL3 (32 * 3) @@ -616,7 +616,7 @@ KERNEL15(32 * 3) KERNEL16(32 * 3) cmpq $128 * 4, %rax - jle .L12 + jle L(12) KERNEL1 (32 * 4) KERNEL2 (32 * 4) KERNEL3 (32 * 4) @@ -634,7 +634,7 @@ KERNEL15(32 * 4) KERNEL16(32 * 4) cmpq $128 * 5, %rax - jle .L12 + jle L(12) KERNEL1 (32 * 5) KERNEL2 (32 * 5) KERNEL3 (32 * 5) @@ -652,7 +652,7 @@ KERNEL15(32 * 5) KERNEL16(32 * 5) cmpq $128 * 6, %rax - jle .L12 + jle L(12) KERNEL1 (32 * 6) KERNEL2 (32 * 6) KERNEL3 (32 * 6) @@ -670,7 +670,7 @@ KERNEL15(32 * 6) KERNEL16(32 * 6) cmpq $128 * 7, %rax - jle .L12 + jle L(12) KERNEL1 (32 * 7) KERNEL2 (32 * 7) KERNEL3 (32 * 7) @@ -691,17 +691,17 @@ addq $64 * 8 * SIZE, AO addq $64 * 8 * SIZE, BO subq $128 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 2), BO # * 64 #else sarq $3, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): KERNEL1 (32 * 0) KERNEL2 (32 * 0) KERNEL3 (32 * 0) @@ -722,11 +722,11 @@ addq $64 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L12 + jne L(12) #endif ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else @@ -736,10 +736,10 @@ movaps ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -770,10 +770,10 @@ addq $8 * SIZE, AO addq $8 * SIZE, BO decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \ defined(NR) || defined(NC) || defined(TR) || defined(TC) @@ -881,12 +881,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -929,10 +929,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm9, %xmm0 @@ -1043,10 +1043,10 @@ addq $64 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1056,10 +1056,10 @@ movaps ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -1077,10 +1077,10 @@ addq $ 4 * SIZE, AO addq $ 8 * SIZE, BO decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \ defined(NR) || defined(NC) || defined(TR) || defined(TC) @@ -1152,9 +1152,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1196,10 +1196,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): shufps $0x50, %xmm9, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulps %xmm8, %xmm9 @@ -1278,10 +1278,10 @@ addq $64 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1291,10 +1291,10 @@ movaps ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): shufps $0x50, %xmm9, %xmm9 mulps %xmm8, %xmm9 addps %xmm9, %xmm0 @@ -1308,10 +1308,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): movaps %xmm0, %xmm6 movlhps %xmm1, %xmm0 movhlps %xmm6, %xmm1 @@ -1371,22 +1371,22 @@ #endif ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif leaq (C, LDC, 2), C # c += 2 * ldc decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L41: +L(41): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1397,10 +1397,10 @@ movq K, %rax sarq $3, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): movddup 0 * SIZE(B), %xmm0 movddup 2 * SIZE(B), %xmm1 movddup 4 * SIZE(B), %xmm2 @@ -1425,17 +1425,17 @@ addq $16 * SIZE, B addq $32 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): movq K, %rax andq $7, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movddup 0 * SIZE(B), %xmm0 movaps %xmm0, 0 * SIZE(BO) @@ -1443,19 +1443,19 @@ addq $2 * SIZE, B addq $4 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1499,10 +1499,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm9, %xmm0 @@ -1621,10 +1621,10 @@ addq $64 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1634,10 +1634,10 @@ movaps ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -1656,10 +1656,10 @@ addq $ 8 * SIZE, AO addq $ 4 * SIZE, BO decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \ defined(NR) || defined(NC) || defined(TR) || defined(TC) @@ -1726,12 +1726,12 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1769,10 +1769,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm9, %xmm0 @@ -1835,10 +1835,10 @@ addq $32 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1848,10 +1848,10 @@ movaps ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -1863,10 +1863,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \ defined(NR) || defined(NC) || defined(TR) || defined(TC) shufps $0xb1, %xmm1, %xmm1 @@ -1909,9 +1909,9 @@ addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L70: +L(70): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1949,10 +1949,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): shufps $0x50, %xmm9, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulps %xmm8, %xmm9 @@ -1998,10 +1998,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2011,10 +2011,10 @@ movaps ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): shufps $0x50, %xmm9, %xmm9 mulps %xmm8, %xmm9 movddup 2 * SIZE(AO), %xmm8 @@ -2024,10 +2024,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm1, %xmm0 movhlps %xmm0, %xmm1 @@ -2070,7 +2070,7 @@ movsd %xmm0, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/zgemm_kernel_4x4_sandy.S b/kernel/x86_64/zgemm_kernel_4x4_sandy.S index e1176d131c..feac59114b 100644 --- a/kernel/x86_64/zgemm_kernel_4x4_sandy.S +++ b/kernel/x86_64/zgemm_kernel_4x4_sandy.S @@ -276,9 +276,9 @@ movq %r11, kk; MOVQ bn,j; SARQ $2,j; # Rn = 4 -JLE .L0_loopE; +JLE L(0_loopE); ALIGN_5; -.L0_bodyB:; +L(0_bodyB):; #if defined(TRMMKERNEL) && defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk; @@ -291,9 +291,9 @@ LEAQ (bb, k, 1), prebb; # Rn=4 SIZE=8 COMPLEX=2 MOVQ ba,ptrba; MOVQ bm,i; SARQ $2,i; # Rm = 4 -JLE .L1_loopE; +JLE L(1_loopE); ALIGN_5; -.L1_bodyB:; +L(1_bodyB):; #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -340,9 +340,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2,k; # Unroll 4 times -JLE .L2_loopE; +JLE L(2_loopE); ALIGN_5; -.L2_bodyB:; +L(2_bodyB):; #### Computing kernel #### #### Unroll time 1 #### @@ -563,17 +563,17 @@ MUL_DY yvec1, yvec5, yvec7; ADD2_DY yvec6, yvec12, yvec12; ADD2_DY yvec7, yvec8, yvec8; DECQ k; -JG .L2_bodyB; +JG L(2_bodyB); ALIGN_5 -.L2_loopE:; +L(2_loopE):; #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L3_loopE; +JLE L(3_loopE); ALIGN_5 -.L3_bodyB: +L(3_bodyB): #### Unroll time 1 #### LD_DY 4*SIZE(ptrba), yvec1; MUL_DY yvec0, yvec2, yvec6; @@ -683,15 +683,15 @@ MUL_DY yvec1, yvec4, yvec6; MUL_DY yvec1, yvec5, yvec7; ADD2_DY yvec6, yvec12, yvec12; ADD2_DY yvec7, yvec8, yvec8; -.L3_loopE:; +L(3_loopE):; #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L4_loopE; +JLE L(4_loopE); ALIGN_5 -.L4_loopB:; +L(4_loopB):; #### Unroll time 1 #### PREFETCH0 PRESIZE*SIZE(ptrba); LD_DY 4*SIZE(ptrba), yvec1; @@ -744,7 +744,7 @@ MUL_DY yvec1, yvec4, yvec6; MUL_DY yvec1, yvec5, yvec7; ADD2_DY yvec6, yvec12, yvec12; ADD2_DY yvec7, yvec8, yvec8; -.L4_loopE:; +L(4_loopE):; #### Handle #### XOR_DY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -831,7 +831,7 @@ ADDSUB_DY yvec4, yvec8, yvec8; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L4_loopEx; +JNE L(4_loopEx); ALIGN_5 #### Store Back #### EXTRA_DY $1,yvec15,xvec7; @@ -888,12 +888,12 @@ ADDQ $4, kk; #endif ADDQ $8*SIZE,C0; ADDQ $8*SIZE,C1; -.L1_bodyE:; +L(1_bodyE):; DECQ i; -JG .L1_bodyB; -JMP .L1_loopE; +JG L(1_bodyB); +JMP L(1_loopE); ALIGN_5 -.L4_loopEx: +L(4_loopEx): EXTRA_DY $1, yvec15, xvec7; EXTRA_DY $1, yvec14, xvec6; #ifndef TRMMKERNEL @@ -1003,13 +1003,13 @@ ADDQ $4, kk; ADDQ $8*SIZE, C0; ADDQ $8*SIZE, C1; DECQ i; -JG .L1_bodyB; +JG L(1_bodyB); ALIGN_5; -.L1_loopE:; +L(1_loopE):; TEST $2, bm; -JLE .L5_loopE; +JLE L(5_loopE); ALIGN_5 -.L5_bodyB: +L(5_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -1039,9 +1039,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L7_loopE; +JLE L(7_loopE); ALIGN_5 -.L7_bodyB: +L(7_bodyB): #### Compute kernel #### #### Unroll times 1 #### LD_DY 0*SIZE(ptrba), yvec0; @@ -1173,17 +1173,17 @@ MUL_DY yvec0, yvec5, yvec7; ADD2_DY yvec7, yvec12, yvec12; ADDQ $32*SIZE, ptrbb; DECQ k; -JG .L7_bodyB; +JG L(7_bodyB); ALIGN_5 -.L7_loopE: +L(7_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L8_loopE; +JLE L(8_loopE); ALIGN_5 -.L8_bodyB: +L(8_bodyB): #### Unroll times 1 #### LD_DY 0*SIZE(ptrba), yvec0; EDUP_DY 0*SIZE(ptrbb), yvec2; @@ -1249,15 +1249,15 @@ ADDQ $8*SIZE, ptrba; MUL_DY yvec0, yvec5, yvec7; ADD2_DY yvec7, yvec12, yvec12; ADDQ $16*SIZE, ptrbb; -.L8_loopE: +L(8_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L9_loopE; +JLE L(9_loopE); ALIGN_5 -.L9_bodyB: +L(9_bodyB): #### Unroll times 1 #### LD_DY 0*SIZE(ptrba), yvec0; EDUP_DY 0*SIZE(ptrbb), yvec2; @@ -1292,7 +1292,7 @@ ADD2_DY yvec7, yvec12, yvec12; ADDQ $4*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L9_loopE: +L(9_loopE): #### Handle #### XOR_DY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -1343,7 +1343,7 @@ ADD2_DY yvec2, yvec12, yvec12; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L9_loopEx; +JNE L(9_loopEx); ALIGN_5 #### Writing back #### EXTRA_DY $1, yvec15, xvec7; @@ -1380,9 +1380,9 @@ ADDQ $2, kk; #endif ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -JMP .L5_loopE; +JMP L(5_loopE); ALIGN_5 -.L9_loopEx: +L(9_loopEx): EXTRA_DY $1, yvec15, xvec7; EXTRA_DY $1, yvec14, xvec6; EXTRA_DY $1, yvec13, xvec5; @@ -1443,11 +1443,11 @@ ADDQ $2, kk; #endif ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -.L5_loopE: +L(5_loopE): TEST $1, bm; -JLE .L6_loopE; +JLE L(6_loopE); ALIGN_5 -.L6_bodyB: +L(6_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -1475,9 +1475,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L10_loopE; +JLE L(10_loopE); ALIGN_5 -.L10_bodyB: +L(10_bodyB): LD_DY 0*SIZE(ptrba), yvec0; #### A1r A1i A2r A2i EDUP_DY 0*SIZE(ptrbb), yvec2; EDUP_DY 4*SIZE(ptrbb), yvec3; @@ -1549,17 +1549,17 @@ ADD2_DY yvec7, yvec14, yvec14 ADDQ $8*SIZE, ptrba; ADDQ $32*SIZE, ptrbb; DECQ k; -JG .L10_bodyB; +JG L(10_bodyB); ALIGN_5 -.L10_loopE: +L(10_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L11_loopE; +JLE L(11_loopE); ALIGN_5 -.L11_bodyB: +L(11_bodyB): LD_DY 0*SIZE(ptrba), yvec0; #### A1r A1i A2r A2i EDUP_DY 0*SIZE(ptrbb), yvec2; EDUP_DY 4*SIZE(ptrbb), yvec3; @@ -1597,15 +1597,15 @@ ADD2_DY yvec7, yvec14, yvec14; ADDQ $4*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; -.L11_loopE: +L(11_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L12_loopE; +JLE L(12_loopE); ALIGN_5 -.L12_bodyB: +L(12_bodyB): LD_DY 0*SIZE(ptrba), yvec0; #### A1r A1i A2r A2i EDUP_DY 0*SIZE(ptrbb), yvec2; EDUP_DY 4*SIZE(ptrbb), yvec3; @@ -1626,7 +1626,7 @@ ADD2_DY yvec7, yvec14, yvec14; ADDQ $2*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L12_loopE: +L(12_loopE): #### Handle #### XOR_DY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -1691,7 +1691,7 @@ ADDQ $1, kk; #endif ADDQ $2*SIZE, C0; ADDQ $2*SIZE, C1; -.L6_loopE: +L(6_loopE): #if defined(TRMMKERNEL) && !defined(LEFT) ADDQ $4, kk; #endif @@ -1699,15 +1699,15 @@ MOVQ bk,k; SALQ $6,k; ADDQ k,bb; LEAQ (C,ldc,4),C; -.L0_bodyE:; +L(0_bodyE):; DECQ j; -JG .L0_bodyB; +JG L(0_bodyB); ALIGN_5; -.L0_loopE:; +L(0_loopE):; TEST $2, bn; -JLE .L20_loopE; +JLE L(20_loopE); ALIGN_5 -.L20_bodyB: +L(20_bodyB): #if defined(TRMMKERNEL) && defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk; @@ -1717,9 +1717,9 @@ LEAQ (C, ldc, 1), C1; MOVQ ba, ptrba; MOVQ bm, i; SARQ $2, i; -JLE .L21_loopE; +JLE L(21_loopE); ALIGN_5 -.L21_bodyB: +L(21_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -1749,9 +1749,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L211_loopE; +JLE L(211_loopE); ALIGN_5 -.L211_bodyB: +L(211_bodyB): #### Unroll time 1 #### EDUP_DY 0*SIZE(ptrbb), yvec2; LD_DY 0*SIZE(ptrba), yvec0; @@ -1870,17 +1870,17 @@ MUL_DY yvec1, yvec5, yvec7; ADD2_DY yvec7, yvec12, yvec12; ADDQ $32*SIZE, ptrba; DECQ k; -JG .L211_bodyB; +JG L(211_bodyB); ALIGN_5 -.L211_loopE: +L(211_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L212_loopE; +JLE L(212_loopE); ALIGN_5 -.L212_bodyB: +L(212_bodyB): #### Unroll time 1 #### EDUP_DY 0*SIZE(ptrbb), yvec2; LD_DY 0*SIZE(ptrba), yvec0; @@ -1942,15 +1942,15 @@ ADD2_DY yvec7, yvec12, yvec12; ADDQ $8*SIZE, ptrbb; ADDQ $16*SIZE, ptrba; -.L212_loopE: +L(212_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L213_loopE; +JLE L(213_loopE); ALIGN_5 -.L213_bodyB: +L(213_bodyB): #### Unroll time 1 #### EDUP_DY 0*SIZE(ptrbb), yvec2; LD_DY 0*SIZE(ptrba), yvec0; @@ -1982,7 +1982,7 @@ ADD2_DY yvec7, yvec12, yvec12; ADDQ $4*SIZE, ptrbb; ADDQ $8*SIZE, ptrba; -.L213_loopE: +L(213_loopE): #### Handle #### XOR_DY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -2037,7 +2037,7 @@ EXTRA_DY $1, yvec12, xvec4; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L213_loopEx; +JNE L(213_loopEx); ALIGN_5 #### Writing back #### #ifndef TRMMKERNEL @@ -2071,10 +2071,10 @@ ADDQ $4, kk; ADDQ $8*SIZE, C0; ADDQ $8*SIZE, C1; DECQ i; -JG .L21_bodyB; -JMP .L21_loopE; +JG L(21_bodyB); +JMP L(21_loopE); ALIGN_5 -.L213_loopEx: +L(213_loopEx): #ifndef TRMMKERNEL LDL_DX 0*SIZE(C0), xvec0, xvec0; LDH_DX 1*SIZE(C0), xvec0, xvec0; @@ -2132,13 +2132,13 @@ ADDQ $4, kk; ADDQ $8*SIZE, C0; ADDQ $8*SIZE, C1; DECQ i; -JG .L21_bodyB; +JG L(21_bodyB); ALIGN_5 -.L21_loopE: +L(21_loopE): TEST $2, bm; -JLE .L22_loopE; +JLE L(22_loopE); ALIGN_5 -.L22_bodyB: +L(22_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -2166,9 +2166,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L221_loopE; +JLE L(221_loopE); ALIGN_5 -.L221_bodyB: +L(221_bodyB): #### Unroll time 1 #### EDUP_DY 0*SIZE(ptrbb), yvec2; LD_DY 0*SIZE(ptrba), yvec0; @@ -2247,17 +2247,17 @@ MUL_DY yvec0, yvec5, yvec6; ADD2_DY yvec6, yvec13, yvec13; ADDQ $16*SIZE, ptrba; DECQ k; -JG .L221_bodyB; +JG L(221_bodyB); ALIGN_5 -.L221_loopE: +L(221_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L222_loopE; +JLE L(222_loopE); ALIGN_5 -.L222_bodyB: +L(222_bodyB): #### Unroll time 1 #### EDUP_DY 0*SIZE(ptrbb), yvec2; LD_DY 0*SIZE(ptrba), yvec0; @@ -2298,15 +2298,15 @@ ADD2_DY yvec6, yvec13, yvec13; ADDQ $8*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L222_loopE: +L(222_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L223_loopE; +JLE L(223_loopE); ALIGN_5 -.L223_bodyB: +L(223_bodyB): #### Unroll time 1 #### EDUP_DY 0*SIZE(ptrbb), yvec2; LD_DY 0*SIZE(ptrba), yvec0; @@ -2328,7 +2328,7 @@ ADD2_DY yvec6, yvec13, yvec13; ADDQ $4*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L223_loopE: +L(223_loopE): #### Handle #### XOR_DY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -2396,11 +2396,11 @@ ADDQ $2, kk; ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -.L22_loopE: +L(22_loopE): TEST $1, bm; -JLE .L23_loopE; +JLE L(23_loopE); ALIGN_5 -.L23_bodyB: +L(23_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -2427,9 +2427,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L231_loopE; +JLE L(231_loopE); ALIGN_5 -.L231_bodyB: +L(231_bodyB): LD_DY 0*SIZE(ptrba), yvec0; #### A1r A1i A2r A2i EDUP_DY 0*SIZE(ptrbb), yvec2; @@ -2477,17 +2477,17 @@ ADD2_DY yvec6, yvec15, yvec15; ADDQ $8*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; DECQ k; -JG .L231_bodyB; +JG L(231_bodyB); ALIGN_5 -.L231_loopE: +L(231_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L232_loopE; +JLE L(232_loopE); ALIGN_5 -.L232_bodyB: +L(232_bodyB): LD_DY 0*SIZE(ptrba), yvec0; #### A1r A1i A2r A2i EDUP_DY 0*SIZE(ptrbb), yvec2; @@ -2513,15 +2513,15 @@ ADD2_DY yvec6, yvec15, yvec15; ADDQ $4*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L232_loopE: +L(232_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L233_loopE; +JLE L(233_loopE); ALIGN_5 -.L233_bodyB: +L(233_bodyB): LD_DY 0*SIZE(ptrba), yvec0; #### A1r A1i A2r A2i EDUP_DY 0*SIZE(ptrbb), yvec2; @@ -2536,7 +2536,7 @@ ADD2_DY yvec6, yvec15, yvec15; ADDQ $2*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L233_loopE: +L(233_loopE): #### Handle #### XOR_DY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -2583,7 +2583,7 @@ ADDQ $1, kk; #endif ADDQ $2*SIZE, C0; ADDQ $2*SIZE, C0; -.L23_loopE: +L(23_loopE): #if defined(TRMMKERNEL) && !defined(LEFT) ADDQ $2, kk; #endif @@ -2591,11 +2591,11 @@ MOVQ bk, k; SALQ $5, k; ADDQ k, bb; LEAQ (C, ldc, 2), C; -.L20_loopE: +L(20_loopE): TEST $1, bn; -JLE .L30_loopE; +JLE L(30_loopE); ALIGN_5 -.L30_bodyB: +L(30_bodyB): #if defined(TRMMKERNEL) && defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk; @@ -2604,9 +2604,9 @@ MOVQ ba, ptrba; MOVQ C, C0; MOVQ bm, i; SARQ $2, i; -JLE .L31_loopE; +JLE L(31_loopE); ALIGN_5 -.L31_bodyB: +L(31_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -2634,9 +2634,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L311_loopE; +JLE L(311_loopE); ALIGN_5 -.L311_bodyB: +L(311_bodyB): LD_DY 0*SIZE(ptrba), yvec0; BROAD_DY 0*SIZE(ptrbb), yvec2; MUL_DY yvec0, yvec2, yvec6; @@ -2711,17 +2711,17 @@ ADD2_DY yvec7, yvec14, yvec14; ADDQ $32*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L311_bodyB; +JG L(311_bodyB); ALIGN_5 -.L311_loopE: +L(311_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L312_loopE; +JLE L(312_loopE); ALIGN_5 -.L312_bodyB: +L(312_bodyB): LD_DY 0*SIZE(ptrba), yvec0; BROAD_DY 0*SIZE(ptrbb), yvec2; MUL_DY yvec0, yvec2, yvec6; @@ -2760,15 +2760,15 @@ ADD2_DY yvec7, yvec14, yvec14; ADDQ $16*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L312_loopE: +L(312_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L313_loopE; +JLE L(313_loopE); ALIGN_5 -.L313_bodyB: +L(313_bodyB): LD_DY 0*SIZE(ptrba), yvec0; BROAD_DY 0*SIZE(ptrbb), yvec2; MUL_DY yvec0, yvec2, yvec6; @@ -2789,7 +2789,7 @@ ADD2_DY yvec7, yvec14, yvec14; ADDQ $8*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L313_loopE: +L(313_loopE): #### Handle #### XOR_DY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -2856,13 +2856,13 @@ ADDQ $4, kk; #endif ADDQ $8*SIZE, C0; DECQ i; -JG .L31_bodyB; +JG L(31_bodyB); ALIGN_5 -.L31_loopE: +L(31_loopE): TEST $2, bm; -JLE .L32_loopE; +JLE L(32_loopE); ALIGN_5 -.L32_bodyB: +L(32_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -2889,9 +2889,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L321_loopE; +JLE L(321_loopE); ALIGN_5 -.L321_bodyB: +L(321_bodyB): LD_DY 0*SIZE(ptrba), yvec0; BROAD_DY 0*SIZE(ptrbb), yvec2; MUL_DY yvec0, yvec2, yvec6; @@ -2930,17 +2930,17 @@ ADD2_DY yvec7, yvec15, yvec15; ADDQ $16*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L321_bodyB; +JG L(321_bodyB); ALIGN_5 -.L321_loopE: +L(321_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L322_loopE; +JLE L(322_loopE); ALIGN_5 -.L322_bodyB: +L(322_bodyB): LD_DY 0*SIZE(ptrba), yvec0; BROAD_DY 0*SIZE(ptrbb), yvec2; MUL_DY yvec0, yvec2, yvec6; @@ -2961,15 +2961,15 @@ ADD2_DY yvec7, yvec15, yvec15; ADDQ $8*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L322_loopE: +L(322_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L323_loopE; +JLE L(323_loopE); ALIGN_5 -.L323_bodyB: +L(323_bodyB): LD_DY 0*SIZE(ptrba), yvec0; BROAD_DY 0*SIZE(ptrbb), yvec2; MUL_DY yvec0, yvec2, yvec6; @@ -2980,7 +2980,7 @@ MUL_DY yvec1, yvec3, yvec7; ADD2_DY yvec7, yvec15, yvec15; ADDQ $4*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L323_loopE: +L(323_loopE): #### Handle #### XOR_DY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -3026,11 +3026,11 @@ ADDQ %rax, ptrbb; ADDQ $2, kk; #endif ADDQ $4*SIZE, C0; -.L32_loopE: +L(32_loopE): TEST $1, bm; -JLE .L33_loopE; +JLE L(33_loopE); ALIGN_5 -.L33_bodyB: +L(33_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -3057,9 +3057,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L331_loopE; +JLE L(331_loopE); ALIGN_5 -.L331_bodyB: +L(331_bodyB): LD_DX 0*SIZE(ptrba), xvec0; BROAD_DX 0*SIZE(ptrbb), xvec2; MUL_DX xvec0, xvec2, xvec2; @@ -3102,17 +3102,17 @@ ADDSUB_DX xvec3, xvec15, xvec15; ADDQ $8*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L331_bodyB; +JG L(331_bodyB); ALIGN_5 -.L331_loopE: +L(331_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L332_loopE; +JLE L(332_loopE); ALIGN_5 -.L332_bodyB: +L(332_bodyB): LD_DX 0*SIZE(ptrba), xvec0; BROAD_DX 0*SIZE(ptrbb), xvec2; MUL_DX xvec0, xvec2, xvec2; @@ -3135,15 +3135,15 @@ ADDSUB_DX xvec3, xvec15, xvec15; ADDQ $4*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L332_loopE: +L(332_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L333_loopE; +JLE L(333_loopE); ALIGN_5 -.L333_bodyB: +L(333_bodyB): LD_DX 0*SIZE(ptrba), xvec0; BROAD_DX 0*SIZE(ptrbb), xvec2; MUL_DX xvec0, xvec2, xvec2; @@ -3156,7 +3156,7 @@ ADDSUB_DX xvec3, xvec15, xvec15; ADDQ $2*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L333_loopE: +L(333_loopE): #### Handle #### XOR_DY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -3199,7 +3199,7 @@ ADDQ %rax, ptrbb; ADDQ $1, kk; #endif ADDQ $2*SIZE, C0; -.L33_loopE: +L(33_loopE): #if defined(TRMMKERNEL) && !defined(LEFT) ADDQ $1, kk; #endif @@ -3207,7 +3207,7 @@ MOVQ bk, k; SALQ $4*SIZE, k; ADDQ k, bb; LEAQ (C, ldc, 1), C; -.L30_loopE: +L(30_loopE): movq 0(%rsp), %rbx; movq 8(%rsp), %rbp; movq 16(%rsp), %r12; diff --git a/kernel/x86_64/zgemm_ncopy_1.S b/kernel/x86_64/zgemm_ncopy_1.S index 60b51f53e4..b2dbf5494a 100644 --- a/kernel/x86_64/zgemm_ncopy_1.S +++ b/kernel/x86_64/zgemm_ncopy_1.S @@ -102,19 +102,19 @@ testq N, N movq N, J - jle .L999 + jle L(999) ALIGN_4 -.L12: +L(12): movq A, AO1 addq LDA, A movq M, I sarq $2, I - jle .L14 + jle L(14) ALIGN_4 -.L13: +L(13): #ifndef DOUBLE movsd 0 * SIZE(AO1), %xmm0 movhps 2 * SIZE(AO1), %xmm0 @@ -146,16 +146,16 @@ addq $8 * SIZE, AO1 addq $8 * SIZE, B decq I - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): movq M, I andq $3, I - jle .L16 + jle L(16) ALIGN_4 -.L15: +L(15): #ifndef DOUBLE movsd 0 * SIZE(AO1), %xmm0 movsd %xmm0, 0 * SIZE(B) @@ -169,15 +169,15 @@ addq $2 * SIZE, AO1 addq $2 * SIZE, B decq I - jg .L15 + jg L(15) ALIGN_4 -.L16: +L(16): decq J - jg .L12 + jg L(12) ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 movups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/zgemm_ncopy_2.S b/kernel/x86_64/zgemm_ncopy_2.S index 8451982260..1c97f1eb60 100644 --- a/kernel/x86_64/zgemm_ncopy_2.S +++ b/kernel/x86_64/zgemm_ncopy_2.S @@ -120,20 +120,20 @@ movq N, J sarq $1, J - jle .L20 + jle L(20) ALIGN_4 -.L12: +L(12): movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), A movq M, I sarq $2, I - jle .L14 + jle L(14) ALIGN_4 -.L13: +L(13): #ifdef HAVE_3DNOW prefetchw (WPREFETCHSIZE + 0) * SIZE(B) prefetchw (WPREFETCHSIZE + 8) * SIZE(B) @@ -217,16 +217,16 @@ addq $8 * SIZE, AO2 subq $-16 * SIZE, B decq I - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): movq M, I andq $3, I - jle .L16 + jle L(16) ALIGN_4 -.L15: +L(15): #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 movhps 0 * SIZE(AO2), %xmm0 @@ -246,26 +246,26 @@ addq $2 * SIZE, AO2 addq $4 * SIZE, B decq I - jg .L15 + jg L(15) ALIGN_4 -.L16: +L(16): decq J - jg .L12 + jg L(12) ALIGN_4 -.L20: +L(20): testq $1, N - jle .L999 + jle L(999) movq A, AO1 movq M, I sarq $2, I - jle .L24 + jle L(24) ALIGN_4 -.L23: +L(23): #ifdef HAVE_3DNOW prefetchw (WPREFETCHSIZE + 0) * SIZE(B) prefetchw (WPREFETCHSIZE + 8) * SIZE(B) @@ -305,16 +305,16 @@ addq $8 * SIZE, AO1 addq $8 * SIZE, B decq I - jg .L23 + jg L(23) ALIGN_4 -.L24: +L(24): movq M, I andq $3, I - jle .L999 + jle L(999) ALIGN_4 -.L25: +L(25): #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 @@ -329,11 +329,11 @@ addq $2 * SIZE, AO1 addq $2 * SIZE, B decq I - jg .L25 + jg L(25) ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 movups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/zgemm_tcopy_1.S b/kernel/x86_64/zgemm_tcopy_1.S index 02c0614930..8d8d1afab4 100644 --- a/kernel/x86_64/zgemm_tcopy_1.S +++ b/kernel/x86_64/zgemm_tcopy_1.S @@ -102,19 +102,19 @@ testq N, N movq N, J - jle .L999 + jle L(999) ALIGN_4 -.L12: +L(12): movq A, AO1 addq $2 * SIZE, A movq M, I sarq $1, I - jle .L14 + jle L(14) ALIGN_4 -.L13: +L(13): #ifndef DOUBLE movsd 0 * SIZE(AO1), %xmm0 movhps 0 * SIZE(AO1, LDA, 1), %xmm0 @@ -140,12 +140,12 @@ leaq (AO1, LDA, 2), AO1 addq $4 * SIZE, B decq I - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): testq $1, M - jle .L16 + jle L(16) #ifndef DOUBLE movsd 0 * SIZE(AO1), %xmm0 @@ -159,12 +159,12 @@ addq $2 * SIZE, B ALIGN_4 -.L16: +L(16): decq J - jg .L12 + jg L(12) ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 movups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/zgemm_tcopy_2.S b/kernel/x86_64/zgemm_tcopy_2.S index 121bbc4283..a2f232425f 100644 --- a/kernel/x86_64/zgemm_tcopy_2.S +++ b/kernel/x86_64/zgemm_tcopy_2.S @@ -125,10 +125,10 @@ movq M, J sarq $1, J - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): movq A, AO1 leaq (A, LDA ), AO2 leaq (A, LDA, 2), A @@ -138,10 +138,10 @@ movq N, I sarq $2, I - jle .L13 + jle L(13) ALIGN_4 -.L12: +L(12): #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 movhps 2 * SIZE(AO1), %xmm0 @@ -228,12 +228,12 @@ addq $8 * SIZE, AO2 leaq (BO, M8, 4), BO decq I - jg .L12 + jg L(12) ALIGN_4 -.L13: +L(13): testq $2, N - jle .L14 + jle L(14) #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 @@ -266,9 +266,9 @@ leaq (BO, M8, 4), BO ALIGN_4 -.L14: +L(14): testq $1, N - jle .L19 + jle L(19) #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 @@ -288,27 +288,27 @@ addq $4 * SIZE, BO1 ALIGN_4 -.L19: +L(19): decq J - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M - jle .L999 + jle L(999) ALIGN_4 -.L21: +L(21): movq A, AO1 movq B, BO movq N, I sarq $2, I - jle .L23 + jle L(23) ALIGN_4 -.L22: +L(22): #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 movhps 2 * SIZE(AO1), %xmm0 @@ -360,12 +360,12 @@ addq $8 * SIZE, AO1 leaq (BO, M8, 4), BO decq I - jg .L22 + jg L(22) ALIGN_4 -.L23: +L(23): testq $2, N - jle .L24 + jle L(24) #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 @@ -386,9 +386,9 @@ leaq (BO, M8, 4), BO ALIGN_4 -.L24: +L(24): testq $1, N - jle .L999 + jle L(999) #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 @@ -403,7 +403,7 @@ ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 movups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/zgemv_n.S b/kernel/x86_64/zgemv_n.S index b903cfb26b..c786cb34a9 100644 --- a/kernel/x86_64/zgemv_n.S +++ b/kernel/x86_64/zgemv_n.S @@ -161,7 +161,7 @@ movsd %xmm0,ALPHAR movsd %xmm1,ALPHAI -.L0t: +L(0t): xorq I,I addq $1,I salq $18,I @@ -169,13 +169,13 @@ movq I,M movsd ALPHAR,%xmm0 movsd ALPHAI,%xmm1 - jge .L00t + jge L(00t) movq MMM,M addq I,M - jle .L999x + jle L(999x) -.L00t: +L(00t): movq AA, A movq NN, N movq LDAX, LDA @@ -196,9 +196,9 @@ subq $-16 * SIZE, A testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) ALIGN_3 movq BUFFER, Y1 @@ -210,7 +210,7 @@ sarq $3, %rax ALIGN_3 -.L01: +L(01): movapd %xmm4, 0 * SIZE(Y1) movapd %xmm4, 2 * SIZE(Y1) movapd %xmm4, 4 * SIZE(Y1) @@ -222,22 +222,22 @@ subq $-16 * SIZE, Y1 decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L10: +L(10): #ifdef ALIGNED_ACCESS testq $SIZE, A - jne .L100 + jne L(100) #endif #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $4, N leaq 16 * SIZE(BUFFER), Y1 @@ -336,16 +336,16 @@ movq M, I sarq $2, I - jle .L15 + jle L(15) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm6) decq I - jle .L14 + jle L(14) ALIGN_3 -.L13: +L(13): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -494,10 +494,10 @@ subq $1, I BRANCH - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): pshufd $0x4e, %xmm4, %xmm5 mulpd %xmm8, %xmm4 addpd %xmm4, %xmm0 @@ -623,9 +623,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L15: +L(15): testq $2, M - je .L17 + je L(17) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm6) @@ -693,9 +693,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L17: +L(17): testq $1, M - je .L19 + je L(19) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm6) @@ -729,23 +729,23 @@ MOVUPS_YS1(-16 * SIZE, Y1, %xmm0) ALIGN_3 -.L19: +L(19): cmpq $4, N - jge .L11 + jge L(11) ALIGN_3 -.L20: +L(20): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 2 ALIGN_3 -.L21: +L(21): #endif subq $2, N @@ -817,7 +817,7 @@ movq M, I sarq $2, I - jle .L25 + jle L(25) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm6) @@ -825,10 +825,10 @@ MOVUPS_A1(-10 * SIZE, A1, %xmm10) decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A2) #endif @@ -913,10 +913,10 @@ subq $1, I BRANCH - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): pshufd $0x4e, %xmm4, %xmm5 mulpd %xmm12, %xmm4 addpd %xmm4, %xmm0 @@ -984,9 +984,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L25: +L(25): testq $2, M - je .L27 + je L(27) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm6) @@ -1026,12 +1026,12 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L27: +L(27): testq $1, M #if GEMV_UNROLL == 2 - je .L29 + je L(29) #else - je .L30 + je L(30) #endif MOVUPS_A1(-16 * SIZE, A1, %xmm4) @@ -1054,20 +1054,20 @@ #if GEMV_UNROLL == 2 ALIGN_3 -.L29: +L(29): cmpq $2, N - jge .L21 + jge L(21) #endif ALIGN_3 -.L30: +L(30): #endif cmpq $1, N - jl .L980 + jl L(980) #if GEMV_UNROLL == 1 -.L31: +L(31): decq N #endif @@ -1124,7 +1124,7 @@ movq M, I sarq $2, I - jle .L35 + jle L(35) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm6) @@ -1132,10 +1132,10 @@ MOVUPS_A1(-10 * SIZE, A1, %xmm10) decq I - jle .L34 + jle L(34) ALIGN_3 -.L33: +L(33): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -1187,10 +1187,10 @@ subq $1, I BRANCH - jg .L33 + jg L(33) ALIGN_3 -.L34: +L(34): pshufd $0x4e, %xmm4, %xmm5 mulpd %xmm12, %xmm4 addpd %xmm4, %xmm0 @@ -1229,9 +1229,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L35: +L(35): testq $2, M - je .L37 + je L(37) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm6) @@ -1256,12 +1256,12 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L37: +L(37): testq $1, M #if GEMV_UNROLL == 1 - je .L39 + je L(39) #else - je .L980 + je L(980) #endif MOVUPS_A1(-16 * SIZE, A1, %xmm4) @@ -1276,24 +1276,24 @@ #if GEMV_UNROLL == 1 ALIGN_3 -.L39: +L(39): cmpq $1, N - jge .L31 + jge L(31) #endif #ifdef ALIGNED_ACCESS - jmp .L980 + jmp L(980) ALIGN_3 -.L100: +L(100): #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L110 + jl L(110) ALIGN_3 -.L101: +L(101): subq $4, N leaq 16 * SIZE(BUFFER), Y1 @@ -1392,7 +1392,7 @@ movq M, I sarq $2, I - jle .L105 + jle L(105) movsd -16 * SIZE(A1), %xmm4 movhpd -15 * SIZE(A1), %xmm4 @@ -1400,10 +1400,10 @@ movhpd -13 * SIZE(A1), %xmm6 decq I - jle .L104 + jle L(104) ALIGN_3 -.L103: +L(103): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -1568,10 +1568,10 @@ subq $1, I BRANCH - jg .L103 + jg L(103) ALIGN_3 -.L104: +L(104): pshufd $0x4e, %xmm4, %xmm5 mulpd %xmm8, %xmm4 addpd %xmm4, %xmm0 @@ -1711,9 +1711,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L105: +L(105): testq $2, M - je .L107 + je L(107) movsd -16 * SIZE(A1), %xmm4 movhpd -15 * SIZE(A1), %xmm4 @@ -1789,9 +1789,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L107: +L(107): testq $1, M - je .L109 + je L(109) movsd -16 * SIZE(A1), %xmm4 movhpd -15 * SIZE(A1), %xmm4 @@ -1829,23 +1829,23 @@ MOVUPS_YS1(-16 * SIZE, Y1, %xmm0) ALIGN_3 -.L109: +L(109): cmpq $4, N - jge .L101 + jge L(101) ALIGN_3 -.L110: +L(110): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L120 + jl L(120) #if GEMV_UNROLL == 2 ALIGN_3 -.L111: +L(111): #endif subq $2, N @@ -1917,7 +1917,7 @@ movq M, I sarq $2, I - jle .L115 + jle L(115) movsd -16 * SIZE(A1), %xmm4 movhpd -15 * SIZE(A1), %xmm4 @@ -1929,10 +1929,10 @@ movhpd -9 * SIZE(A1), %xmm10 decq I - jle .L114 + jle L(114) ALIGN_3 -.L113: +L(113): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A2) #endif @@ -2025,10 +2025,10 @@ subq $1, I BRANCH - jg .L113 + jg L(113) ALIGN_3 -.L114: +L(114): pshufd $0x4e, %xmm4, %xmm5 mulpd %xmm12, %xmm4 addpd %xmm4, %xmm0 @@ -2100,9 +2100,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L115: +L(115): testq $2, M - je .L117 + je L(117) movsd -16 * SIZE(A1), %xmm4 movhpd -15 * SIZE(A1), %xmm4 @@ -2147,12 +2147,12 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L117: +L(117): testq $1, M #if GEMV_UNROLL == 2 - je .L119 + je L(119) #else - je .L120 + je L(120) #endif movsd -16 * SIZE(A1), %xmm4 @@ -2177,20 +2177,20 @@ #if GEMV_UNROLL == 2 ALIGN_3 -.L119: +L(119): cmpq $2, N - jge .L111 + jge L(111) #endif ALIGN_3 -.L120: +L(120): #endif cmpq $1, N - jl .L980 + jl L(980) #if GEMV_UNROLL == 1 -.L121: +L(121): decq N #endif @@ -2247,7 +2247,7 @@ movq M, I sarq $2, I - jle .L125 + jle L(125) movsd -16 * SIZE(A1), %xmm4 movhpd -15 * SIZE(A1), %xmm4 @@ -2259,10 +2259,10 @@ movhpd -9 * SIZE(A1), %xmm10 decq I - jle .L124 + jle L(124) ALIGN_3 -.L123: +L(123): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -2318,10 +2318,10 @@ subq $1, I BRANCH - jg .L123 + jg L(123) ALIGN_3 -.L124: +L(124): pshufd $0x4e, %xmm4, %xmm5 mulpd %xmm12, %xmm4 addpd %xmm4, %xmm0 @@ -2360,9 +2360,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L125: +L(125): testq $2, M - je .L127 + je L(127) movsd -16 * SIZE(A1), %xmm4 @@ -2390,12 +2390,12 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L127: +L(127): testq $1, M #if GEMV_UNROLL == 1 - je .L129 + je L(129) #else - je .L980 + je L(980) #endif movsd -16 * SIZE(A1), %xmm4 @@ -2411,27 +2411,27 @@ #if GEMV_UNROLL == 1 ALIGN_3 -.L129: +L(129): cmpq $1, N - jge .L121 + jge L(121) #endif #endif ALIGN_3 -.L980: +L(980): testq $SIZE, Y - jne .L990 + jne L(990) movq Y, Y1 movq M, %rax sarq $3, %rax - jle .L184 + jle L(184) ALIGN_3 -.L182: +L(182): movapd (Y), %xmm0 addq INCY, Y movapd (Y), %xmm1 @@ -2477,15 +2477,15 @@ subq $-16 * SIZE, BUFFER decq %rax - jg .L182 + jg L(182) ALIGN_3 -.L184: +L(184): testq $7, M - jle .L999 + jle L(999) testq $4, M - jle .L185 + jle L(185) movapd (Y), %xmm0 addq INCY, Y @@ -2513,9 +2513,9 @@ addq $8 * SIZE, BUFFER ALIGN_3 -.L185: +L(185): testq $2, M - jle .L186 + jle L(186) movapd (Y), %xmm0 addq INCY, Y @@ -2532,27 +2532,27 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L186: +L(186): testq $1, M - jle .L999 + jle L(999) movapd (Y), %xmm0 addpd (BUFFER), %xmm0 movapd %xmm0, (Y1) - jmp .L999 + jmp L(999) ALIGN_3 -.L990: +L(990): movq Y, Y1 movq M, %rax sarq $3, %rax - jle .L994 + jle L(994) ALIGN_3 -.L992: +L(992): movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 addq INCY, Y @@ -2628,15 +2628,15 @@ subq $-16 * SIZE, BUFFER decq %rax - jg .L992 + jg L(992) ALIGN_3 -.L994: +L(994): testq $7, M - jle .L999 + jle L(999) testq $4, M - jle .L995 + jle L(995) movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 @@ -2678,9 +2678,9 @@ addq $8 * SIZE, BUFFER ALIGN_3 -.L995: +L(995): testq $2, M - jle .L996 + jle L(996) movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 @@ -2704,9 +2704,9 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L996: +L(996): testq $1, M - jle .L999 + jle L(999) movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 @@ -2717,12 +2717,12 @@ movhpd %xmm0, 1 * SIZE(Y1) ALIGN_3 -.L999: +L(999): movq M, I salq $ZBASE_SHIFT,I addq I,AA - jmp .L0t -.L999x: + jmp L(0t) +L(999x): movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/zgemv_n_atom.S b/kernel/x86_64/zgemv_n_atom.S index 4fa70925ee..df9c786be8 100644 --- a/kernel/x86_64/zgemv_n_atom.S +++ b/kernel/x86_64/zgemv_n_atom.S @@ -175,9 +175,9 @@ subq $-16 * SIZE, A testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) ALIGN_3 movq BUFFER, Y1 @@ -189,7 +189,7 @@ sarq $3, %rax ALIGN_3 -.L01: +L(01): movapd %xmm4, 0 * SIZE(Y1) movapd %xmm4, 2 * SIZE(Y1) movapd %xmm4, 4 * SIZE(Y1) @@ -201,16 +201,16 @@ subq $-16 * SIZE, Y1 decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L10: +L(10): movq N, J sarq $1, J - jle .L20 + jle L(20) ALIGN_3 -.L11: +L(11): leaq 16 * SIZE(BUFFER), Y1 movq A, A1 leaq (A, LDA, 1), A2 @@ -257,7 +257,7 @@ movq M, I sarq $2, I - jle .L15 + jle L(15) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -276,10 +276,10 @@ ADD2 %xmm12, %xmm1 decq I - jle .L14 + jle L(14) ALIGN_3 -.L13: +L(13): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(A2) #endif @@ -422,10 +422,10 @@ subq $1, I BRANCH - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): movapd %xmm9, %xmm12 mulsd %xmm5, %xmm9 ADD1 %xmm10, %xmm2 @@ -546,9 +546,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L15: +L(15): testq $2, M - je .L17 + je L(17) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -622,9 +622,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L17: +L(17): testq $1, M - je .L19 + je L(19) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -660,14 +660,14 @@ movlpd %xmm1, -15 * SIZE(Y1) ALIGN_3 -.L19: +L(19): decq J - jg .L11 + jg L(11) ALIGN_3 -.L20: +L(20): testq $1, N - jle .L90 + jle L(90) leaq 16 * SIZE(BUFFER), Y1 movq A, A1 @@ -698,7 +698,7 @@ movq M, I sarq $2, I - jle .L25 + jle L(25) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -717,10 +717,10 @@ ADD2 %xmm12, %xmm1 decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(A2) #endif @@ -803,10 +803,10 @@ subq $1, I BRANCH - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): movapd %xmm9, %xmm12 mulsd %xmm5, %xmm9 ADD1 %xmm10, %xmm2 @@ -871,9 +871,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L25: +L(25): testq $2, M - je .L27 + je L(27) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -916,9 +916,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L27: +L(27): testq $1, M - je .L90 + je L(90) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -940,15 +940,15 @@ movlpd %xmm1, -15 * SIZE(Y1) ALIGN_3 -.L90: +L(90): movq Y, Y1 movq M, %rax sarq $3, %rax - jle .L94 + jle L(94) ALIGN_3 -.L92: +L(92): movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 addq INCY, Y @@ -1024,15 +1024,15 @@ subq $-16 * SIZE, BUFFER decq %rax - jg .L92 + jg L(92) ALIGN_3 -.L94: +L(94): testq $7, M - jle .L999 + jle L(999) testq $4, M - jle .L95 + jle L(95) movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 @@ -1074,9 +1074,9 @@ addq $8 * SIZE, BUFFER ALIGN_3 -.L95: +L(95): testq $2, M - jle .L96 + jle L(96) movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 @@ -1100,9 +1100,9 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L96: +L(96): testq $1, M - jle .L999 + jle L(999) movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 @@ -1113,7 +1113,7 @@ movhpd %xmm0, 1 * SIZE(Y1) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemv_n_dup.S b/kernel/x86_64/zgemv_n_dup.S index 42c1963c21..7fb28be4b9 100644 --- a/kernel/x86_64/zgemv_n_dup.S +++ b/kernel/x86_64/zgemv_n_dup.S @@ -151,9 +151,9 @@ subq $-16 * SIZE, A testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) ALIGN_3 movq BUFFER, Y1 @@ -165,7 +165,7 @@ sarq $3, %rax ALIGN_3 -.L01: +L(01): movaps %xmm4, 0 * SIZE(Y1) movaps %xmm4, 2 * SIZE(Y1) movaps %xmm4, 4 * SIZE(Y1) @@ -177,17 +177,17 @@ subq $-16 * SIZE, Y1 decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L10: +L(10): #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $4, N leaq 16 * SIZE(BUFFER), Y1 @@ -270,7 +270,7 @@ movq M, I sarq $2, I - jle .L15 + jle L(15) movddup -16 * SIZE(A1), %xmm4 movddup -14 * SIZE(A1), %xmm5 @@ -278,10 +278,10 @@ movddup -10 * SIZE(A1), %xmm7 decq I - jle .L14 + jle L(14) ALIGN_3 -.L13: +L(13): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -422,10 +422,10 @@ subq $1, I BRANCH - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): mulpd %xmm8, %xmm4 addpd %xmm4, %xmm0 movddup -15 * SIZE(A1), %xmm4 @@ -541,9 +541,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L15: +L(15): testq $2, M - je .L17 + je L(17) movddup -16 * SIZE(A1), %xmm4 movddup -15 * SIZE(A1), %xmm5 @@ -611,9 +611,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L17: +L(17): testq $1, M - je .L19 + je L(19) movddup -16 * SIZE(A1), %xmm4 movddup -15 * SIZE(A1), %xmm5 @@ -647,23 +647,23 @@ MOVUPS_YS1(-16 * SIZE, Y1, %xmm0) ALIGN_3 -.L19: +L(19): cmpq $4, N - jge .L11 + jge L(11) ALIGN_3 -.L20: +L(20): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 2 ALIGN_3 -.L21: +L(21): #endif subq $2, N @@ -726,7 +726,7 @@ movq M, I sarq $2, I - jle .L25 + jle L(25) movddup -16 * SIZE(A1), %xmm4 movddup -14 * SIZE(A1), %xmm5 @@ -734,10 +734,10 @@ movddup -10 * SIZE(A1), %xmm7 decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A2) #endif @@ -818,10 +818,10 @@ subq $1, I BRANCH - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): mulpd %xmm8, %xmm4 addpd %xmm4, %xmm0 movddup -15 * SIZE(A1), %xmm4 @@ -885,9 +885,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L25: +L(25): testq $2, M - je .L27 + je L(27) movddup -16 * SIZE(A1), %xmm4 movddup -15 * SIZE(A1), %xmm5 @@ -927,12 +927,12 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L27: +L(27): testq $1, M #if GEMV_UNROLL == 2 - je .L29 + je L(29) #else - je .L30 + je L(30) #endif movddup -16 * SIZE(A1), %xmm4 @@ -955,20 +955,20 @@ #if GEMV_UNROLL == 2 ALIGN_3 -.L29: +L(29): cmpq $2, N - jge .L21 + jge L(21) #endif ALIGN_3 -.L30: +L(30): #endif cmpq $1, N - jl .L980 + jl L(980) #if GEMV_UNROLL == 1 -.L31: +L(31): decq N #endif @@ -1021,7 +1021,7 @@ movq M, I sarq $2, I - jle .L35 + jle L(35) movddup -16 * SIZE(A1), %xmm4 movddup -14 * SIZE(A1), %xmm5 @@ -1029,10 +1029,10 @@ movddup -10 * SIZE(A1), %xmm7 decq I - jle .L34 + jle L(34) ALIGN_3 -.L33: +L(33): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -1082,10 +1082,10 @@ subq $1, I BRANCH - jg .L33 + jg L(33) ALIGN_3 -.L34: +L(34): mulpd %xmm8, %xmm4 addpd %xmm4, %xmm0 movddup -15 * SIZE(A1), %xmm4 @@ -1122,9 +1122,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L35: +L(35): testq $2, M - je .L37 + je L(37) movddup -16 * SIZE(A1), %xmm4 movddup -15 * SIZE(A1), %xmm5 @@ -1149,12 +1149,12 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L37: +L(37): testq $1, M #if GEMV_UNROLL == 1 - je .L39 + je L(39) #else - je .L980 + je L(980) #endif movddup -16 * SIZE(A1), %xmm4 @@ -1169,23 +1169,23 @@ #if GEMV_UNROLL == 1 ALIGN_3 -.L39: +L(39): cmpq $1, N - jge .L31 + jge L(31) #endif -.L980: +L(980): testq $SIZE, Y - jne .L990 + jne L(990) movq Y, Y1 movq M, %rax sarq $3, %rax - jle .L184 + jle L(184) ALIGN_3 -.L182: +L(182): movaps (Y), %xmm0 addq INCY, Y movaps (Y), %xmm1 @@ -1231,15 +1231,15 @@ subq $-16 * SIZE, BUFFER decq %rax - jg .L182 + jg L(182) ALIGN_3 -.L184: +L(184): testq $7, M - jle .L999 + jle L(999) testq $4, M - jle .L185 + jle L(185) movaps (Y), %xmm0 addq INCY, Y @@ -1267,9 +1267,9 @@ addq $8 * SIZE, BUFFER ALIGN_3 -.L185: +L(185): testq $2, M - jle .L186 + jle L(186) movaps (Y), %xmm0 addq INCY, Y @@ -1286,27 +1286,27 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L186: +L(186): testq $1, M - jle .L999 + jle L(999) movaps (Y), %xmm0 addpd (BUFFER), %xmm0 movaps %xmm0, (Y1) - jmp .L999 + jmp L(999) ALIGN_3 -.L990: +L(990): movq Y, Y1 movq M, %rax sarq $3, %rax - jle .L994 + jle L(994) ALIGN_3 -.L992: +L(992): movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 addq INCY, Y @@ -1382,15 +1382,15 @@ subq $-16 * SIZE, BUFFER decq %rax - jg .L992 + jg L(992) ALIGN_3 -.L994: +L(994): testq $7, M - jle .L999 + jle L(999) testq $4, M - jle .L995 + jle L(995) movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 @@ -1432,9 +1432,9 @@ addq $8 * SIZE, BUFFER ALIGN_3 -.L995: +L(995): testq $2, M - jle .L996 + jle L(996) movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 @@ -1458,9 +1458,9 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L996: +L(996): testq $1, M - jle .L999 + jle L(999) movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 @@ -1471,7 +1471,7 @@ movhpd %xmm0, 1 * SIZE(Y1) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemv_t.S b/kernel/x86_64/zgemv_t.S index c78927953f..1f9cbada1a 100644 --- a/kernel/x86_64/zgemv_t.S +++ b/kernel/x86_64/zgemv_t.S @@ -156,7 +156,7 @@ movsd %xmm0,ALPHAR movsd %xmm1,ALPHAI -.L0t: +L(0t): xorq I,I addq $1,I salq $19,I @@ -164,13 +164,13 @@ movq I,M movsd ALPHAR,%xmm0 movsd ALPHAI,%xmm1 - jge .L00t + jge L(00t) movq MMM,M addq I,M - jle .L999x + jle L(999x) -.L00t: +L(00t): movq AA, A movq NN, N movq LDAX, LDA @@ -195,9 +195,9 @@ subq $-16 * SIZE, A testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) ALIGN_3 movq BUFFER, X1 @@ -206,10 +206,10 @@ movq M, I sarq $2, I - jle .L05 + jle L(05) ALIGN_4 -.L02: +L(02): movsd 0 * SIZE(X), %xmm0 movhpd 1 * SIZE(X), %xmm0 addq INCX, X @@ -233,38 +233,38 @@ addq $8 * SIZE, X1 decq I - jg .L02 + jg L(02) ALIGN_4 -.L05: +L(05): movq M, I andq $3, I - jle .L10 + jle L(10) ALIGN_2 -.L06: +L(06): movsd 0 * SIZE(X), %xmm0 movhpd 1 * SIZE(X), %xmm0 addq INCX, X movapd %xmm0, 0 * SIZE(X1) addq $2 * SIZE, X1 decq I - jg .L06 + jg L(06) ALIGN_4 -.L10: +L(10): #ifdef ALIGNED_ACCESS testq $SIZE, A - jne .L100 + jne L(100) #endif #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $4, N leaq 16 * SIZE(BUFFER), X1 @@ -290,16 +290,16 @@ movq M, I sarq $2, I - jle .L15 + jle L(15) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm10) decq I - jle .L14 + jle L(14) ALIGN_3 -.L13: +L(13): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -442,10 +442,10 @@ subq $1, I BRANCH - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): pshufd $0x4e, %xmm8, %xmm9 mulpd %xmm12, %xmm8 addpd %xmm8, %xmm0 @@ -565,9 +565,9 @@ subq $-8 * SIZE, X1 ALIGN_3 -.L15: +L(15): testq $2, M - je .L17 + je L(17) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm10) @@ -631,9 +631,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L17: +L(17): testq $1, M - je .L19 + je L(19) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm10) @@ -665,7 +665,7 @@ SUBPD %xmm11, %xmm7 ALIGN_3 -.L19: +L(19): pcmpeqb %xmm13, %xmm13 psllq $63, %xmm13 shufps $0xc0, %xmm13, %xmm13 @@ -768,21 +768,21 @@ addq INCY, Y1 cmpq $4, N - jge .L11 + jge L(11) ALIGN_3 -.L20: +L(20): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 2 ALIGN_3 -.L21: +L(21): #endif subq $2, N @@ -807,7 +807,7 @@ movq M, I sarq $2, I - jle .L25 + jle L(25) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A1(-16 * SIZE, A2, %xmm10) @@ -815,10 +815,10 @@ MOVUPS_A1(-14 * SIZE, A2, %xmm6) decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -901,10 +901,10 @@ subq $1, I BRANCH - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): pshufd $0x4e, %xmm8, %xmm9 mulpd %xmm4, %xmm8 addpd %xmm8, %xmm0 @@ -970,9 +970,9 @@ subq $-8 * SIZE, X1 ALIGN_3 -.L25: +L(25): testq $2, M - je .L27 + je L(27) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A1(-16 * SIZE, A2, %xmm10) @@ -1010,9 +1010,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L27: +L(27): testq $1, M - je .L29 + je L(29) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A1(-16 * SIZE, A2, %xmm10) @@ -1030,7 +1030,7 @@ SUBPD %xmm11, %xmm3 ALIGN_3 -.L29: +L(29): pcmpeqb %xmm11, %xmm11 psllq $63, %xmm11 shufps $0xc0, %xmm11, %xmm11 @@ -1092,18 +1092,18 @@ #if GEMV_UNROLL == 2 cmpq $2, N - jge .L21 + jge L(21) #endif ALIGN_3 -.L30: +L(30): #endif cmpq $1, N - jl .L999 + jl L(999) #if GEMV_UNROLL == 1 -.L31: +L(31): decq N #endif @@ -1122,16 +1122,16 @@ movq M, I sarq $2, I - jle .L35 + jle L(35) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A1(-14 * SIZE, A1, %xmm12) decq I - jle .L34 + jle L(34) ALIGN_3 -.L33: +L(33): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -1181,10 +1181,10 @@ subq $1, I BRANCH - jg .L33 + jg L(33) ALIGN_3 -.L34: +L(34): pshufd $0x4e, %xmm8, %xmm9 mulpd %xmm4, %xmm8 addpd %xmm8, %xmm0 @@ -1223,9 +1223,9 @@ subq $-8 * SIZE, X1 ALIGN_3 -.L35: +L(35): testq $2, M - je .L37 + je L(37) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A1(-14 * SIZE, A1, %xmm12) @@ -1247,9 +1247,9 @@ addq $4 * SIZE, A1 ALIGN_3 -.L37: +L(37): testq $1, M - je .L39 + je L(39) MOVUPS_A1(-16 * SIZE, A1, %xmm8) @@ -1260,7 +1260,7 @@ SUBPD %xmm9, %xmm1 ALIGN_3 -.L39: +L(39): pcmpeqb %xmm11, %xmm11 psllq $63, %xmm11 shufps $0xc0, %xmm11, %xmm11 @@ -1303,21 +1303,21 @@ addq INCY, Y1 cmpq $1, N - jge .L31 + jge L(31) #endif #ifdef ALIGNED_ACCESS - jmp .L999 + jmp L(999) ALIGN_3 -.L100: +L(100): #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L110 + jl L(110) ALIGN_3 -.L101: +L(101): subq $4, N leaq 16 * SIZE(BUFFER), X1 @@ -1343,7 +1343,7 @@ movq M, I sarq $2, I - jle .L105 + jle L(105) movsd -16 * SIZE(A1), %xmm8 movhpd -15 * SIZE(A1), %xmm8 @@ -1352,10 +1352,10 @@ movhpd -15 * SIZE(A1, LDA), %xmm10 decq I - jle .L104 + jle L(104) ALIGN_3 -.L103: +L(103): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -1514,10 +1514,10 @@ subq $1, I BRANCH - jg .L103 + jg L(103) ALIGN_3 -.L104: +L(104): pshufd $0x4e, %xmm8, %xmm9 mulpd %xmm12, %xmm8 addpd %xmm8, %xmm0 @@ -1651,9 +1651,9 @@ subq $-8 * SIZE, X1 ALIGN_3 -.L105: +L(105): testq $2, M - je .L107 + je L(107) movsd -16 * SIZE(A1), %xmm8 movhpd -15 * SIZE(A1), %xmm8 @@ -1726,9 +1726,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L107: +L(107): testq $1, M - je .L109 + je L(109) movsd -16 * SIZE(A1), %xmm8 movhpd -15 * SIZE(A1), %xmm8 @@ -1765,7 +1765,7 @@ SUBPD %xmm11, %xmm7 ALIGN_3 -.L109: +L(109): pcmpeqb %xmm13, %xmm13 psllq $63, %xmm13 shufps $0xc0, %xmm13, %xmm13 @@ -1868,21 +1868,21 @@ addq INCY, Y1 cmpq $4, N - jge .L101 + jge L(101) ALIGN_3 -.L110: +L(110): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L120 + jl L(120) #if GEMV_UNROLL == 2 ALIGN_3 -.L111: +L(111): #endif subq $2, N @@ -1907,7 +1907,7 @@ movq M, I sarq $2, I - jle .L115 + jle L(115) movsd -16 * SIZE(A1), %xmm8 movhpd -15 * SIZE(A1), %xmm8 @@ -1920,10 +1920,10 @@ movhpd -13 * SIZE(A2), %xmm6 decq I - jle .L114 + jle L(114) ALIGN_3 -.L113: +L(113): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -2014,10 +2014,10 @@ subq $1, I BRANCH - jg .L113 + jg L(113) ALIGN_3 -.L114: +L(114): pshufd $0x4e, %xmm8, %xmm9 mulpd %xmm4, %xmm8 addpd %xmm8, %xmm0 @@ -2087,9 +2087,9 @@ subq $-8 * SIZE, X1 ALIGN_3 -.L115: +L(115): testq $2, M - je .L117 + je L(117) movsd -16 * SIZE(A1), %xmm8 movhpd -15 * SIZE(A1), %xmm8 @@ -2131,9 +2131,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L117: +L(117): testq $1, M - je .L119 + je L(119) movsd -16 * SIZE(A1), %xmm8 movhpd -15 * SIZE(A1), %xmm8 @@ -2153,7 +2153,7 @@ SUBPD %xmm11, %xmm3 ALIGN_3 -.L119: +L(119): pcmpeqb %xmm11, %xmm11 psllq $63, %xmm11 shufps $0xc0, %xmm11, %xmm11 @@ -2215,18 +2215,18 @@ #if GEMV_UNROLL == 2 cmpq $2, N - jge .L111 + jge L(111) #endif ALIGN_3 -.L120: +L(120): #endif cmpq $1, N - jl .L999 + jl L(999) #if GEMV_UNROLL == 1 -.L121: +L(121): decq N #endif @@ -2245,7 +2245,7 @@ movq M, I sarq $2, I - jle .L125 + jle L(125) movsd -16 * SIZE(A1), %xmm8 movhpd -15 * SIZE(A1), %xmm8 @@ -2253,10 +2253,10 @@ movhpd -13 * SIZE(A1), %xmm12 decq I - jle .L124 + jle L(124) ALIGN_3 -.L123: +L(123): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -2310,10 +2310,10 @@ subq $1, I BRANCH - jg .L123 + jg L(123) ALIGN_3 -.L124: +L(124): pshufd $0x4e, %xmm8, %xmm9 mulpd %xmm4, %xmm8 addpd %xmm8, %xmm0 @@ -2354,9 +2354,9 @@ subq $-8 * SIZE, X1 ALIGN_3 -.L125: +L(125): testq $2, M - je .L127 + je L(127) movsd -16 * SIZE(A1), %xmm8 movhpd -15 * SIZE(A1), %xmm8 @@ -2380,9 +2380,9 @@ addq $4 * SIZE, A1 ALIGN_3 -.L127: +L(127): testq $1, M - je .L129 + je L(129) movsd -16 * SIZE(A1), %xmm8 movhpd -15 * SIZE(A1), %xmm8 @@ -2394,7 +2394,7 @@ SUBPD %xmm9, %xmm1 ALIGN_3 -.L129: +L(129): pcmpeqb %xmm11, %xmm11 psllq $63, %xmm11 shufps $0xc0, %xmm11, %xmm11 @@ -2437,19 +2437,19 @@ addq INCY, Y1 cmpq $1, N - jge .L121 + jge L(121) #endif #endif ALIGN_3 -.L999: +L(999): movq M, I salq $ZBASE_SHIFT,I addq I,AA - jmp .L0t -.L999x: + jmp L(0t) +L(999x): movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/zgemv_t_atom.S b/kernel/x86_64/zgemv_t_atom.S index 73a013a245..501014188d 100644 --- a/kernel/x86_64/zgemv_t_atom.S +++ b/kernel/x86_64/zgemv_t_atom.S @@ -176,9 +176,9 @@ subq $-16 * SIZE, A testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) ALIGN_3 movq BUFFER, X1 @@ -187,10 +187,10 @@ movq M, I sarq $2, I - jle .L05 + jle L(05) ALIGN_4 -.L02: +L(02): movsd 0 * SIZE(X), %xmm0 movhpd 1 * SIZE(X), %xmm0 addq INCX, X @@ -214,32 +214,32 @@ addq $8 * SIZE, X1 decq I - jg .L02 + jg L(02) ALIGN_4 -.L05: +L(05): movq M, I andq $3, I - jle .L10 + jle L(10) ALIGN_2 -.L06: +L(06): movsd 0 * SIZE(X), %xmm0 movhpd 1 * SIZE(X), %xmm0 addq INCX, X movapd %xmm0, 0 * SIZE(X1) addq $2 * SIZE, X1 decq I - jg .L06 + jg L(06) ALIGN_4 -.L10: +L(10): movq N, J sarq $1, J - jle .L20 + jle L(20) ALIGN_3 -.L11: +L(11): leaq 16 * SIZE(BUFFER), X1 movq A, A1 @@ -262,7 +262,7 @@ movq M, I sarq $2, I - jle .L15 + jle L(15) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -274,10 +274,10 @@ mulsd %xmm5, %xmm12 decq I - jle .L14 + jle L(14) ALIGN_3 -.L13: +L(13): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(A1) #endif @@ -413,10 +413,10 @@ subq $1, I BRANCH - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): movapd %xmm9, %xmm13 mulsd %xmm5, %xmm9 ADD1 %xmm8, %xmm0 @@ -537,9 +537,9 @@ subq $-8 * SIZE, X1 ALIGN_3 -.L15: +L(15): testq $2, M - je .L17 + je L(17) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -606,9 +606,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L17: +L(17): testq $1, M - je .L19 + je L(19) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -641,7 +641,7 @@ ADD4 %xmm13, %xmm3 ALIGN_3 -.L19: +L(19): movsd 0 * SIZE(Y), %xmm4 movapd %xmm0, %xmm10 mulsd ALPHA_R, %xmm0 @@ -681,12 +681,12 @@ addq INCY, Y1 decq J - jg .L11 + jg L(11) ALIGN_3 -.L20: +L(20): testq $1, N - jle .L999 + jle L(999) leaq 16 * SIZE(BUFFER), X1 @@ -702,7 +702,7 @@ movq M, I sarq $2, I - jle .L25 + jle L(25) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -712,10 +712,10 @@ mulsd %xmm5, %xmm12 decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(A1) #endif @@ -790,10 +790,10 @@ subq $1, I BRANCH - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): movapd %xmm9, %xmm13 mulsd %xmm5, %xmm9 movsd -11 * SIZE(X1), %xmm5 @@ -858,9 +858,9 @@ subq $-8 * SIZE, X1 ALIGN_3 -.L25: +L(25): testq $2, M - je .L27 + je L(27) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -898,9 +898,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L27: +L(27): testq $1, M - je .L29 + je L(29) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -919,7 +919,7 @@ ADD4 %xmm13, %xmm1 ALIGN_3 -.L29: +L(29): movsd 0 * SIZE(Y), %xmm4 movapd %xmm0, %xmm10 mulsd ALPHA_R, %xmm0 @@ -939,7 +939,7 @@ movlpd %xmm1, 1 * SIZE(Y1) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemv_t_dup.S b/kernel/x86_64/zgemv_t_dup.S index d509f0eb8d..6218b5a3a0 100644 --- a/kernel/x86_64/zgemv_t_dup.S +++ b/kernel/x86_64/zgemv_t_dup.S @@ -158,9 +158,9 @@ subq $-16 * SIZE, A testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) ALIGN_3 movq BUFFER, X1 @@ -169,10 +169,10 @@ movq M, I sarq $2, I - jle .L05 + jle L(05) ALIGN_4 -.L02: +L(02): movsd 0 * SIZE(X), %xmm0 movhpd 1 * SIZE(X), %xmm0 addq INCX, X @@ -196,33 +196,33 @@ addq $8 * SIZE, X1 decq I - jg .L02 + jg L(02) ALIGN_4 -.L05: +L(05): movq M, I andq $3, I - jle .L10 + jle L(10) ALIGN_2 -.L06: +L(06): movsd 0 * SIZE(X), %xmm0 movhpd 1 * SIZE(X), %xmm0 addq INCX, X movapd %xmm0, 0 * SIZE(X1) addq $2 * SIZE, X1 decq I - jg .L06 + jg L(06) ALIGN_4 -.L10: +L(10): #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $4, N leaq 16 * SIZE(BUFFER), X1 @@ -248,7 +248,7 @@ movq M, I sarq $2, I - jle .L15 + jle L(15) movddup -16 * SIZE(A1), %xmm8 movddup -15 * SIZE(A1), %xmm9 @@ -256,10 +256,10 @@ movddup -15 * SIZE(A1, LDA), %xmm11 decq I - jle .L14 + jle L(14) ALIGN_3 -.L13: +L(13): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -394,10 +394,10 @@ subq $1, I BRANCH - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): mulpd %xmm12, %xmm8 addpd %xmm8, %xmm0 movddup -16 * SIZE(A2), %xmm8 @@ -507,9 +507,9 @@ subq $-8 * SIZE, X1 ALIGN_3 -.L15: +L(15): testq $2, M - je .L17 + je L(17) movddup -16 * SIZE(A1), %xmm8 movddup -15 * SIZE(A1), %xmm9 @@ -570,9 +570,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L17: +L(17): testq $1, M - je .L19 + je L(19) movddup -16 * SIZE(A1), %xmm8 movddup -15 * SIZE(A1), %xmm9 @@ -602,7 +602,7 @@ addpd %xmm11, %xmm7 ALIGN_3 -.L19: +L(19): pcmpeqb %xmm13, %xmm13 psllq $63, %xmm13 shufps $0x40, %xmm13, %xmm13 @@ -691,21 +691,21 @@ addq INCY, Y1 cmpq $4, N - jge .L11 + jge L(11) ALIGN_3 -.L20: +L(20): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 2 ALIGN_3 -.L21: +L(21): #endif subq $2, N @@ -729,7 +729,7 @@ movq M, I sarq $2, I - jle .L25 + jle L(25) movddup -16 * SIZE(A1), %xmm8 movddup -15 * SIZE(A1), %xmm9 @@ -737,10 +737,10 @@ movddup -15 * SIZE(A1, LDA), %xmm11 decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -815,10 +815,10 @@ subq $1, I BRANCH - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): mulpd %xmm12, %xmm8 addpd %xmm8, %xmm0 movddup -14 * SIZE(A1), %xmm8 @@ -876,9 +876,9 @@ subq $-8 * SIZE, X1 ALIGN_3 -.L25: +L(25): testq $2, M - je .L27 + je L(27) movddup -16 * SIZE(A1), %xmm8 movddup -15 * SIZE(A1), %xmm9 @@ -912,9 +912,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L27: +L(27): testq $1, M - je .L29 + je L(29) movddup -16 * SIZE(A1), %xmm8 movddup -15 * SIZE(A1), %xmm9 @@ -931,7 +931,7 @@ addpd %xmm11, %xmm3 ALIGN_3 -.L29: +L(29): pcmpeqb %xmm13, %xmm13 psllq $63, %xmm13 shufps $0x40, %xmm13, %xmm13 @@ -987,18 +987,18 @@ #if GEMV_UNROLL == 2 cmpq $2, N - jge .L21 + jge L(21) #endif ALIGN_3 -.L30: +L(30): #endif cmpq $1, N - jl .L999 + jl L(999) #if GEMV_UNROLL == 1 -.L31: +L(31): decq N #endif @@ -1018,7 +1018,7 @@ movq M, I sarq $2, I - jle .L35 + jle L(35) movddup -16 * SIZE(A1), %xmm8 movddup -15 * SIZE(A1), %xmm9 @@ -1026,10 +1026,10 @@ movddup -13 * SIZE(A1), %xmm11 decq I - jle .L34 + jle L(34) ALIGN_3 -.L33: +L(33): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -1075,10 +1075,10 @@ subq $1, I BRANCH - jg .L33 + jg L(33) ALIGN_3 -.L34: +L(34): mulpd %xmm12, %xmm8 addpd %xmm8, %xmm0 movddup -12 * SIZE(A1), %xmm8 @@ -1111,9 +1111,9 @@ subq $-8 * SIZE, X1 ALIGN_3 -.L35: +L(35): testq $2, M - je .L37 + je L(37) movddup -16 * SIZE(A1), %xmm8 movddup -15 * SIZE(A1), %xmm9 @@ -1134,9 +1134,9 @@ addq $4 * SIZE, A1 ALIGN_3 -.L37: +L(37): testq $1, M - je .L39 + je L(39) movddup -16 * SIZE(A1), %xmm8 movddup -15 * SIZE(A1), %xmm9 @@ -1147,7 +1147,7 @@ addpd %xmm9, %xmm1 ALIGN_3 -.L39: +L(39): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -1190,11 +1190,11 @@ addq INCY, Y1 cmpq $1, N - jge .L31 + jge L(31) #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/znrm2.S b/kernel/x86_64/znrm2.S index 748fde3102..13cc8ab5c2 100644 --- a/kernel/x86_64/znrm2.S +++ b/kernel/x86_64/znrm2.S @@ -56,9 +56,9 @@ fldz testq M, M - jle .L999 + jle L(999) testq INCX, INCX - je .L999 + je L(999) salq $ZBASE_SHIFT, INCX @@ -66,14 +66,14 @@ fldz fldz cmpq $SIZE * 2, INCX - jne .L40 + jne L(40) movq M, I sarq $2, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #if defined(PREFETCH) PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -109,16 +109,16 @@ faddp %st, %st(1) decq I - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): andq $3, M - jle .L998 + jle L(998) ALIGN_4 -.L21: +L(21): FLD 0 * SIZE(X) fmul %st(0), %st FLD 1 * SIZE(X) @@ -127,17 +127,17 @@ faddp %st,%st(1) addq $2 * SIZE, X decq M - jg .L21 - jmp .L998 + jg L(21) + jmp L(998) ALIGN_4 -.L40: +L(40): movq M, I sarq $2, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): FLD 0 * SIZE(X) fmul %st(0), %st FLD 1 * SIZE(X) @@ -171,16 +171,16 @@ faddp %st, %st(1) decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): andq $3, M - jle .L998 + jle L(998) ALIGN_4 -.L61: +L(61): FLD 0 * SIZE(X) fmul %st(0), %st FLD 1 * SIZE(X) @@ -189,16 +189,16 @@ faddp %st,%st(3) faddp %st,%st(1) decq M - jg .L61 + jg L(61) ALIGN_4 -.L998: +L(998): faddp %st,%st(2) faddp %st,%st(1) faddp %st,%st(1) ALIGN_4 -.L999: +L(999): fsqrt #ifndef XDOUBLE subq $2 * SIZE, %rsp diff --git a/kernel/x86_64/znrm2_sse.S b/kernel/x86_64/znrm2_sse.S index 2274f2e985..43fe15f3eb 100644 --- a/kernel/x86_64/znrm2_sse.S +++ b/kernel/x86_64/znrm2_sse.S @@ -55,10 +55,10 @@ pxor %xmm0, %xmm0 testq M, M - jle .L999 + jle L(999) pxor %xmm1, %xmm1 testq INCX, INCX - je .L999 + je L(999) xorq FLAG, FLAG @@ -66,10 +66,10 @@ leaq (, INCX, 2 * SIZE), INCX pxor %xmm3, %xmm3 cmpq $2 * SIZE, INCX - jne .L40 + jne L(40) testq $SIZE, X - je .L05 + je L(05) movss (X), %xmm4 cvtss2sd %xmm4, %xmm6 @@ -78,13 +78,13 @@ addq $SIZE, X movq $1, FLAG decq M - jle .L19 + jle L(19) ALIGN_3 -.L05: +L(05): movq M, I sarq $3, I - jle .L14 + jle L(14) movsd 0 * SIZE(X), %xmm4 movsd 2 * SIZE(X), %xmm5 @@ -97,10 +97,10 @@ addq $16 * SIZE, X decq I - jle .L12 + jle L(12) ALIGN_3 -.L10: +L(10): #if defined(PREFETCH) PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -147,10 +147,10 @@ subq $-16 * SIZE, X decq I - jg .L10 + jg L(10) ALIGN_3 -.L12: +L(12): cvtps2pd %xmm4, %xmm12 cvtps2pd %xmm5, %xmm13 cvtps2pd %xmm6, %xmm14 @@ -183,9 +183,9 @@ ALIGN_3 -.L14: +L(14): testq $4, M - je .L15 + je L(15) movsd 0 * SIZE(X), %xmm4 movsd 2 * SIZE(X), %xmm5 @@ -210,9 +210,9 @@ addq $8 * SIZE, X ALIGN_3 -.L15: +L(15): testq $2, M - je .L16 + je L(16) movsd 0 * SIZE(X), %xmm4 movsd 2 * SIZE(X), %xmm5 @@ -225,9 +225,9 @@ addq $4 * SIZE, X ALIGN_3 -.L16: +L(16): testq $1, M - je .L19 + je L(19) movsd (X), %xmm4 cvtps2pd %xmm4, %xmm6 @@ -236,24 +236,24 @@ addq $2 * SIZE, X ALIGN_3 -.L19: +L(19): testq FLAG, FLAG - je .L998 + je L(998) movss (X), %xmm4 cvtss2sd %xmm4, %xmm6 mulsd %xmm6, %xmm6 addsd %xmm6, %xmm3 - jmp .L998 + jmp L(998) ALIGN_4 -.L40: +L(40): movq M, I sarq $3, I - jle .L44 + jle L(44) ALIGN_4 -.L41: +L(41): movsd (X), %xmm4 addq INCX, X movsd (X), %xmm5 @@ -302,12 +302,12 @@ addpd %xmm11, %xmm3 decq I - jg .L41 + jg L(41) ALIGN_3 -.L44: +L(44): testq $4, M - je .L45 + je L(45) movsd (X), %xmm4 addq INCX, X @@ -334,9 +334,9 @@ addpd %xmm11, %xmm3 ALIGN_3 -.L45: +L(45): testq $2, M - je .L46 + je L(46) movsd (X), %xmm4 addq INCX, X @@ -351,9 +351,9 @@ addpd %xmm7, %xmm1 ALIGN_3 -.L46: +L(46): testq $1, M - je .L998 + je L(998) movsd (X), %xmm4 cvtps2pd %xmm4, %xmm6 @@ -361,7 +361,7 @@ addpd %xmm6, %xmm3 ALIGN_4 -.L998: +L(998): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -375,7 +375,7 @@ #endif ALIGN_4 -.L999: +L(999): sqrtsd %xmm0, %xmm0 cvtsd2ss %xmm0, %xmm0 diff --git a/kernel/x86_64/zrot.S b/kernel/x86_64/zrot.S index 22d031c076..5f1fb76e25 100644 --- a/kernel/x86_64/zrot.S +++ b/kernel/x86_64/zrot.S @@ -69,19 +69,19 @@ salq $ZBASE_SHIFT, INCY testq N, N - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX - jne .L50 + jne L(50) cmpq $2 * SIZE, INCY - jne .L50 + jne L(50) movq N, I sarq $1, I - jle .L15 + jle L(15) ALIGN_4 -.L10: +L(10): #if defined(PREFETCHW) PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -170,16 +170,16 @@ addq $4 * SIZE, Y decq I - jg .L10 + jg L(10) ALIGN_4 -.L15: +L(15): movq N, I andq $1, I - jle .L999 + jle L(999) ALIGN_4 -.L16: +L(16): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -217,16 +217,16 @@ fsubrp %st, %st(1) FST 1 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_4 -.L50: +L(50): movq N, I sarq $1, I - jle .L55 + jle L(55) ALIGN_4 -.L51: +L(51): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -310,16 +310,16 @@ addq INCY, Y decq I - jg .L51 + jg L(51) ALIGN_4 -.L55: +L(55): movq N, I andq $1, I - jle .L999 + jle L(999) ALIGN_4 -.L56: +L(56): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -359,7 +359,7 @@ FST 1 * SIZE(Y) ALIGN_4 -.L999: +L(999): ffreep %st ffreep %st ret diff --git a/kernel/x86_64/zrot_sse.S b/kernel/x86_64/zrot_sse.S index 6391311528..0ba3c64963 100644 --- a/kernel/x86_64/zrot_sse.S +++ b/kernel/x86_64/zrot_sse.S @@ -72,15 +72,15 @@ pshufd $0x0, %xmm1, S cmpq $0, N - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX - jne .L50 + jne L(50) cmpq $2 * SIZE, INCY - jne .L50 + jne L(50) testq $2 * SIZE, X - je .L10 + je L(10) movsd 0 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -102,18 +102,18 @@ addq $2 * SIZE, X addq $2 * SIZE, Y decq N - jle .L999 + jle L(999) -.L10: +L(10): testq $1 * SIZE, X - jne .L30 + jne L(30) testq $3 * SIZE, Y - jne .L20 + jne L(20) movq N, %rax sarq $4, %rax - jle .L14 + jle L(14) movaps 0 * SIZE(Y), %xmm1 movaps 4 * SIZE(Y), %xmm3 @@ -126,10 +126,10 @@ movaps 12 * SIZE(X), %xmm10 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #if defined(PREFETCHW) PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -262,10 +262,10 @@ addq $32 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movaps %xmm1, %xmm4 mulps S, %xmm1 movaps %xmm3, %xmm6 @@ -375,12 +375,12 @@ addq $32 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $15, N - jle .L999 + jle L(999) testq $8, N - jle .L15 + jle L(15) movaps 0 * SIZE(Y), %xmm1 movaps 0 * SIZE(X), %xmm0 @@ -446,9 +446,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $4, N - jle .L16 + jle L(16) movaps 0 * SIZE(Y), %xmm1 movaps 0 * SIZE(X), %xmm0 @@ -484,9 +484,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $2, N - jle .L17 + jle L(17) movaps 0 * SIZE(Y), %xmm1 movaps 0 * SIZE(X), %xmm0 @@ -510,9 +510,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $1, N - jle .L999 + jle L(999) movsd 0 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -531,13 +531,13 @@ movlps %xmm0, 0 * SIZE(X) movlps %xmm2, 0 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L20: +L(20): movq N, %rax sarq $4, %rax - jle .L24 + jle L(24) movsd 0 * SIZE(Y), %xmm1 movhps 2 * SIZE(Y), %xmm1 @@ -554,10 +554,10 @@ movaps 12 * SIZE(X), %xmm10 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): #if defined(PREFETCHW) PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -706,10 +706,10 @@ addq $32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): movaps %xmm1, %xmm4 mulps S, %xmm1 movaps %xmm3, %xmm6 @@ -831,12 +831,12 @@ addq $32 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $15, N - jle .L999 + jle L(999) testq $8, N - jle .L25 + jle L(25) movsd 0 * SIZE(Y), %xmm1 movhps 2 * SIZE(Y), %xmm1 @@ -910,9 +910,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $4, N - jle .L26 + jle L(26) movsd 0 * SIZE(Y), %xmm1 movhps 2 * SIZE(Y), %xmm1 @@ -952,9 +952,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $2, N - jle .L27 + jle L(27) movsd 0 * SIZE(Y), %xmm1 movhps 2 * SIZE(Y), %xmm1 @@ -980,9 +980,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $1, N - jle .L999 + jle L(999) movsd 0 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -1001,13 +1001,13 @@ movlps %xmm0, 0 * SIZE(X) movlps %xmm2, 0 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L30: +L(30): movq N, %rax sarq $4, %rax - jle .L34 + jle L(34) movsd 0 * SIZE(Y), %xmm1 movhps 2 * SIZE(Y), %xmm1 @@ -1028,10 +1028,10 @@ movhps 14 * SIZE(X), %xmm10 decq %rax - jle .L32 + jle L(32) ALIGN_3 -.L31: +L(31): #if defined(PREFETCHW) PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -1196,10 +1196,10 @@ addq $32 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L32: +L(32): movaps %xmm1, %xmm4 mulps S, %xmm1 movaps %xmm3, %xmm6 @@ -1333,12 +1333,12 @@ addq $32 * SIZE, Y ALIGN_3 -.L34: +L(34): testq $15, N - jle .L999 + jle L(999) testq $8, N - jle .L35 + jle L(35) movsd 0 * SIZE(Y), %xmm1 movhps 2 * SIZE(Y), %xmm1 @@ -1420,9 +1420,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L35: +L(35): testq $4, N - jle .L36 + jle L(36) movsd 0 * SIZE(Y), %xmm1 movhps 2 * SIZE(Y), %xmm1 @@ -1466,9 +1466,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L36: +L(36): testq $2, N - jle .L37 + jle L(37) movsd 0 * SIZE(Y), %xmm1 movhps 2 * SIZE(Y), %xmm1 @@ -1496,9 +1496,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L37: +L(37): testq $1, N - jle .L999 + jle L(999) movsd 0 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -1517,21 +1517,21 @@ movlps %xmm0, 0 * SIZE(X) movlps %xmm2, 0 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 ALIGN_3 -.L50: +L(50): movq N, %rax cmpq $0, INCX - je .L56 + je L(56) cmpq $0, INCY - je .L56 + je L(56) sarq $2, %rax - jle .L55 + jle L(55) ALIGN_3 -.L53: +L(53): movsd (Y), %xmm1 movhps (Y, INCY), %xmm1 movsd (X), %xmm0 @@ -1583,16 +1583,16 @@ leaq (Y, INCY, 2), Y decq %rax - jg .L53 + jg L(53) ALIGN_3 -.L55: +L(55): movq N, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_3 -.L56: +L(56): movsd (Y), %xmm1 movsd (X), %xmm0 @@ -1615,10 +1615,10 @@ addq INCY, Y decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/zrot_sse2.S b/kernel/x86_64/zrot_sse2.S index e6288c3d5c..f4a0653d6a 100644 --- a/kernel/x86_64/zrot_sse2.S +++ b/kernel/x86_64/zrot_sse2.S @@ -72,23 +72,23 @@ pshufd $0x44, %xmm1, S cmpq $0, N - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX - jne .L50 + jne L(50) cmpq $2 * SIZE, INCY - jne .L50 + jne L(50) -.L10: +L(10): testq $SIZE, X - jne .L30 + jne L(30) testq $SIZE, Y - jne .L20 + jne L(20) movq N, %rax sarq $3, %rax - jle .L14 + jle L(14) movapd 0 * SIZE(Y), %xmm1 movapd 2 * SIZE(Y), %xmm3 @@ -101,10 +101,10 @@ movapd 6 * SIZE(X), %xmm10 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -237,10 +237,10 @@ addq $16 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movapd %xmm1, %xmm4 mulpd S, %xmm1 movapd %xmm3, %xmm6 @@ -350,12 +350,12 @@ addq $16 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $7, N - jle .L999 + jle L(999) testq $4, N - jle .L15 + jle L(15) movapd 0 * SIZE(Y), %xmm1 movapd 0 * SIZE(X), %xmm0 @@ -421,9 +421,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $2, N - jle .L16 + jle L(16) movapd 0 * SIZE(Y), %xmm1 movapd 0 * SIZE(X), %xmm0 @@ -459,9 +459,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $1, N - jle .L999 + jle L(999) movapd 0 * SIZE(Y), %xmm1 movapd 0 * SIZE(X), %xmm0 @@ -480,18 +480,18 @@ movapd %xmm0, 0 * SIZE(X) movapd %xmm2, 0 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L20: +L(20): movapd -1 * SIZE(Y), %xmm1 movq N, %rax sarq $3, %rax - jle .L24 + jle L(24) ALIGN_3 -.L21: +L(21): #if defined(PREFETCHW) PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -652,15 +652,15 @@ addq $16 * SIZE, X addq $16 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L24: +L(24): testq $7, N - jle .L999 + jle L(999) testq $4, N - jle .L25 + jle L(25) movapd 1 * SIZE(Y), %xmm3 movapd 3 * SIZE(Y), %xmm8 @@ -736,9 +736,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $2, N - jle .L26 + jle L(26) movapd 1 * SIZE(Y), %xmm3 movapd 3 * SIZE(Y), %xmm8 @@ -780,9 +780,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $1, N - jle .L999 + jle L(999) movapd 1 * SIZE(Y), %xmm4 movapd 0 * SIZE(X), %xmm0 @@ -803,21 +803,21 @@ movapd %xmm0, 0 * SIZE(X) movlps %xmm2, 0 * SIZE(Y) movhps %xmm2, 1 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L30: +L(30): testq $SIZE, Y - jne .L40 + jne L(40) movapd -1 * SIZE(X), %xmm0 movq N, %rax sarq $3, %rax - jle .L34 + jle L(34) ALIGN_3 -.L31: +L(31): #if defined(PREFETCHW) PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -977,15 +977,15 @@ addq $16 * SIZE, Y addq $16 * SIZE, X decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L34: +L(34): testq $7, N - jle .L999 + jle L(999) testq $4, N - jle .L35 + jle L(35) movapd 1 * SIZE(X), %xmm2 movapd 3 * SIZE(X), %xmm8 @@ -1061,9 +1061,9 @@ addq $8 * SIZE, X ALIGN_3 -.L35: +L(35): testq $2, N - jle .L36 + jle L(36) movapd 1 * SIZE(X), %xmm2 movapd 3 * SIZE(X), %xmm8 @@ -1106,9 +1106,9 @@ addq $4 * SIZE, X ALIGN_3 -.L36: +L(36): testq $1, N - jle .L999 + jle L(999) movapd 1 * SIZE(X), %xmm4 movapd 0 * SIZE(Y), %xmm1 @@ -1129,10 +1129,10 @@ movlps %xmm0, 0 * SIZE(X) movhps %xmm0, 1 * SIZE(X) movapd %xmm2, 0 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L40: +L(40): movsd 0 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -1154,11 +1154,11 @@ addq $1 * SIZE, X decq N - jle .L47 + jle L(47) movq N, %rax sarq $3, %rax - jle .L44 + jle L(44) movapd 0 * SIZE(Y), %xmm1 movapd 2 * SIZE(Y), %xmm3 @@ -1171,10 +1171,10 @@ movapd 6 * SIZE(X), %xmm10 decq %rax - jle .L42 + jle L(42) ALIGN_3 -.L41: +L(41): #if defined(PREFETCHW) PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -1307,10 +1307,10 @@ addq $16 * SIZE, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L42: +L(42): movapd %xmm1, %xmm4 mulpd S, %xmm1 movapd %xmm3, %xmm6 @@ -1420,9 +1420,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L44: +L(44): testq $4, N - jle .L45 + jle L(45) movapd 0 * SIZE(Y), %xmm1 movapd 0 * SIZE(X), %xmm0 @@ -1488,9 +1488,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L45: +L(45): testq $2, N - jle .L46 + jle L(46) movapd 0 * SIZE(Y), %xmm1 movapd 0 * SIZE(X), %xmm0 @@ -1526,9 +1526,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L46: +L(46): testq $1, N - jle .L47 + jle L(47) movapd 0 * SIZE(Y), %xmm1 movapd 0 * SIZE(X), %xmm0 @@ -1551,7 +1551,7 @@ addq $2 * SIZE, X ALIGN_3 -.L47: +L(47): movsd 0 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -1569,16 +1569,16 @@ movsd %xmm0, 0 * SIZE(X) movsd %xmm2, 0 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L50: +L(50): movq N, %rax sarq $2, %rax - jle .L55 + jle L(55) ALIGN_3 -.L53: +L(53): movsd 0 * SIZE(Y), %xmm1 movhps 1 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -1680,16 +1680,16 @@ addq INCY, Y decq %rax - jg .L53 + jg L(53) ALIGN_3 -.L55: +L(55): movq N, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_3 -.L56: +L(56): movsd 0 * SIZE(Y), %xmm1 movhps 1 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -1716,10 +1716,10 @@ addq INCY, Y decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/zscal.S b/kernel/x86_64/zscal.S index 5ed4c4576b..2b1acefc26 100644 --- a/kernel/x86_64/zscal.S +++ b/kernel/x86_64/zscal.S @@ -60,7 +60,7 @@ FLD 24(%rsp) testq N, N - jle .L999 + jle L(999) fld %st(1) fabs @@ -71,21 +71,21 @@ fldz fcomip %st(1), %st ffreep %st - jne .L30 + jne L(30) EMMS pxor %mm0, %mm0 cmpq $2 * SIZE, INCX - jne .L20 + jne L(20) movq N, I sarq $2, I - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -108,16 +108,16 @@ movq %mm0, 120(X) addq $8 * SIZE, X decq I - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): movq N, I andq $3, I - jle .L18 + jle L(18) ALIGN_2 -.L16: +L(16): movq %mm0, 0(X) movq %mm0, 8(X) movq %mm0, 16(X) @@ -125,21 +125,21 @@ addq $2 * SIZE, X decq I - jg .L16 + jg L(16) -.L18: +L(18): EMMS ret ALIGN_2 -.L20: +L(20): movq N, I sarq $2, I - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): movq %mm0, 0(X) movq %mm0, 8(X) movq %mm0, 16(X) @@ -165,16 +165,16 @@ addq INCX, X decq I - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): movq N, I andq $3, I - jle .L28 + jle L(28) ALIGN_3 -.L26: +L(26): movq %mm0, 0(X) movq %mm0, 8(X) movq %mm0, 16(X) @@ -182,19 +182,19 @@ addq INCX, X decq I - jg .L26 + jg L(26) -.L28: +L(28): EMMS ret ALIGN_3 -.L30: +L(30): movq N, I ALIGN_2 -.L32: +L(32): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -215,10 +215,10 @@ FST 1 * SIZE(X) addq INCX, X decq I - jg .L32 + jg L(32) ALIGN_2 -.L999: +L(999): ffreep %st ffreep %st diff --git a/kernel/x86_64/zscal_atom.S b/kernel/x86_64/zscal_atom.S index 7713626c9a..5f6be9605b 100644 --- a/kernel/x86_64/zscal_atom.S +++ b/kernel/x86_64/zscal_atom.S @@ -69,26 +69,26 @@ salq $ZBASE_SHIFT, INCX testq M, M - jle .L999 + jle L(999) pxor %xmm15, %xmm15 comisd %xmm0, %xmm15 - jne .L30 # Alpha_r != ZERO - jp .L30 + jne L(30) # Alpha_r != ZERO + jp L(30) comisd %xmm1, %xmm15 - jne .L30 # Alpha_i != ZERO + jne L(30) # Alpha_i != ZERO /* Alpha == ZERO */ cmpq $2 * SIZE, INCX - jne .L20 + jne L(20) movq M, I sarq $2, I - jle .L12 + jle L(12) ALIGN_4 -.L11: +L(11): movsd %xmm1, 0 * SIZE(X) movsd %xmm1, 1 * SIZE(X) movsd %xmm1, 2 * SIZE(X) @@ -101,12 +101,12 @@ addq $8 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_4 -.L12: +L(12): testq $2, M - je .L14 + je L(14) movsd %xmm1, 0 * SIZE(X) movsd %xmm1, 1 * SIZE(X) @@ -116,23 +116,23 @@ addq $4 * SIZE, X ALIGN_3 -.L14: +L(14): testq $1, M - je .L999 + je L(999) movsd %xmm1, 0 * SIZE(X) movsd %xmm1, 1 * SIZE(X) addq $2 * SIZE, X - jmp .L999 + jmp L(999) ALIGN_4 -.L20: +L(20): movq M, I # rcx = n sarq $2, I - jle .L22 + jle L(22) ALIGN_4 -.L21: +L(21): movsd %xmm1, 0 * SIZE(X) movsd %xmm1, 1 * SIZE(X) addq INCX, X @@ -146,12 +146,12 @@ movsd %xmm1, 1 * SIZE(X) addq INCX, X decq I - jg .L21 + jg L(21) ALIGN_4 -.L22: +L(22): testq $2, M - je .L23 + je L(23) movsd %xmm1, 0 * SIZE(X) movsd %xmm1, 1 * SIZE(X) @@ -161,22 +161,22 @@ addq INCX, X ALIGN_3 -.L23: +L(23): testq $1, M - je .L999 + je L(999) movsd %xmm1, 0 * SIZE(X) movsd %xmm1, 1 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_4 /* Alpha != ZERO */ -.L30: +L(30): movq X, XX movq M, I sarq $2, I - jle .L35 + jle L(35) movsd 0 * SIZE(X), %xmm2 movsd 1 * SIZE(X), %xmm3 @@ -209,10 +209,10 @@ mulsd %xmm1, %xmm4 decq I - jle .L32 + jle L(32) ALIGN_4 -.L31: +L(31): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -282,10 +282,10 @@ mulsd %xmm1, %xmm4 decq I - jg .L31 + jg L(31) ALIGN_4 -.L32: +L(32): subsd %xmm5, %xmm6 movsd %xmm2, 0 * SIZE(XX) addsd %xmm4, %xmm7 @@ -323,9 +323,9 @@ addq INCX, XX ALIGN_3 -.L35: +L(35): testq $2, M - je .L37 + je L(37) movsd 0 * SIZE(X), %xmm2 movsd 1 * SIZE(X), %xmm3 @@ -362,9 +362,9 @@ addq INCX, XX ALIGN_3 -.L37: +L(37): testq $1, M - je .L999 + je L(999) movsd 0 * SIZE(X), %xmm2 movsd 1 * SIZE(X), %xmm3 @@ -383,7 +383,7 @@ movsd %xmm3, 1 * SIZE(XX) ALIGN_3 -.L999: +L(999): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/zscal_sse.S b/kernel/x86_64/zscal_sse.S index acd6c36549..41d59cf270 100644 --- a/kernel/x86_64/zscal_sse.S +++ b/kernel/x86_64/zscal_sse.S @@ -71,47 +71,47 @@ xor FLAG, FLAG testq M, M - jle .L999 + jle L(999) pxor %xmm15, %xmm15 comiss %xmm0, %xmm15 - jne .L100 # Alpha_r != ZERO - jp .L100 # Alpha_r == NAN + jne L(100) # Alpha_r != ZERO + jp L(100) # Alpha_r == NAN comiss %xmm1, %xmm15 - jne .L100 # Alpha_i != ZERO + jne L(100) # Alpha_i != ZERO /* Alpha == ZERO */ cmpq $2 * SIZE, INCX - jne .L50 + jne L(50) /* INCX == 1 */ cmpq $3, M - jle .L13 + jle L(13) testq $4, X - je .L05 + je L(05) movss %xmm15, 0 * SIZE(X) addq $SIZE, X movq $1, FLAG decq M ALIGN_3 -.L05: +L(05): testq $8, X - je .L06 + je L(06) movlps %xmm15, 0 * SIZE(X) addq $2 * SIZE, X subq $1, M ALIGN_3 -.L06: +L(06): movq M, I # rcx = n sarq $3, I - jle .L12 + jle L(12) ALIGN_4 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -122,53 +122,53 @@ movaps %xmm15, 12 * SIZE(X) addq $16 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_4 -.L12: +L(12): testq $7, M - je .L19 + je L(19) testq $4, M - je .L13 + je L(13) movaps %xmm15, 0 * SIZE(X) movaps %xmm15, 4 * SIZE(X) addq $8 * SIZE, X ALIGN_3 -.L13: +L(13): testq $2, M - je .L14 + je L(14) movlps %xmm15, 0 * SIZE(X) movhps %xmm15, 2 * SIZE(X) addq $4 * SIZE, X ALIGN_3 -.L14: +L(14): testq $1, M - je .L19 + je L(19) movlps %xmm15, 0 * SIZE(X) addq $2 * SIZE, X ALIGN_3 -.L19: +L(19): testq $1, FLAG - je .L999 + je L(999) movss %xmm15, 0 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_4 /* incx != 1 */ -.L50: +L(50): movq M, I # rcx = n sarq $2, I - jle .L52 + jle L(52) ALIGN_4 -.L51: +L(51): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -182,12 +182,12 @@ movsd %xmm15, 0 * SIZE(X) addq INCX, X decq I - jg .L51 + jg L(51) ALIGN_4 -.L52: +L(52): testq $2, M - je .L53 + je L(53) movsd %xmm15, 0 * SIZE(X) addq INCX, X @@ -195,22 +195,22 @@ addq INCX, X ALIGN_3 -.L53: +L(53): testq $1, M - je .L999 + je L(999) movsd %xmm15, 0 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_4 /* Alpha != ZERO */ -.L100: +L(100): testq $SIZE, X - jne .L130 + jne L(130) cmpq $2 * SIZE, INCX - jne .L120 + jne L(120) pshufd $0, %xmm0, %xmm14 pshufd $0, %xmm1, %xmm1 @@ -220,7 +220,7 @@ subq $-32 * SIZE, X testq $2 * SIZE, X - je .L105 + je L(105) movsd -32 * SIZE(X), %xmm0 @@ -232,13 +232,13 @@ movlps %xmm0, -32 * SIZE(X) addq $2 * SIZE, X decq M - jle .L999 + jle L(999) ALIGN_3 -.L105: +L(105): movq M, I sarq $4, I - jle .L115 + jle L(115) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -250,10 +250,10 @@ movaps -4 * SIZE(X), %xmm7 decq I - jle .L112 + jle L(112) ALIGN_4 -.L111: +L(111): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -320,10 +320,10 @@ subq $-32 * SIZE, X decq I - jg .L111 + jg L(111) ALIGN_4 -.L112: +L(112): pshufd $0xb1, %xmm0, %xmm8 mulps %xmm14, %xmm0 mulps %xmm15, %xmm8 @@ -375,9 +375,9 @@ subq $-32 * SIZE, X ALIGN_4 -.L115: +L(115): testq $8, M - je .L116 + je L(116) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -412,9 +412,9 @@ addq $16 * SIZE, X ALIGN_3 -.L116: +L(116): testq $4, M - je .L117 + je L(117) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -434,9 +434,9 @@ addq $8 * SIZE, X ALIGN_3 -.L117: +L(117): testq $2, M - je .L118 + je L(118) movaps -32 * SIZE(X), %xmm0 @@ -449,9 +449,9 @@ addq $4 * SIZE, X ALIGN_3 -.L118: +L(118): testq $1, M - je .L999 + je L(999) movsd -32 * SIZE(X), %xmm0 @@ -461,10 +461,10 @@ addps %xmm8, %xmm0 movlps %xmm0, -32 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_3 -.L120: +L(120): pshufd $0, %xmm0, %xmm14 pshufd $0, %xmm1, %xmm1 subps %xmm1, %xmm15 @@ -474,7 +474,7 @@ movq M, I sarq $3, I - jle .L125 + jle L(125) movsd (X), %xmm0 addq INCX, X @@ -497,10 +497,10 @@ addq INCX, X decq I - jle .L122 + jle L(122) ALIGN_4 -.L121: +L(121): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -566,10 +566,10 @@ addq INCX, X decq I - jg .L121 + jg L(121) ALIGN_4 -.L122: +L(122): pshufd $0xb1, %xmm0, %xmm8 mulps %xmm14, %xmm0 mulps %xmm15, %xmm8 @@ -611,9 +611,9 @@ addq INCX, XX ALIGN_4 -.L125: +L(125): testq $4, M - je .L127 + je L(127) movsd (X), %xmm0 addq INCX, X @@ -646,9 +646,9 @@ addq INCX, XX ALIGN_3 -.L127: +L(127): testq $2, M - je .L128 + je L(128) movsd (X), %xmm0 addq INCX, X @@ -666,9 +666,9 @@ addq INCX, XX ALIGN_3 -.L128: +L(128): testq $1, M - je .L999 + je L(999) movsd (X), %xmm0 @@ -678,12 +678,12 @@ addps %xmm8, %xmm0 movlps %xmm0, (XX) - jmp .L999 + jmp L(999) ALIGN_3 -.L130: +L(130): cmpq $2 * SIZE, INCX - jne .L120 + jne L(120) #if defined(ALIGNED_ACCESS) && !defined(NEHALEM) && !defined(SANDYBRIDGE) @@ -695,7 +695,7 @@ subq $-31 * SIZE, X testq $2 * SIZE, X - je .L130x + je L(130x) movsd -31 * SIZE(X), %xmm0 @@ -707,10 +707,10 @@ movlps %xmm0, -31 * SIZE(X) addq $2 * SIZE, X decq M - jle .L999 + jle L(999) ALIGN_3 -.L130x: +L(130x): shufps $0xb1, %xmm15, %xmm15 movaps -32 * SIZE(X), %xmm0 @@ -718,7 +718,7 @@ movq M, I sarq $4, I - jle .L135 + jle L(135) movaps -28 * SIZE(X), %xmm1 movaps -24 * SIZE(X), %xmm2 @@ -729,10 +729,10 @@ movaps -4 * SIZE(X), %xmm7 decq I - jle .L132 + jle L(132) ALIGN_4 -.L131: +L(131): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -831,10 +831,10 @@ subq $-32 * SIZE, X decq I - jg .L131 + jg L(131) ALIGN_4 -.L132: +L(132): movss %xmm1, %xmm0 pshufd $0x1b, %xmm0, %xmm8 mulps %xmm14, %xmm0 @@ -912,9 +912,9 @@ subq $-32 * SIZE, X ALIGN_4 -.L135: +L(135): testq $8, M - je .L136 + je L(136) movaps -28 * SIZE(X), %xmm1 @@ -967,9 +967,9 @@ addq $16 * SIZE, X ALIGN_3 -.L136: +L(136): testq $4, M - je .L137 + je L(137) movaps -28 * SIZE(X), %xmm1 @@ -1000,9 +1000,9 @@ addq $8 * SIZE, X ALIGN_3 -.L137: +L(137): testq $2, M - je .L138 + je L(138) movaps -28 * SIZE(X), %xmm1 @@ -1021,11 +1021,11 @@ addq $4 * SIZE, X ALIGN_3 -.L138: +L(138): movss %xmm9, -32 * SIZE(X) testq $1, M - je .L999 + je L(999) pshufd $0x1b, %xmm0, %xmm8 mulps %xmm14, %xmm0 @@ -1035,7 +1035,7 @@ pshufd $0x39, %xmm0, %xmm0 movlps %xmm0, -31 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_3 @@ -1049,7 +1049,7 @@ subq $-32 * SIZE, X testq $2 * SIZE, X - je .L130x + je L(130x) movsd -32 * SIZE(X), %xmm0 @@ -1061,13 +1061,13 @@ movlps %xmm0, -32 * SIZE(X) addq $2 * SIZE, X decq M - jle .L999 + jle L(999) ALIGN_3 -.L130x: +L(130x): movq M, I sarq $4, I - jle .L135 + jle L(135) movsd -32 * SIZE(X), %xmm0 movhps -30 * SIZE(X), %xmm0 @@ -1087,10 +1087,10 @@ movhps -2 * SIZE(X), %xmm7 decq I - jle .L132 + jle L(132) ALIGN_4 -.L131: +L(131): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -1181,10 +1181,10 @@ subq $-32 * SIZE, X decq I - jg .L131 + jg L(131) ALIGN_4 -.L132: +L(132): pshufd $0xb1, %xmm0, %xmm8 mulps %xmm14, %xmm0 mulps %xmm15, %xmm8 @@ -1244,9 +1244,9 @@ subq $-32 * SIZE, X ALIGN_4 -.L135: +L(135): testq $8, M - je .L136 + je L(136) movsd -32 * SIZE(X), %xmm0 movhps -30 * SIZE(X), %xmm0 @@ -1291,9 +1291,9 @@ addq $16 * SIZE, X ALIGN_3 -.L136: +L(136): testq $4, M - je .L137 + je L(137) movsd -32 * SIZE(X), %xmm0 movhps -30 * SIZE(X), %xmm0 @@ -1317,9 +1317,9 @@ addq $8 * SIZE, X ALIGN_3 -.L137: +L(137): testq $2, M - je .L138 + je L(138) movsd -32 * SIZE(X), %xmm0 movhps -30 * SIZE(X), %xmm0 @@ -1334,9 +1334,9 @@ addq $4 * SIZE, X ALIGN_3 -.L138: +L(138): testq $1, M - je .L999 + je L(999) movsd -32 * SIZE(X), %xmm0 @@ -1349,7 +1349,7 @@ ALIGN_3 #endif -.L999: +L(999): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/zscal_sse2.S b/kernel/x86_64/zscal_sse2.S index d6a49136d1..2a0bab379a 100644 --- a/kernel/x86_64/zscal_sse2.S +++ b/kernel/x86_64/zscal_sse2.S @@ -77,38 +77,38 @@ xor FLAG, FLAG testq M, M - jle .L999 + jle L(999) pxor %xmm15, %xmm15 comisd %xmm0, %xmm15 - jne .L100 - jp .L100 + jne L(100) + jp L(100) comisd %xmm1, %xmm15 - jne .L100 + jne L(100) /* Alpha == ZERO */ cmpq $2 * SIZE, INCX - jne .L20 + jne L(20) /* INCX == 1 */ testq $SIZE, X - je .L05 + je L(05) movsd %xmm15, 0 * SIZE(X) addq $SIZE, X movq $1, FLAG decq M - jle .L19 + jle L(19) ALIGN_3 -.L05: +L(05): movq M, I # rcx = n sarq $3, I - jle .L12 + jle L(12) ALIGN_4 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -129,12 +129,12 @@ addq $16 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_4 -.L12: +L(12): testq $4, M - je .L13 + je L(13) movaps %xmm15, 0 * SIZE(X) movaps %xmm15, 2 * SIZE(X) @@ -143,42 +143,42 @@ addq $8 * SIZE, X ALIGN_3 -.L13: +L(13): testq $2, M - je .L14 + je L(14) movaps %xmm15, 0 * SIZE(X) movaps %xmm15, 2 * SIZE(X) addq $4 * SIZE, X ALIGN_3 -.L14: +L(14): testq $1, M - je .L19 + je L(19) movaps %xmm15, 0 * SIZE(X) addq $2 * SIZE, X ALIGN_3 -.L19: +L(19): testq $1, FLAG - je .L999 + je L(999) movsd %xmm15, 0 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_4 /* incx != 1 */ -.L20: +L(20): testq $SIZE, X - jne .L30 + jne L(30) /* Aligned Mode */ movq M, I # rcx = n sarq $2, I - jle .L22 + jle L(22) ALIGN_4 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -192,15 +192,15 @@ movaps %xmm15, (X) addq INCX, X decq I - jg .L21 + jg L(21) ALIGN_4 -.L22: +L(22): testq $3, M - je .L999 + je L(999) testq $2, M - je .L23 + je L(23) movaps %xmm15, (X) addq INCX, X @@ -208,23 +208,23 @@ addq INCX, X ALIGN_3 -.L23: +L(23): testq $1, M - je .L999 + je L(999) movaps %xmm15, (X) - jmp .L999 + jmp L(999) ALIGN_4 /* Unaligned Mode */ -.L30: +L(30): movq M, I # rcx = n sarq $2, I - jle .L32 + jle L(32) ALIGN_4 -.L31: +L(31): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -242,15 +242,15 @@ movlps %xmm15, 1 * SIZE(X) addq INCX, X decq I - jg .L31 + jg L(31) ALIGN_4 -.L32: +L(32): testq $3, M - je .L999 + je L(999) testq $2, M - je .L33 + je L(33) movlps %xmm15, 0 * SIZE(X) movlps %xmm15, 1 * SIZE(X) @@ -260,19 +260,19 @@ addq INCX, X ALIGN_3 -.L33: +L(33): testq $1, M - je .L999 + je L(999) movlps %xmm15, 0 * SIZE(X) movlps %xmm15, 1 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_4 /* Alpha != ZERO */ -.L100: +L(100): testq $SIZE, X - jne .L200 + jne L(200) #ifdef HAVE_SSE3 movddup %xmm0, %xmm14 @@ -284,13 +284,13 @@ movlhps %xmm1, %xmm15 cmpq $2 * SIZE, INCX - jne .L120 + jne L(120) subq $-16 * SIZE, X movq M, I sarq $3, I - jle .L115 + jle L(115) movaps -16 * SIZE(X), %xmm0 movaps -14 * SIZE(X), %xmm1 @@ -302,10 +302,10 @@ movaps -2 * SIZE(X), %xmm7 decq I - jle .L112 + jle L(112) ALIGN_4 -.L111: +L(111): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -412,10 +412,10 @@ subq $-16 * SIZE, X decq I - jg .L111 + jg L(111) ALIGN_4 -.L112: +L(112): pshufd $0x4e, %xmm0, %xmm8 mulpd %xmm14, %xmm0 mulpd %xmm15, %xmm8 @@ -467,12 +467,12 @@ subq $-16 * SIZE, X ALIGN_3 -.L115: +L(115): testq $7, M - je .L999 + je L(999) testq $4, M - je .L116 + je L(116) movaps -16 * SIZE(X), %xmm0 movaps -14 * SIZE(X), %xmm1 @@ -507,9 +507,9 @@ addq $8 * SIZE, X ALIGN_3 -.L116: +L(116): testq $2, M - je .L117 + je L(117) movaps -16 * SIZE(X), %xmm0 movaps -14 * SIZE(X), %xmm1 @@ -529,9 +529,9 @@ addq $4 * SIZE, X ALIGN_3 -.L117: +L(117): testq $1, M - je .L999 + je L(999) movaps -16 * SIZE(X), %xmm0 @@ -541,15 +541,15 @@ addpd %xmm8, %xmm0 movaps %xmm0, -16 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_3 -.L120: +L(120): movq X, XX movq M, I sarq $3, I - jle .L125 + jle L(125) movaps (X), %xmm0 addq INCX, X @@ -569,10 +569,10 @@ addq INCX, X decq I - jle .L122 + jle L(122) ALIGN_4 -.L121: +L(121): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -654,10 +654,10 @@ addq INCX, X decq I - jg .L121 + jg L(121) ALIGN_4 -.L122: +L(122): pshufd $0x4e, %xmm0, %xmm8 mulpd %xmm14, %xmm0 mulpd %xmm15, %xmm8 @@ -715,12 +715,12 @@ addq INCX, XX ALIGN_3 -.L125: +L(125): testq $7, M - je .L999 + je L(999) testq $4, M - je .L126 + je L(126) movaps (X), %xmm0 addq INCX, X @@ -761,9 +761,9 @@ addq INCX, XX ALIGN_3 -.L126: +L(126): testq $2, M - je .L127 + je L(127) movaps (X), %xmm0 addq INCX, X @@ -785,9 +785,9 @@ addq INCX, XX ALIGN_3 -.L127: +L(127): testq $1, M - je .L999 + je L(999) movaps (X), %xmm0 @@ -797,12 +797,12 @@ addpd %xmm8, %xmm0 movaps %xmm0, (XX) - jmp .L999 + jmp L(999) ALIGN_3 -.L200: +L(200): cmpq $2 * SIZE, INCX - jne .L220 + jne L(220) #if defined(ALIGNED_ACCESS) && !defined(NEHALEM) && !defined(SANDYBRIDGE) @@ -827,17 +827,17 @@ movq M, I sarq $3, I - jle .L205 + jle L(205) movaps -13 * SIZE(X), %xmm2 movaps -11 * SIZE(X), %xmm3 movaps -9 * SIZE(X), %xmm4 decq I - jle .L202 + jle L(202) ALIGN_4 -.L201: +L(201): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -912,10 +912,10 @@ subq $-16 * SIZE, X decq I - jg .L201 + jg L(201) ALIGN_4 -.L202: +L(202): movaps %xmm1, %xmm8 SHUFPD_1 %xmm2, %xmm0 mulpd %xmm14, %xmm8 @@ -980,9 +980,9 @@ subq $-16 * SIZE, X ALIGN_3 -.L205: +L(205): testq $4, M - je .L206 + je L(206) movaps -13 * SIZE(X), %xmm2 @@ -1023,9 +1023,9 @@ addq $8 * SIZE, X ALIGN_3 -.L206: +L(206): testq $2, M - je .L207 + je L(207) movaps -13 * SIZE(X), %xmm2 @@ -1053,9 +1053,9 @@ addq $4 * SIZE, X ALIGN_3 -.L207: +L(207): testq $1, M - je .L208 + je L(208) movaps -13 * SIZE(X), %xmm2 @@ -1072,13 +1072,13 @@ addq $2 * SIZE, X ALIGN_3 -.L208: +L(208): unpckhpd %xmm0, %xmm0 mulsd %xmm14, %xmm1 mulsd %xmm15, %xmm0 addsd %xmm1, %xmm0 movlps %xmm0, -15 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_3 #else @@ -1092,7 +1092,7 @@ movq M, I sarq $3, I - jle .L205 + jle L(205) movsd -16 * SIZE(X), %xmm0 movhps -15 * SIZE(X), %xmm0 @@ -1113,10 +1113,10 @@ movhps -1 * SIZE(X), %xmm7 decq I - jle .L202 + jle L(202) ALIGN_4 -.L201: +L(201): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -1239,10 +1239,10 @@ subq $-16 * SIZE, X decq I - jg .L201 + jg L(201) ALIGN_4 -.L202: +L(202): pshufd $0x4e, %xmm0, %xmm8 mulpd %xmm14, %xmm0 mulpd %xmm15, %xmm8 @@ -1302,12 +1302,12 @@ subq $-16 * SIZE, X ALIGN_3 -.L205: +L(205): testq $7, M - je .L999 + je L(999) testq $4, M - je .L206 + je L(206) movsd -16 * SIZE(X), %xmm0 movhps -15 * SIZE(X), %xmm0 @@ -1350,9 +1350,9 @@ addq $8 * SIZE, X ALIGN_3 -.L206: +L(206): testq $2, M - je .L207 + je L(207) movsd -16 * SIZE(X), %xmm0 movhps -15 * SIZE(X), %xmm0 @@ -1377,9 +1377,9 @@ addq $4 * SIZE, X ALIGN_3 -.L207: +L(207): testq $1, M - je .L999 + je L(999) movsd -16 * SIZE(X), %xmm0 movhps -15 * SIZE(X), %xmm0 @@ -1391,12 +1391,12 @@ movlps %xmm0, -16 * SIZE(X) movhps %xmm0, -15 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_3 #endif -.L220: +L(220): movddup %xmm0, %xmm14 pxor %xmm15, %xmm15 subsd %xmm1, %xmm15 @@ -1406,7 +1406,7 @@ movq M, I sarq $3, I - jle .L225 + jle L(225) movsd 0 * SIZE(X), %xmm0 movhps 1 * SIZE(X), %xmm0 @@ -1434,10 +1434,10 @@ addq INCX, X decq I - jle .L222 + jle L(222) ALIGN_4 -.L221: +L(221): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -1544,10 +1544,10 @@ decq I - jg .L221 + jg L(221) ALIGN_4 -.L222: +L(222): pshufd $0x4e, %xmm0, %xmm8 mulpd %xmm14, %xmm0 mulpd %xmm15, %xmm8 @@ -1613,12 +1613,12 @@ addq INCX, XX ALIGN_3 -.L225: +L(225): testq $7, M - je .L999 + je L(999) testq $4, M - je .L226 + je L(226) movsd 0 * SIZE(X), %xmm0 movhps 1 * SIZE(X), %xmm0 @@ -1669,9 +1669,9 @@ addq INCX, XX ALIGN_3 -.L226: +L(226): testq $2, M - je .L227 + je L(227) movsd 0 * SIZE(X), %xmm0 movhps 1 * SIZE(X), %xmm0 @@ -1698,9 +1698,9 @@ addq INCX, XX ALIGN_3 -.L227: +L(227): testq $1, M - je .L999 + je L(999) movsd 0 * SIZE(X), %xmm0 movhps 1 * SIZE(X), %xmm0 @@ -1714,7 +1714,7 @@ movhps %xmm0, 1 * SIZE(XX) ALIGN_3 -.L999: +L(999): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/zsum.S b/kernel/x86_64/zsum.S index aa02637e47..159bfb44b8 100644 --- a/kernel/x86_64/zsum.S +++ b/kernel/x86_64/zsum.S @@ -56,9 +56,9 @@ fldz testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) salq $ZBASE_SHIFT, INCX @@ -66,14 +66,14 @@ fldz fldz cmpq $SIZE * 2, INCX - jne .L40 + jne L(40) movq M, I sarq $2, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -101,33 +101,33 @@ faddp %st, %st(1) decq I - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): andq $3, M - jle .L998 + jle L(998) ALIGN_4 -.L21: +L(21): FLD 0 * SIZE(X) FLD 1 * SIZE(X) faddp %st,%st(3) faddp %st,%st(1) addq $2 * SIZE, X decq M - jg .L21 - jmp .L998 + jg L(21) + jmp L(998) ALIGN_4 -.L40: +L(40): movq M, I sarq $2, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): FLD 0 * SIZE(X) FLD 1 * SIZE(X) addq INCX, X @@ -153,32 +153,32 @@ faddp %st, %st(1) decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): andq $3, M - jle .L998 + jle L(998) ALIGN_4 -.L61: +L(61): FLD 0 * SIZE(X) FLD 1 * SIZE(X) addq INCX, X faddp %st,%st(3) faddp %st,%st(1) decq M - jg .L61 + jg L(61) ALIGN_4 -.L998: +L(998): faddp %st,%st(2) faddp %st,%st(1) faddp %st,%st(1) ALIGN_4 -.L999: +L(999): ret EPILOGUE diff --git a/kernel/x86_64/zsum_sse.S b/kernel/x86_64/zsum_sse.S index b679b42b0e..eeacae1d97 100644 --- a/kernel/x86_64/zsum_sse.S +++ b/kernel/x86_64/zsum_sse.S @@ -54,9 +54,9 @@ pxor %xmm0, %xmm0 testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) pxor %xmm1, %xmm1 pxor %xmm2, %xmm2 @@ -65,25 +65,25 @@ salq $ZBASE_SHIFT, INCX cmpq $2 * SIZE, INCX - jne .L100 + jne L(100) subq $-32 * SIZE, X addq M, M cmpq $3, M - jle .L18 + jle L(18) testq $4, X - je .L05 + je L(05) movss -32 * SIZE(X), %xmm0 addq $SIZE, X decq M - jle .L998 + jle L(998) ALIGN_3 -.L05: +L(05): testq $8, X - je .L10 + je L(10) #ifdef movsd xorps %xmm1, %xmm1 @@ -91,13 +91,13 @@ movsd -32 * SIZE(X), %xmm1 addq $2 * SIZE, X subq $2, M - jle .L998 + jle L(998) ALIGN_3 -.L10: +L(10): movq M, I sarq $5, I - jle .L14 + jle L(14) movaps -32 * SIZE(X), %xmm4 movaps -28 * SIZE(X), %xmm5 @@ -109,10 +109,10 @@ movaps -8 * SIZE(X), %xmm10 movaps -4 * SIZE(X), %xmm11 decq I - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -147,10 +147,10 @@ subq $-32 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): addps %xmm4, %xmm0 addps %xmm5, %xmm1 @@ -166,13 +166,13 @@ addq $32 * SIZE, X ALIGN_3 -.L14: +L(14): testq $31, M - jle .L998 + jle L(998) -.L15: +L(15): testq $16, M - je .L16 + je L(16) movaps -32 * SIZE(X), %xmm4 addps %xmm4, %xmm0 @@ -189,9 +189,9 @@ addq $16 * SIZE, X ALIGN_3 -.L16: +L(16): testq $8, M - je .L17 + je L(17) movaps -32 * SIZE(X), %xmm4 addps %xmm4, %xmm0 @@ -202,18 +202,18 @@ addq $8 * SIZE, X ALIGN_3 -.L17: +L(17): testq $4, M - je .L18 + je L(18) movaps -32 * SIZE(X), %xmm6 addps %xmm6, %xmm2 addq $4 * SIZE, X ALIGN_3 -.L18: +L(18): testq $2, M - je .L19 + je L(19) #ifdef movsd xorps %xmm7, %xmm7 @@ -223,22 +223,22 @@ addq $2 * SIZE, X ALIGN_3 -.L19: +L(19): testq $1, M - je .L998 + je L(998) movss -32 * SIZE(X), %xmm6 addps %xmm6, %xmm2 - jmp .L998 + jmp L(998) ALIGN_4 -.L100: +L(100): movq M, I sarq $2, I - jle .L105 + jle L(105) ALIGN_4 -.L101: +L(101): movsd (X), %xmm4 addq INCX, X movhps (X), %xmm4 @@ -254,26 +254,26 @@ addps %xmm5, %xmm1 decq I - jg .L101 + jg L(101) ALIGN_4 -.L105: +L(105): #ifdef movsd xorps %xmm4, %xmm4 #endif andq $3, M - jle .L998 + jle L(998) ALIGN_4 -.L106: +L(106): movsd (X), %xmm4 addps %xmm4, %xmm0 addq INCX, X decq M - jg .L106 + jg L(106) ALIGN_4 -.L998: +L(998): addps %xmm1, %xmm0 addps %xmm3, %xmm2 addps %xmm2, %xmm0 @@ -291,7 +291,7 @@ #endif ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/zsum_sse2.S b/kernel/x86_64/zsum_sse2.S index 6f667164dc..7f9ef34221 100644 --- a/kernel/x86_64/zsum_sse2.S +++ b/kernel/x86_64/zsum_sse2.S @@ -54,9 +54,9 @@ xorps %xmm0, %xmm0 testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) xorps %xmm1, %xmm1 xorps %xmm2, %xmm2 @@ -65,13 +65,13 @@ salq $ZBASE_SHIFT, INCX cmpq $2 * SIZE, INCX - jne .L40 + jne L(40) subq $-16 * SIZE, X addq M, M testq $SIZE, X - je .L05 + je L(05) #ifdef movsd xorps %xmm0, %xmm0 @@ -80,13 +80,13 @@ addq $SIZE, X subq $1, M - jle .L999 + jle L(999) ALIGN_3 -.L05: +L(05): movq M, I sarq $4, I - jle .L20 + jle L(20) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -99,10 +99,10 @@ movaps -2 * SIZE(X), %xmm11 decq I - jle .L11 + jle L(11) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -137,10 +137,10 @@ subq $-16 * SIZE, X decq I - jg .L10 + jg L(10) ALIGN_4 -.L11: +L(11): addpd %xmm4, %xmm0 addpd %xmm5, %xmm1 @@ -155,12 +155,12 @@ subq $-16 * SIZE, X ALIGN_3 -.L20: +L(20): andq $15, M - jle .L998 + jle L(998) testq $8, M - je .L21 + je L(21) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -174,9 +174,9 @@ addq $8 * SIZE, X ALIGN_3 -.L21: +L(21): testq $4, M - je .L22 + je L(22) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -187,34 +187,34 @@ addq $4 * SIZE, X ALIGN_3 -.L22: +L(22): testq $2, M - je .L23 + je L(23) movaps -16 * SIZE(X), %xmm6 addpd %xmm6, %xmm3 addq $2 * SIZE, X -.L23: +L(23): testq $1, M - je .L998 + je L(998) #ifdef movsd xorps %xmm4, %xmm4 #endif movsd -16 * SIZE(X), %xmm4 addsd %xmm4, %xmm0 - jmp .L998 + jmp L(998) ALIGN_3 -.L40: +L(40): movq M, I sarq $2, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) prefetcht0 PREFETCHSIZE * SIZE(X) #endif @@ -244,25 +244,25 @@ addpd %xmm7, %xmm3 decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): andq $3, M - jle .L998 + jle L(998) ALIGN_4 -.L61: +L(61): movsd 0 * SIZE(X), %xmm4 movhpd 1 * SIZE(X), %xmm4 addpd %xmm4, %xmm0 addq INCX, X decq M - jg .L61 + jg L(61) ALIGN_4 -.L998: +L(998): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -275,7 +275,7 @@ #endif ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/zswap.S b/kernel/x86_64/zswap.S index 68568f7ddd..b9ab1f4b9e 100644 --- a/kernel/x86_64/zswap.S +++ b/kernel/x86_64/zswap.S @@ -82,16 +82,16 @@ salq $ZBASE_SHIFT, INCY cmpq $2 * SIZE, INCX - jne .L14 + jne L(14) cmpq $2 * SIZE, INCY - jne .L14 + jne L(14) movq N, %rax sarq $2, %rax - jle .L15 + jle L(15) ALIGN_3 -.L16: +L(16): #ifdef XDOUBLE movq 0(X), %mm0 movq 8(X), %mm1 @@ -233,16 +233,16 @@ addq $8 * SIZE, X addq $8 * SIZE, Y decq %rax - jg .L16 + jg L(16) ALIGN_3 -.L15: +L(15): movq N, %rax andq $3, %rax - jle .L27 + jle L(27) ALIGN_3 -.L22: +L(22): #ifdef XDOUBLE movq 0(X), %mm0 movq 8(X), %mm1 @@ -279,21 +279,21 @@ addq $2 * SIZE, X addq $2 * SIZE, Y decq %rax - jg .L22 - jmp .L27 + jg L(22) + jmp L(27) ALIGN_3 /* INCX != 1 or INCY != 1 */ -.L14: +L(14): movq N, %rax movq X, XX movq Y, YY sarq $1, %rax - jle .L28 + jle L(28) ALIGN_2 -.L29: +L(29): #ifdef XDOUBLE movq 0(X), %mm0 movq 8(X), %mm1 @@ -388,16 +388,16 @@ addq INCY, YY #endif decq %rax - jg .L29 + jg L(29) ALIGN_3 -.L28: +L(28): movq N, %rax andq $1, %rax - jle .L27 + jle L(27) ALIGN_3 -.L35: +L(35): #ifdef XDOUBLE movq 0(X), %mm0 movq 8(X), %mm1 @@ -436,10 +436,10 @@ addq INCY, Y decq %rax - jg .L35 + jg L(35) ALIGN_3 -.L27: +L(27): EMMS xorq %rax,%rax diff --git a/kernel/x86_64/zswap_sse.S b/kernel/x86_64/zswap_sse.S index 12f9875ba8..720a676cd2 100644 --- a/kernel/x86_64/zswap_sse.S +++ b/kernel/x86_64/zswap_sse.S @@ -75,12 +75,12 @@ salq $ZBASE_SHIFT, INCY testq M, M - jle .L19 + jle L(19) cmpq $2 * SIZE, INCX - jne .L50 + jne L(50) cmpq $2 * SIZE, INCY - jne .L50 + jne L(50) addq M, M @@ -88,10 +88,10 @@ subq $-32 * SIZE, Y cmpq $3, M - jle .L16 + jle L(16) testq $SIZE, Y - je .L05 + je L(05) movss -32 * SIZE(X), %xmm0 movss -32 * SIZE(Y), %xmm1 @@ -104,9 +104,9 @@ decq M ALIGN_3 -.L05: +L(05): testq $2 * SIZE, Y - je .L10 + je L(10) movsd -32 * SIZE(X), %xmm0 movsd -32 * SIZE(Y), %xmm1 @@ -117,25 +117,25 @@ addq $2 * SIZE, X addq $2 * SIZE, Y subq $2, M - jle .L19 + jle L(19) ALIGN_3 -.L10: +L(10): cmpq $3, M - jle .L16 + jle L(16) testq $2 * SIZE, X - jne .L30 + jne L(30) testq $1 * SIZE, X - jne .L20 + jne L(20) movq M, %rax sarq $5, %rax - jle .L13 + jle L(13) ALIGN_3 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -204,12 +204,12 @@ subq $-32 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L13: +L(13): testq $16, M - jle .L14 + jle L(14) movaps -32 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -239,9 +239,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $8, M - jle .L15 + jle L(15) movaps -32 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -259,9 +259,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $4, M - jle .L16 + jle L(16) movaps -32 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -273,9 +273,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $2, M - jle .L17 + jle L(17) movsd -32 * SIZE(X), %xmm0 movsd -32 * SIZE(Y), %xmm1 @@ -286,9 +286,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $1, M - jle .L19 + jle L(19) movss -32 * SIZE(X), %xmm0 movss -32 * SIZE(Y), %xmm1 @@ -297,7 +297,7 @@ movss %xmm0, -32 * SIZE(Y) ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -309,7 +309,7 @@ ret ALIGN_3 -.L20: +L(20): movaps -33 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -321,10 +321,10 @@ movq M, %rax sarq $5, %rax - jle .L23 + jle L(23) ALIGN_4 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -425,12 +425,12 @@ subq $-32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L23: +L(23): testq $16, M - jle .L24 + jle L(24) movaps -29 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -476,9 +476,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $8, M - jle .L25 + jle L(25) movaps -29 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -504,9 +504,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $4, M - jle .L26 + jle L(26) movaps -29 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -525,7 +525,7 @@ addq $4 * SIZE, Y ALIGN_3 -.L26: +L(26): pshufd $0x39, %xmm0, %xmm2 pshufd $0xff, %xmm0, %xmm0 @@ -533,7 +533,7 @@ movss %xmm0, -30 * SIZE(Y) testq $2, M - jle .L27 + jle L(27) movsd -29 * SIZE(X), %xmm0 movsd -29 * SIZE(Y), %xmm1 @@ -545,9 +545,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $1, M - jle .L29 + jle L(29) movss -29 * SIZE(X), %xmm0 movss -29 * SIZE(Y), %xmm1 @@ -556,7 +556,7 @@ movss %xmm1, -29 * SIZE(X) ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -568,9 +568,9 @@ ret ALIGN_3 -.L30: +L(30): testq $1 * SIZE, X - jne .L40 + jne L(40) movhps -32 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -580,10 +580,10 @@ movq M, %rax sarq $5, %rax - jle .L33 + jle L(33) ALIGN_4 -.L31: +L(31): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -668,12 +668,12 @@ subq $-32 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L33: +L(33): testq $16, M - jle .L34 + jle L(34) movaps -30 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -711,9 +711,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L34: +L(34): testq $8, M - jle .L35 + jle L(35) movaps -30 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -735,9 +735,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L35: +L(35): testq $4, M - jle .L36 + jle L(36) movaps -30 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -754,11 +754,11 @@ addq $4 * SIZE, Y ALIGN_3 -.L36: +L(36): movhps %xmm0, -32 * SIZE(Y) testq $2, M - jle .L37 + jle L(37) movsd -30 * SIZE(X), %xmm0 movsd -30 * SIZE(Y), %xmm1 @@ -770,9 +770,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L37: +L(37): testq $1, M - jle .L39 + jle L(39) movss -30 * SIZE(X), %xmm0 movss -30 * SIZE(Y), %xmm1 @@ -781,7 +781,7 @@ movss %xmm1, -30 * SIZE(X) ALIGN_3 -.L39: +L(39): xorq %rax,%rax RESTOREREGISTERS @@ -793,7 +793,7 @@ ret ALIGN_3 -.L40: +L(40): movaps -35 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -803,10 +803,10 @@ movq M, %rax sarq $5, %rax - jle .L43 + jle L(43) ALIGN_4 -.L41: +L(41): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -907,12 +907,12 @@ subq $-32 * SIZE, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L43: +L(43): testq $16, M - jle .L44 + jle L(44) movaps -31 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -958,9 +958,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L44: +L(44): testq $8, M - jle .L45 + jle L(45) movaps -31 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -986,9 +986,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L45: +L(45): testq $4, M - jle .L46 + jle L(46) movaps -31 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -1007,7 +1007,7 @@ addq $4 * SIZE, Y ALIGN_3 -.L46: +L(46): movsd -31 * SIZE(X), %xmm2 pshufd $0x39, %xmm1, %xmm1 @@ -1022,7 +1022,7 @@ addq $3 * SIZE, Y testq $2, M - jle .L47 + jle L(47) movsd -32 * SIZE(X), %xmm0 movsd -32 * SIZE(Y), %xmm1 @@ -1034,9 +1034,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L47: +L(47): testq $1, M - jle .L49 + jle L(49) movss -32 * SIZE(X), %xmm0 movss -32 * SIZE(Y), %xmm1 @@ -1045,7 +1045,7 @@ movss %xmm1, -32 * SIZE(X) ALIGN_3 -.L49: +L(49): xorq %rax,%rax RESTOREREGISTERS @@ -1057,13 +1057,13 @@ ret ALIGN_3 -.L50: +L(50): movq M, %rax sarq $2, %rax - jle .L55 + jle L(55) ALIGN_3 -.L51: +L(51): movsd (X), %xmm0 movsd (Y), %xmm1 @@ -1097,16 +1097,16 @@ addq INCY, Y decq %rax - jg .L51 + jg L(51) ALIGN_3 -.L55: +L(55): movq M, %rax andq $3, %rax - jle .L57 + jle L(57) ALIGN_3 -.L56: +L(56): movsd (X), %xmm0 movsd (Y), %xmm1 @@ -1116,10 +1116,10 @@ addq INCY, Y decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L57: +L(57): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/zswap_sse2.S b/kernel/x86_64/zswap_sse2.S index c505014dd6..8072e4f21b 100644 --- a/kernel/x86_64/zswap_sse2.S +++ b/kernel/x86_64/zswap_sse2.S @@ -75,28 +75,28 @@ salq $ZBASE_SHIFT, INCY testq M, M - jle .L19 + jle L(19) cmpq $2 * SIZE, INCX - jne .L50 + jne L(50) cmpq $2 * SIZE, INCY - jne .L50 + jne L(50) subq $-16 * SIZE, X subq $-16 * SIZE, Y testq $SIZE, Y - jne .L30 + jne L(30) testq $SIZE, X - jne .L20 + jne L(20) movq M, %rax sarq $3, %rax - jle .L13 + jle L(13) ALIGN_3 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -165,12 +165,12 @@ subq $-16 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L13: +L(13): testq $4, M - jle .L14 + jle L(14) movaps -16 * SIZE(X), %xmm0 movaps -16 * SIZE(Y), %xmm1 @@ -200,9 +200,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $2, M - jle .L15 + jle L(15) movaps -16 * SIZE(X), %xmm0 movaps -16 * SIZE(Y), %xmm1 @@ -220,9 +220,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $1, M - jle .L19 + jle L(19) movaps -16 * SIZE(X), %xmm0 movaps -16 * SIZE(Y), %xmm1 @@ -234,7 +234,7 @@ addq $2 * SIZE, Y ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -245,20 +245,20 @@ ret ALIGN_3 -.L20: +L(20): movhps -16 * SIZE(X), %xmm0 movaps -16 * SIZE(Y), %xmm1 movlps %xmm1, -16 * SIZE(X) decq M - jle .L29 + jle L(29) movq M, %rax sarq $3, %rax - jle .L23 + jle L(23) ALIGN_4 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -343,12 +343,12 @@ subq $-16 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L23: +L(23): testq $4, M - jle .L24 + jle L(24) movaps -15 * SIZE(X), %xmm2 movaps -14 * SIZE(Y), %xmm3 @@ -386,9 +386,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $2, M - jle .L25 + jle L(25) movaps -15 * SIZE(X), %xmm2 movaps -14 * SIZE(Y), %xmm3 @@ -410,9 +410,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $1, M - jle .L29 + jle L(29) movaps -15 * SIZE(X), %xmm2 movaps -14 * SIZE(Y), %xmm3 @@ -429,7 +429,7 @@ addq $2 * SIZE, Y ALIGN_3 -.L29: +L(29): movaps -15 * SIZE(X), %xmm2 movhps %xmm1, -15 * SIZE(X) @@ -447,23 +447,23 @@ ret ALIGN_3 -.L30: +L(30): testq $SIZE, X - jne .L40 + jne L(40) movhps -16 * SIZE(Y), %xmm0 movaps -16 * SIZE(X), %xmm1 movlps %xmm1, -16 * SIZE(Y) decq M - jle .L39 + jle L(39) movq M, %rax sarq $3, %rax - jle .L33 + jle L(33) ALIGN_4 -.L31: +L(31): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -548,12 +548,12 @@ subq $-16 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L33: +L(33): testq $4, M - jle .L34 + jle L(34) movaps -15 * SIZE(Y), %xmm2 movaps -14 * SIZE(X), %xmm3 @@ -591,9 +591,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L34: +L(34): testq $2, M - jle .L35 + jle L(35) movaps -15 * SIZE(Y), %xmm2 movaps -14 * SIZE(X), %xmm3 @@ -615,9 +615,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L35: +L(35): testq $1, M - jle .L39 + jle L(39) movaps -15 * SIZE(Y), %xmm2 movaps -14 * SIZE(X), %xmm3 @@ -634,7 +634,7 @@ addq $2 * SIZE, Y ALIGN_3 -.L39: +L(39): movaps -15 * SIZE(Y), %xmm2 movhps %xmm1, -15 * SIZE(Y) @@ -652,7 +652,7 @@ ret ALIGN_3 -.L40: +L(40): movsd -16 * SIZE(X), %xmm0 movsd -16 * SIZE(Y), %xmm1 @@ -662,14 +662,14 @@ addq $SIZE, X addq $SIZE, Y decq M - jle .L49 + jle L(49) movq M, %rax sarq $3, %rax - jle .L43 + jle L(43) ALIGN_3 -.L41: +L(41): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -738,12 +738,12 @@ subq $-16 * SIZE, X decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L43: +L(43): testq $4, M - jle .L44 + jle L(44) movaps -16 * SIZE(X), %xmm0 movaps -16 * SIZE(Y), %xmm1 @@ -773,9 +773,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L44: +L(44): testq $2, M - jle .L45 + jle L(45) movaps -16 * SIZE(X), %xmm0 movaps -16 * SIZE(Y), %xmm1 @@ -793,9 +793,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L45: +L(45): testq $1, M - jle .L49 + jle L(49) movaps -16 * SIZE(X), %xmm0 movaps -16 * SIZE(Y), %xmm1 @@ -807,7 +807,7 @@ addq $2 * SIZE, Y ALIGN_3 -.L49: +L(49): movsd -16 * SIZE(X), %xmm0 movsd -16 * SIZE(Y), %xmm1 @@ -824,18 +824,18 @@ ret ALIGN_3 -.L50: +L(50): testq $SIZE, X - jne .L60 + jne L(60) testq $SIZE, Y - jne .L60 + jne L(60) movq M, %rax sarq $2, %rax - jle .L55 + jle L(55) ALIGN_3 -.L51: +L(51): movaps (X), %xmm0 movaps (Y), %xmm1 @@ -869,16 +869,16 @@ addq INCY, Y decq %rax - jg .L51 + jg L(51) ALIGN_3 -.L55: +L(55): movq M, %rax andq $3, %rax - jle .L57 + jle L(57) ALIGN_3 -.L56: +L(56): movaps (X), %xmm0 movaps (Y), %xmm1 @@ -888,10 +888,10 @@ addq INCY, Y decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L57: +L(57): xorq %rax, %rax RESTOREREGISTERS @@ -903,13 +903,13 @@ ret ALIGN_3 -.L60: +L(60): movq M, %rax sarq $2, %rax - jle .L65 + jle L(65) ALIGN_3 -.L61: +L(61): movsd 0 * SIZE(X), %xmm0 movhps 1 * SIZE(X), %xmm0 movsd 0 * SIZE(Y), %xmm1 @@ -959,16 +959,16 @@ addq INCY, Y decq %rax - jg .L61 + jg L(61) ALIGN_3 -.L65: +L(65): movq M, %rax andq $3, %rax - jle .L67 + jle L(67) ALIGN_3 -.L66: +L(66): movsd 0 * SIZE(X), %xmm0 movhps 1 * SIZE(X), %xmm0 movsd 0 * SIZE(Y), %xmm1 @@ -982,10 +982,10 @@ addq INCY, Y decq %rax - jg .L66 + jg L(66) ALIGN_3 -.L67: +L(67): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/zsymv_L_sse.S b/kernel/x86_64/zsymv_L_sse.S index df190c64c3..0199890eb6 100644 --- a/kernel/x86_64/zsymv_L_sse.S +++ b/kernel/x86_64/zsymv_L_sse.S @@ -188,7 +188,7 @@ salq $ZBASE_SHIFT, LDA testq M, M - jle .L999 + jle L(999) pcmpeqb %xmm2, %xmm2 xorpd %xmm3, %xmm3 @@ -203,10 +203,10 @@ movq M, %rax sarq $2, %rax - jle .L02 + jle L(02) ALIGN_3 -.L01: +L(01): MOVDDUP(0 * SIZE, X, %xmm3) MOVDDUP(1 * SIZE, X, %xmm4) addq INCX, X @@ -259,16 +259,16 @@ subq $-16 * SIZE, XX decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L02: +L(02): movq M, %rax andq $3, %rax - jle .L05 + jle L(05) ALIGN_3 -.L03: +L(03): MOVDDUP(0 * SIZE, X, %xmm3) MOVDDUP(1 * SIZE, X, %xmm4) addq INCX, X @@ -285,10 +285,10 @@ addq $4 * SIZE, XX decq %rax - jg .L03 + jg L(03) ALIGN_3 -.L05: +L(05): /* now we don't need original X */ movq Y, NEW_Y @@ -296,17 +296,17 @@ andq $-512, XX cmpq $2 * SIZE, INCY - je .L10 + je L(10) movq Y, YY movq XX, NEW_Y movq M, %rax sarq $2, %rax - jle .L07 + jle L(07) ALIGN_3 -.L06: +L(06): movsd 0 * SIZE(YY), %xmm0 movhpd 1 * SIZE(YY), %xmm0 addq INCY, YY @@ -327,16 +327,16 @@ addq $8 * SIZE, XX decq %rax - jg .L06 + jg L(06) ALIGN_3 -.L07: +L(07): movq M, %rax andq $3, %rax - jle .L10 + jle L(10) ALIGN_3 -.L08: +L(08): movsd 0 * SIZE(YY), %xmm0 movhpd 1 * SIZE(YY), %xmm0 addq INCY, YY @@ -345,17 +345,17 @@ addq $2 * SIZE, XX decq %rax - jg .L08 + jg L(08) ALIGN_3 -.L10: +L(10): xorq IS, IS # is = 0 cmpq $2, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): movq A, A1 leaq (A, LDA, 1), A2 leaq 4 * SIZE(A, LDA, 2), A @@ -421,10 +421,10 @@ subq IS, I subq $2, I sarq $2, I - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): movapd xtemp1, xt1 mulpd a1, xt1 mulpd atemp1, a1 @@ -579,15 +579,15 @@ addq $ 8 * SIZE, A2 decq I - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): movq M, I subq IS, I subq $2, I testq $2, I - jle .L16 + jle L(16) movapd xtemp1, xt1 mulpd a1, xt1 @@ -663,9 +663,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L16: +L(16): testq $1, M - jle .L18 + jle L(18) MOVDDUP(1 * SIZE, A1, a2) @@ -701,7 +701,7 @@ movhpd yy1, 1 * SIZE(YY) ALIGN_3 -.L18: +L(18): leaq (, IS, SIZE), I movsd 0 * SIZE(NEW_Y, I, 2), yy1 @@ -722,13 +722,13 @@ movq IS, I addq $2, I cmpq M, I - jle .L11 + jle L(11) ALIGN_3 -.L20: +L(20): HALT testq $1, N - jle .L990 + jle L(990) leaq (, IS, SIZE), I @@ -750,16 +750,16 @@ movhpd yy1, 1 * SIZE(NEW_Y, I, 2) ALIGN_3 -.L990: +L(990): cmpq $2 * SIZE, INCY - je .L999 + je L(999) movq M, %rax sarq $2, %rax - jle .L997 + jle L(997) ALIGN_3 -.L996: +L(996): movapd 0 * SIZE(NEW_Y), %xmm0 movapd 2 * SIZE(NEW_Y), %xmm1 movapd 4 * SIZE(NEW_Y), %xmm2 @@ -780,16 +780,16 @@ addq $8 * SIZE, NEW_Y decq %rax - jg .L996 + jg L(996) ALIGN_3 -.L997: +L(997): movq M, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_3 -.L998: +L(998): movapd 0 * SIZE(NEW_Y), %xmm0 movsd %xmm0, 0 * SIZE(Y) @@ -799,10 +799,10 @@ addq $2 * SIZE, NEW_Y decq %rax - jg .L998 + jg L(998) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zsymv_L_sse2.S b/kernel/x86_64/zsymv_L_sse2.S index acc5ad592b..dd2d4c0250 100644 --- a/kernel/x86_64/zsymv_L_sse2.S +++ b/kernel/x86_64/zsymv_L_sse2.S @@ -224,7 +224,7 @@ salq $ZBASE_SHIFT, LDA testq M, M - jle .L999 + jle L(999) pcmpeqb %xmm2, %xmm2 xorpd %xmm3, %xmm3 @@ -239,10 +239,10 @@ movq M, %rax sarq $2, %rax - jle .L02 + jle L(02) ALIGN_3 -.L01: +L(01): MOVDDUP(0 * SIZE, X, %xmm3) MOVDDUP(1 * SIZE, X, %xmm4) addq INCX, X @@ -295,16 +295,16 @@ subq $-16 * SIZE, XX decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L02: +L(02): movq M, %rax andq $3, %rax - jle .L05 + jle L(05) ALIGN_3 -.L03: +L(03): MOVDDUP(0 * SIZE, X, %xmm3) MOVDDUP(1 * SIZE, X, %xmm4) addq INCX, X @@ -321,10 +321,10 @@ addq $4 * SIZE, XX decq %rax - jg .L03 + jg L(03) ALIGN_3 -.L05: +L(05): /* now we don't need original X */ movq Y, NEW_Y @@ -332,17 +332,17 @@ andq $-512, XX cmpq $2 * SIZE, INCY - je .L10 + je L(10) movq Y, YY movq XX, NEW_Y movq M, %rax sarq $2, %rax - jle .L07 + jle L(07) ALIGN_3 -.L06: +L(06): movsd 0 * SIZE(YY), %xmm0 movhpd 1 * SIZE(YY), %xmm0 addq INCY, YY @@ -363,16 +363,16 @@ addq $8 * SIZE, XX decq %rax - jg .L06 + jg L(06) ALIGN_3 -.L07: +L(07): movq M, %rax andq $3, %rax - jle .L10 + jle L(10) ALIGN_3 -.L08: +L(08): movsd 0 * SIZE(YY), %xmm0 movhpd 1 * SIZE(YY), %xmm0 addq INCY, YY @@ -381,17 +381,17 @@ addq $2 * SIZE, XX decq %rax - jg .L08 + jg L(08) ALIGN_3 -.L10: +L(10): xorq IS, IS # is = 0 cmpq $2, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): movq A, A1 leaq (A, LDA, 1), A2 leaq 4 * SIZE(A, LDA, 2), A @@ -465,18 +465,18 @@ subq IS, I subq $2, I sarq $2, I - jle .L14 + jle L(14) MOVDDUP(6 * SIZE - (4 * SIZE), A2, a2) ALIGN_3 -.L12_prep: +L(12_prep): movsd 0 * SIZE(YY), yy1 movhpd 1 * SIZE(YY), yy1 movsd 2 * SIZE(YY), yy2 movhpd 3 * SIZE(YY), yy2 -.L12: +L(12): movapd xtemp1, xt1 mulpd a1, xt1 mulpd atemp1, a1 @@ -635,32 +635,32 @@ addq $ 8 * SIZE, A2 decq I - jg .L12_prep - jmp .L15 + jg L(12_prep) + jmp L(15) ALIGN_3 -.L14: +L(14): movq M, I subq IS, I subq $2, I testq $2, I - jle .L16 + jle L(16) MOVDDUP(6 * SIZE - (4 * SIZE), A2, a2) -.L15: +L(15): movq M, I subq IS, I subq $2, I testq $2, I - jle .L16 + jle L(16) movsd 0 * SIZE(YY), yy1 movhpd 1 * SIZE(YY), yy1 movsd 2 * SIZE(YY), yy2 movhpd 3 * SIZE(YY), yy2 -.L15_pastcheck: +L(15_pastcheck): movapd xtemp1, xt1 mulpd a1, xt1 mulpd atemp1, a1 @@ -731,9 +731,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L16: +L(16): testq $1, M - jle .L18 + jle L(18) MOVDDUP(1 * SIZE, A1, a2) @@ -772,7 +772,7 @@ movhpd yy1, 1 * SIZE(YY) ALIGN_3 -.L18: +L(18): leaq (, IS, SIZE), I movsd 0 * SIZE(NEW_Y, I, 2), yy1 @@ -793,12 +793,12 @@ movq IS, I addq $2, I cmpq N, I - jle .L11 + jle L(11) ALIGN_3 -.L20: +L(20): testq $1, N - jle .L990 + jle L(990) leaq (, IS, SIZE), I @@ -827,16 +827,16 @@ movhpd yy1, 1 * SIZE(NEW_Y, I, 2) ALIGN_3 -.L990: +L(990): cmpq $2 * SIZE, INCY - je .L999 + je L(999) movq M, %rax sarq $2, %rax - jle .L997 + jle L(997) ALIGN_3 -.L996: +L(996): movapd 0 * SIZE(NEW_Y), %xmm0 movapd 2 * SIZE(NEW_Y), %xmm1 movapd 4 * SIZE(NEW_Y), %xmm2 @@ -857,16 +857,16 @@ addq $8 * SIZE, NEW_Y decq %rax - jg .L996 + jg L(996) ALIGN_3 -.L997: +L(997): movq M, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_3 -.L998: +L(998): movapd 0 * SIZE(NEW_Y), %xmm0 movsd %xmm0, 0 * SIZE(Y) @@ -876,10 +876,10 @@ addq $2 * SIZE, NEW_Y decq %rax - jg .L998 + jg L(998) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zsymv_U_sse.S b/kernel/x86_64/zsymv_U_sse.S index 13176ce9c3..cf88ce2c5e 100644 --- a/kernel/x86_64/zsymv_U_sse.S +++ b/kernel/x86_64/zsymv_U_sse.S @@ -217,7 +217,7 @@ salq $ZBASE_SHIFT, LDA testq M, M - jle .L999 + jle L(999) negq IS addq M, IS @@ -243,10 +243,10 @@ movq M, %rax sarq $2, %rax - jle .L02 + jle L(02) ALIGN_3 -.L01: +L(01): movsd 0 * SIZE(X), %xmm4 addq INCX, X movhps 0 * SIZE(X), %xmm4 @@ -283,12 +283,12 @@ subq $-16 * SIZE, XX decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L02: +L(02): testq $2, M - jle .L03 + jle L(03) movsd 0 * SIZE(X), %xmm4 addq INCX, X @@ -312,9 +312,9 @@ subq $-8 * SIZE, XX ALIGN_3 -.L03: +L(03): testq $1, M - jle .L05 + jle L(05) movsd 0 * SIZE(X), %xmm4 addq INCX, X @@ -336,7 +336,7 @@ subq $-4 * SIZE, XX ALIGN_3 -.L05: +L(05): /* now we don't need original X */ movq Y, NEW_Y @@ -344,17 +344,17 @@ andq $-512, XX cmpq $2 * SIZE, INCY - je .L10 + je L(10) movq Y, YY movq XX, NEW_Y movq M, %rax sarq $2, %rax - jle .L07 + jle L(07) ALIGN_3 -.L06: +L(06): movsd 0 * SIZE(YY), %xmm0 addq INCY, YY movhps 0 * SIZE(YY), %xmm0 @@ -369,16 +369,16 @@ addq $8 * SIZE, XX decq %rax - jg .L06 + jg L(06) ALIGN_3 -.L07: +L(07): movq M, %rax andq $3, %rax - jle .L10 + jle L(10) ALIGN_3 -.L08: +L(08): movsd 0 * SIZE(YY), %xmm0 addq INCY, YY @@ -386,17 +386,17 @@ addq $2 * SIZE, XX decq %rax - jg .L08 + jg L(08) ALIGN_3 -.L10: +L(10): movq IS, I addq $2, I cmpq M, I - jg .L20 + jg L(20) ALIGN_3 -.L11: +L(11): movq A, A1 leaq (A, LDA, 1), A2 leaq (A, LDA, 2), A @@ -423,10 +423,10 @@ movq IS, I sarq $2, I - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): HALT subq $-16 * SIZE, XX @@ -435,12 +435,12 @@ addq $ 8 * SIZE, A2 decq I - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): testq $2, IS - jle .L18 + jle L(18) movsd 0 * SIZE(YY), yy1 movhps 2 * SIZE(YY), yy1 @@ -489,7 +489,7 @@ addq $4 * SIZE, A2 ALIGN_3 -.L18: +L(18): leaq (, IS, 4), I movaps 0 * SIZE(NEW_X, I, SIZE), atemp1 @@ -530,24 +530,24 @@ movq IS, I addq $2, I cmpq M, I - jle .L11 + jle L(11) ALIGN_3 -.L20: +L(20): testq $1, M - jle .L990 + jle L(990) -.L990: +L(990): cmpq $2 * SIZE, INCY - je .L999 + je L(999) movq M, %rax sarq $2, %rax - jle .L997 + jle L(997) ALIGN_3 -.L996: +L(996): movaps 0 * SIZE(NEW_Y), %xmm0 movaps 4 * SIZE(NEW_Y), %xmm1 @@ -562,16 +562,16 @@ addq $8 * SIZE, NEW_Y decq %rax - jg .L996 + jg L(996) ALIGN_3 -.L997: +L(997): movq M, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_3 -.L998: +L(998): movlps 0 * SIZE(NEW_Y), %xmm0 addq $2 * SIZE, NEW_Y @@ -579,10 +579,10 @@ addq INCY, Y decq %rax - jg .L998 + jg L(998) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zsymv_U_sse2.S b/kernel/x86_64/zsymv_U_sse2.S index fa1fe9fe9e..6743d81ce3 100644 --- a/kernel/x86_64/zsymv_U_sse2.S +++ b/kernel/x86_64/zsymv_U_sse2.S @@ -223,7 +223,7 @@ salq $ZBASE_SHIFT, LDA testq M, M - jle .L999 + jle L(999) negq IS addq M, IS @@ -245,10 +245,10 @@ movq M, %rax sarq $2, %rax - jle .L02 + jle L(02) ALIGN_3 -.L01: +L(01): MOVDDUP(0 * SIZE, X, %xmm3) MOVDDUP(1 * SIZE, X, %xmm4) addq INCX, X @@ -301,16 +301,16 @@ subq $-16 * SIZE, XX decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L02: +L(02): movq M, %rax andq $3, %rax - jle .L05 + jle L(05) ALIGN_3 -.L03: +L(03): MOVDDUP(0 * SIZE, X, %xmm3) MOVDDUP(1 * SIZE, X, %xmm4) addq INCX, X @@ -327,10 +327,10 @@ addq $4 * SIZE, XX decq %rax - jg .L03 + jg L(03) ALIGN_3 -.L05: +L(05): /* now we don't need original X */ movq Y, NEW_Y @@ -338,17 +338,17 @@ andq $-512, XX cmpq $2 * SIZE, INCY - je .L10 + je L(10) movq Y, YY movq XX, NEW_Y movq M, %rax sarq $2, %rax - jle .L07 + jle L(07) ALIGN_3 -.L06: +L(06): movsd 0 * SIZE(YY), %xmm0 movhpd 1 * SIZE(YY), %xmm0 addq INCY, YY @@ -369,16 +369,16 @@ addq $8 * SIZE, XX decq %rax - jg .L06 + jg L(06) ALIGN_3 -.L07: +L(07): movq M, %rax andq $3, %rax - jle .L10 + jle L(10) ALIGN_3 -.L08: +L(08): movsd 0 * SIZE(YY), %xmm0 movhpd 1 * SIZE(YY), %xmm0 addq INCY, YY @@ -387,17 +387,17 @@ addq $2 * SIZE, XX decq %rax - jg .L08 + jg L(08) ALIGN_3 -.L10: +L(10): movq IS, I addq $2, I cmpq M, I - jg .L20 + jg L(20) ALIGN_3 -.L11: +L(11): movq A, A1 leaq (A, LDA, 1), A2 leaq (A, LDA, 2), A @@ -431,10 +431,10 @@ movq IS, I sarq $2, I - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): movapd xtemp1, xt1 mulpd a1, xt1 mulpd atemp1, a1 @@ -597,12 +597,12 @@ addq $ 8 * SIZE, A2 decq I - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): testq $2, IS - jle .L18 + jle L(18) movapd xtemp1, xt1 mulpd a1, xt1 @@ -673,7 +673,7 @@ addq $4 * SIZE, A2 ALIGN_3 -.L18: +L(18): MOVDDUP(0 * SIZE, A1, a1) MOVDDUP(0 * SIZE, A2, a2) @@ -733,12 +733,12 @@ movq IS, I addq $2, I cmpq M, I - jle .L11 + jle L(11) ALIGN_3 -.L20: +L(20): testq $1, M - jle .L990 + jle L(990) movq A, A1 leaq (, IS, 4), I @@ -767,10 +767,10 @@ movq IS, I sarq $1, I - jle .L28 + jle L(28) ALIGN_3 -.L22: +L(22): movapd xtemp1, xt1 movapd 8 * SIZE(XX), xtemp1 mulpd a1, xt1 @@ -818,10 +818,10 @@ addq $4 * SIZE, A1 decq I - jg .L22 + jg L(22) ALIGN_3 -.L28: +L(28): MOVDDUP(0 * SIZE, A1, a1) #ifndef HEMV @@ -844,16 +844,16 @@ movhpd yy1, 1 * SIZE(YY) ALIGN_3 -.L990: +L(990): cmpq $2 * SIZE, INCY - je .L999 + je L(999) movq M, %rax sarq $2, %rax - jle .L997 + jle L(997) ALIGN_3 -.L996: +L(996): movapd 0 * SIZE(NEW_Y), %xmm0 movapd 2 * SIZE(NEW_Y), %xmm1 movapd 4 * SIZE(NEW_Y), %xmm2 @@ -874,16 +874,16 @@ addq $8 * SIZE, NEW_Y decq %rax - jg .L996 + jg L(996) ALIGN_3 -.L997: +L(997): movq M, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_3 -.L998: +L(998): movapd 0 * SIZE(NEW_Y), %xmm0 movsd %xmm0, 0 * SIZE(Y) @@ -893,10 +893,10 @@ addq $2 * SIZE, NEW_Y decq %rax - jg .L998 + jg L(998) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_LN_2x1_atom.S b/kernel/x86_64/ztrsm_kernel_LN_2x1_atom.S index d3bedff12c..3da8f1d8dc 100644 --- a/kernel/x86_64/ztrsm_kernel_LN_2x1_atom.S +++ b/kernel/x86_64/ztrsm_kernel_LN_2x1_atom.S @@ -177,10 +177,10 @@ movq N, J testq N, N - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -213,7 +213,7 @@ leaq (B, %rax), BB testq $1, M - jle .L20 + jle L(20) #ifdef LN movq K, %rax @@ -253,10 +253,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADDSD2 %xmm2, %xmm9 movaps %xmm0, %xmm2 @@ -334,10 +334,10 @@ addq $8 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -347,10 +347,10 @@ andq $3, %rax BRANCH BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): ADDSD2 %xmm2, %xmm9 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -373,10 +373,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): ADDSD2 %xmm2, %xmm9 ADDSD4 %xmm6, %xmm11 @@ -481,13 +481,13 @@ #endif ALIGN_4 -.L20: +L(20): movq M, I sarq $1, I - jle .L99 + jle L(99) ALIGN_4 -.L10: +L(10): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -535,10 +535,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): ADDSD2 %xmm2, %xmm13 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps %xmm0, %xmm2 @@ -681,10 +681,10 @@ mulsd %xmm3, %xmm6 movsd 1 * SIZE(BO), %xmm3 - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -694,10 +694,10 @@ andq $3, %rax BRANCH BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): ADDSD2 %xmm2, %xmm13 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -736,10 +736,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): ADDSD2 %xmm2, %xmm13 ADDSD3 %xmm7, %xmm14 ADDSD4 %xmm6, %xmm15 @@ -942,10 +942,10 @@ addq %rax, AORIG #endif decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -963,10 +963,10 @@ subq $1, KK #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_LN_2x2_core2.S b/kernel/x86_64/ztrsm_kernel_LN_2x2_core2.S index 80485c0def..dc7f2a9eb5 100644 --- a/kernel/x86_64/ztrsm_kernel_LN_2x2_core2.S +++ b/kernel/x86_64/ztrsm_kernel_LN_2x2_core2.S @@ -196,10 +196,10 @@ movq N, J sarq $1, J # j = (n >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L01: +L(01): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -234,12 +234,12 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): prefetcht0 (PREFETCH_R + 0) * SIZE(B) movddup -16 * SIZE(B), %xmm8 @@ -268,10 +268,10 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -280,10 +280,10 @@ #endif andq $3, %rax BRANCH - jle .L05 + jle L(05) ALIGN_4 -.L04: +L(04): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 movddup -14 * SIZE(B), %xmm10 @@ -298,10 +298,10 @@ addq $ 8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L05: +L(05): #if defined(LT) || defined(RN) movq A, AO #else @@ -321,7 +321,7 @@ #endif testq $1, M - jle .L30 + jle L(30) #ifdef LN movq K, %rax @@ -356,9 +356,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L42 + je L(42) -.L41: +L(41): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -428,9 +428,9 @@ subq $ -8 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L41 + jne L(41) -.L42: +L(42): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -441,9 +441,9 @@ andq $3, %rax # if (k & 1) BRANCH - jle .L44 + jle L(44) -.L43: +L(43): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -463,10 +463,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -681,13 +681,13 @@ #endif ALIGN_4 -.L30: +L(30): movq M, I sarq $1, I # i = (m >> 2) - jle .L99 + jle L(99) ALIGN_4 -.L10: +L(10): leaq (PREFETCH_R + 0) * SIZE(B), BB #ifdef LN @@ -749,10 +749,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -873,10 +873,10 @@ subq $1, %rax BRANCH BRANCH - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -888,10 +888,10 @@ andq $3, %rax BRANCH BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): ADD1 %xmm2, %xmm10 ADD1 %xmm3, %xmm14 ADD2 %xmm4, %xmm11 @@ -928,10 +928,10 @@ addq $8 * SIZE, BO subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): ADD1 %xmm2, %xmm10 ADD1 %xmm3, %xmm14 ADD2 %xmm4, %xmm11 @@ -1309,10 +1309,10 @@ #endif decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1334,13 +1334,13 @@ #endif decq J # j -- - jg .L01 + jg L(01) -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) -.L101: +L(101): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -1375,10 +1375,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 movddup -14 * SIZE(B), %xmm10 @@ -1400,10 +1400,10 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1412,10 +1412,10 @@ #endif andq $3, %rax BRANCH - jle .L105 + jle L(105) ALIGN_4 -.L104: +L(104): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 @@ -1425,10 +1425,10 @@ addq $4 * SIZE, BO addq $2 * SIZE, B decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq A, AO #else @@ -1445,10 +1445,10 @@ #endif testq $1, M - jle .L130 + jle L(130) ALIGN_4 -.L140: +L(140): #ifdef LN movq K, %rax salq $0 + ZBASE_SHIFT, %rax @@ -1482,9 +1482,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L142 + je L(142) -.L141: +L(141): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1524,9 +1524,9 @@ subq $ -8 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L141 + jne L(141) -.L142: +L(142): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1537,9 +1537,9 @@ andq $3, %rax # if (k & 1) BRANCH - jle .L144 + jle L(144) -.L143: +L(143): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -1553,10 +1553,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L143 + jg L(143) ALIGN_4 -.L144: +L(144): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1714,13 +1714,13 @@ #endif ALIGN_4 -.L130: +L(130): movq M, I sarq $1, I # i = (m >> 2) - jle .L199 + jle L(199) ALIGN_4 -.L110: +L(110): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1755,9 +1755,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L112 + je L(112) -.L111: +L(111): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1835,10 +1835,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1848,9 +1848,9 @@ movapd POSINV, %xmm7 andq $3, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): movapd -16 * SIZE(AO), %xmm0 movapd -14 * SIZE(AO), %xmm1 @@ -1872,10 +1872,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2105,10 +2105,10 @@ #endif decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L199: +L(199): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2131,7 +2131,7 @@ ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/ztrsm_kernel_LN_2x2_penryn.S b/kernel/x86_64/ztrsm_kernel_LN_2x2_penryn.S index 3a691ca8cc..ad4cc0ad57 100644 --- a/kernel/x86_64/ztrsm_kernel_LN_2x2_penryn.S +++ b/kernel/x86_64/ztrsm_kernel_LN_2x2_penryn.S @@ -171,10 +171,10 @@ movq N, J sarq $1, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -211,7 +211,7 @@ testq $1, M BRANCH - jle .L20 + jle L(20) ALIGN_4 #ifdef LN @@ -250,10 +250,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_4 -.L22: +L(22): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -319,10 +319,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -331,10 +331,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -355,10 +355,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -556,14 +556,14 @@ #endif ALIGN_4 -.L20: +L(20): movq M, I sarq $1, I NOBRANCH - jle .L39 + jle L(39) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -619,10 +619,10 @@ #endif sarq $3, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 @@ -868,10 +868,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -880,10 +880,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 @@ -919,10 +919,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax subq $2, %rax @@ -1266,10 +1266,10 @@ decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1289,13 +1289,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(LT) || defined(RN) movq A, AO @@ -1332,7 +1332,7 @@ testq $1, M BRANCH - jle .L60 + jle L(60) ALIGN_4 #ifdef LN @@ -1366,10 +1366,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x4e, %xmm2, %xmm7 @@ -1412,10 +1412,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1424,10 +1424,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm7 @@ -1442,10 +1442,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -1588,14 +1588,14 @@ #endif ALIGN_4 -.L60: +L(60): movq M, I sarq $1, I # i = (m >> 2) NOBRANCH - jle .L79 + jle L(79) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1633,10 +1633,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_4 -.L52: +L(52): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -1711,10 +1711,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1723,10 +1723,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -1749,10 +1749,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1965,10 +1965,10 @@ #endif decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -1987,7 +1987,7 @@ #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_LN_2x2_sse2.S b/kernel/x86_64/ztrsm_kernel_LN_2x2_sse2.S index 542bd59474..df3bd43a63 100644 --- a/kernel/x86_64/ztrsm_kernel_LN_2x2_sse2.S +++ b/kernel/x86_64/ztrsm_kernel_LN_2x2_sse2.S @@ -319,10 +319,10 @@ movq N, J sarq $1, J # j = (n >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L01: +L(01): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -358,12 +358,12 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): PREFETCHNTA 56 * SIZE(B) movlpd 0 * SIZE(B), %xmm0 @@ -395,10 +395,10 @@ subq $-16 * SIZE, BO addq $ 8 * SIZE, B decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -407,10 +407,10 @@ #endif andq $3, %rax BRANCH - jle .L05 + jle L(05) ALIGN_4 -.L04: +L(04): movlpd 0 * SIZE(B), %xmm0 movlpd 1 * SIZE(B), %xmm1 movlpd 2 * SIZE(B), %xmm2 @@ -429,10 +429,10 @@ addq $ 8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L05: +L(05): #if defined(LT) || defined(RN) movq A, AO #else @@ -452,7 +452,7 @@ #endif testq $1, M - jle .L30 + jle L(30) #ifdef LN movq K, %rax @@ -487,9 +487,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L42 + je L(42) -.L41: +L(41): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 @@ -565,9 +565,9 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L41 + jne L(41) -.L42: +L(42): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -577,9 +577,9 @@ movapd POSINV, %xmm15 andq $3, %rax # if (k & 1) BRANCH - jle .L44 + jle L(44) -.L43: +L(43): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 @@ -602,10 +602,10 @@ addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -833,13 +833,13 @@ #endif ALIGN_4 -.L30: +L(30): movq M, I sarq $1, I # i = (m >> 2) - jle .L99 + jle L(99) ALIGN_4 -.L10: +L(10): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -897,8 +897,8 @@ #endif andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -916,7 +916,7 @@ KERNEL7(16 * 1) KERNEL8(16 * 1) cmpq $64 * 2, %rax - jle .L12 + jle L(12) KERNEL1(16 * 2) KERNEL2(16 * 2) KERNEL3(16 * 2) @@ -934,7 +934,7 @@ KERNEL7(16 * 3) KERNEL8(16 * 3) cmpq $64 * 4, %rax - jle .L12 + jle L(12) KERNEL1(16 * 4) KERNEL2(16 * 4) KERNEL3(16 * 4) @@ -952,7 +952,7 @@ KERNEL7(16 * 5) KERNEL8(16 * 5) cmpq $64 * 6, %rax - jle .L12 + jle L(12) KERNEL1(16 * 6) KERNEL2(16 * 6) KERNEL3(16 * 6) @@ -973,14 +973,14 @@ addq $16 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $64 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 4), BO # * 64 ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -990,10 +990,10 @@ movapd POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -1022,10 +1022,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1421,11 +1421,11 @@ #endif decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1447,13 +1447,13 @@ #endif decq J # j -- - jg .L01 + jg L(01) -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) -.L101: +L(101): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -1489,10 +1489,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movlpd 0 * SIZE(B), %xmm0 movlpd 1 * SIZE(B), %xmm1 movlpd 2 * SIZE(B), %xmm2 @@ -1522,10 +1522,10 @@ subq $-16 * SIZE, BO addq $ 8 * SIZE, B decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1534,10 +1534,10 @@ #endif andq $3, %rax BRANCH - jle .L105 + jle L(105) ALIGN_4 -.L104: +L(104): movlpd 0 * SIZE(B), %xmm0 movlpd 1 * SIZE(B), %xmm1 @@ -1549,10 +1549,10 @@ addq $4 * SIZE, BO addq $2 * SIZE, B decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq A, AO #else @@ -1569,10 +1569,10 @@ #endif testq $1, M - jle .L130 + jle L(130) ALIGN_4 -.L140: +L(140): #ifdef LN movq K, %rax salq $0 + ZBASE_SHIFT, %rax @@ -1606,9 +1606,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L142 + je L(142) -.L141: +L(141): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -1640,9 +1640,9 @@ addq $8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L141 + jne L(141) -.L142: +L(142): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -1656,9 +1656,9 @@ #endif andq $3, %rax # if (k & 1) BRANCH - jle .L144 + jle L(144) -.L143: +L(143): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -1669,10 +1669,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L143 + jg L(143) ALIGN_4 -.L144: +L(144): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1834,13 +1834,13 @@ #endif ALIGN_4 -.L130: +L(130): movq M, I sarq $1, I # i = (m >> 2) - jle .L199 + jle L(199) ALIGN_4 -.L110: +L(110): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1880,9 +1880,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L112 + je L(112) -.L111: +L(111): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -1942,10 +1942,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1955,9 +1955,9 @@ movapd POSINV, %xmm15 andq $3, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -1975,10 +1975,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2223,10 +2223,10 @@ #endif decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L199: +L(199): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2248,7 +2248,7 @@ #endif ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/ztrsm_kernel_LN_2x2_sse3.S b/kernel/x86_64/ztrsm_kernel_LN_2x2_sse3.S index 7547421efa..063d01cefd 100644 --- a/kernel/x86_64/ztrsm_kernel_LN_2x2_sse3.S +++ b/kernel/x86_64/ztrsm_kernel_LN_2x2_sse3.S @@ -408,10 +408,10 @@ movq N, J sarq $1, J # j = (n >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -443,7 +443,7 @@ #endif testq $1, M - jle .L30 + jle L(30) #ifdef LN movq K, %rax @@ -477,9 +477,9 @@ subq KK, %rax #endif sarq $3, %rax - je .L42 + je L(42) -.L41: +L(41): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -589,9 +589,9 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L41 + jne L(41) -.L42: +L(42): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -600,9 +600,9 @@ #endif andq $7, %rax # if (k & 1) BRANCH - jle .L44 + jle L(44) -.L43: +L(43): mulpd %xmm8, %xmm9 ADD1 %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -620,10 +620,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm3, %xmm3 @@ -863,13 +863,13 @@ #endif ALIGN_4 -.L30: +L(30): movq M, I sarq $1, I # i = (m >> 2) - jle .L99 + jle L(99) ALIGN_4 -.L10: +L(10): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -920,9 +920,9 @@ #endif andq $-8, %rax salq $4, %rax - je .L12 + je L(12) -.L1X: +L(1X): KERNEL1 (16 * 0) KERNEL2 (16 * 0) KERNEL3 (16 * 0) @@ -941,7 +941,7 @@ KERNEL16(16 * 0) cmpq $128 * 1, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 1) KERNEL2 (16 * 1) KERNEL3 (16 * 1) @@ -960,7 +960,7 @@ KERNEL16(16 * 1) cmpq $128 * 2, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 2) KERNEL2 (16 * 2) KERNEL3 (16 * 2) @@ -979,7 +979,7 @@ KERNEL16(16 * 2) cmpq $128 * 3, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 3) KERNEL2 (16 * 3) KERNEL3 (16 * 3) @@ -998,7 +998,7 @@ KERNEL16(16 * 3) cmpq $128 * 4, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 4) KERNEL2 (16 * 4) KERNEL3 (16 * 4) @@ -1017,7 +1017,7 @@ KERNEL16(16 * 4) cmpq $128 * 5, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 5) KERNEL2 (16 * 5) KERNEL3 (16 * 5) @@ -1036,7 +1036,7 @@ KERNEL16(16 * 5) cmpq $128 * 6, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 6) KERNEL2 (16 * 6) KERNEL3 (16 * 6) @@ -1055,7 +1055,7 @@ KERNEL16(16 * 6) cmpq $128 * 7, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 7) KERNEL2 (16 * 7) KERNEL3 (16 * 7) @@ -1076,14 +1076,14 @@ addq $32 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $128 * 8, %rax - jg .L1X + jg L(1X) -.L11: +L(11): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 2), BO # * 64 ALIGN_4 -.L12: +L(12): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1092,10 +1092,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L14 + je L(14) ALIGN_4 -.L13: +L(13): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 ADD1 %xmm9, %xmm0 @@ -1126,10 +1126,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm3, %xmm3 SHUFPD_1 %xmm5, %xmm5 @@ -1402,10 +1402,10 @@ addq %rax, AORIG #endif decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1423,13 +1423,13 @@ subq $2, KK #endif decq J # j -- - jg .L01 + jg L(01) -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) -.L101: +L(101): #if defined(LT) || defined(RN) movq A, AO #else @@ -1457,7 +1457,7 @@ #endif testq $1, M - jle .L130 + jle L(130) ALIGN_4 #ifdef LN @@ -1492,9 +1492,9 @@ subq KK, %rax #endif sarq $3, %rax - je .L142 + je L(142) -.L141: +L(141): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -1556,9 +1556,9 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L141 + jne L(141) -.L142: +L(142): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1567,9 +1567,9 @@ #endif andq $7, %rax # if (k & 1) BRANCH - jle .L144 + jle L(144) -.L143: +L(143): mulpd %xmm8, %xmm9 ADD1 %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -1581,10 +1581,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L143 + jg L(143) ALIGN_4 -.L144: +L(144): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -1735,13 +1735,13 @@ #endif ALIGN_4 -.L130: +L(130): movq M, I sarq $1, I # i = (m >> 2) - jle .L149 + jle L(149) ALIGN_4 -.L110: +L(110): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1780,9 +1780,9 @@ subq KK, %rax #endif sarq $3, %rax - je .L112 + je L(112) -.L111: +L(111): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -1901,10 +1901,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1913,9 +1913,9 @@ #endif andq $7, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 ADD1 %xmm9, %xmm0 @@ -1934,10 +1934,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm5, %xmm5 @@ -2152,10 +2152,10 @@ #endif decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L149: +L(149): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2174,7 +2174,7 @@ #endif ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_LN_2x4_nehalem.S b/kernel/x86_64/ztrsm_kernel_LN_2x4_nehalem.S index 5d931cea85..2e15200dfc 100644 --- a/kernel/x86_64/ztrsm_kernel_LN_2x4_nehalem.S +++ b/kernel/x86_64/ztrsm_kernel_LN_2x4_nehalem.S @@ -171,10 +171,10 @@ movq N, J sarq $2, J NOBRANCH - jle .L30 + jle L(30) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -211,7 +211,7 @@ testq $1, M BRANCH - jle .L20 + jle L(20) #ifdef LN movq K, %rax @@ -251,10 +251,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -324,10 +324,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -336,10 +336,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -361,10 +361,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -782,14 +782,14 @@ #endif ALIGN_4 -.L20: +L(20): movq M, I sarq $1, I NOBRANCH - jle .L29 + jle L(29) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -839,10 +839,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm12 @@ -969,10 +969,10 @@ subq $-16 * SIZE, AO subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -981,10 +981,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm1, %xmm12 movaps -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm13 @@ -1020,10 +1020,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1584,10 +1584,10 @@ decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L29: +L(29): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -1608,13 +1608,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L30: +L(30): testq $2, N BRANCH - jle .L50 + jle L(50) #if defined(LT) || defined(RN) movq A, AO @@ -1648,7 +1648,7 @@ testq $1, M BRANCH - jle .L40 + jle L(40) #ifdef LN movq K, %rax @@ -1688,10 +1688,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1733,10 +1733,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1745,10 +1745,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1763,10 +1763,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1980,14 +1980,14 @@ #endif ALIGN_4 -.L40: +L(40): movq M, I sarq $1, I NOBRANCH - jle .L49 + jle L(49) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -2027,10 +2027,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -2094,10 +2094,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2106,10 +2106,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): ADD1 %xmm1, %xmm8 movaps -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -2129,10 +2129,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2443,10 +2443,10 @@ decq I # i -- BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L49: +L(49): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -2466,10 +2466,10 @@ #endif ALIGN_4 -.L50: +L(50): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(LT) || defined(RN) movq A, AO @@ -2501,7 +2501,7 @@ testq $1, M BRANCH - jle .L60 + jle L(60) #ifdef LN movq K, %rax @@ -2536,10 +2536,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2581,10 +2581,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2593,10 +2593,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2611,10 +2611,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -2730,14 +2730,14 @@ #endif ALIGN_4 -.L60: +L(60): movq M, I sarq $1, I NOBRANCH - jle .L69 + jle L(69) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -2776,10 +2776,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -2819,10 +2819,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2831,10 +2831,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): ADD1 %xmm1, %xmm8 movddup -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -2848,10 +2848,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3064,10 +3064,10 @@ decq I # i -- BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L69: +L(69): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -3087,7 +3087,7 @@ #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_LN_4x2_sse.S b/kernel/x86_64/ztrsm_kernel_LN_4x2_sse.S index 02b5098a23..45478766bb 100644 --- a/kernel/x86_64/ztrsm_kernel_LN_4x2_sse.S +++ b/kernel/x86_64/ztrsm_kernel_LN_4x2_sse.S @@ -332,10 +332,10 @@ movq N, J sarq $1, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -370,10 +370,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 @@ -423,10 +423,10 @@ addq $16 * SIZE, B addq $64 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -435,10 +435,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movaps 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -454,10 +454,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -477,7 +477,7 @@ #endif testq $1, M - je .L20 + je L(20) #ifdef LN movq K, %rax @@ -522,10 +522,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -643,10 +643,10 @@ addq $128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -656,10 +656,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -678,10 +678,10 @@ addq $ 2 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm3, %xmm3 @@ -956,9 +956,9 @@ #endif ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) #ifdef LN movq K, %rax @@ -1003,10 +1003,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -1121,10 +1121,10 @@ addq $128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1134,10 +1134,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1154,10 +1154,10 @@ addq $ 4 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm3, %xmm3 @@ -1516,13 +1516,13 @@ #endif ALIGN_4 -.L30: +L(30): movq M, I sarq $2, I # i = (m >> 2) - jle .L39 + jle L(39) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + ZBASE_SHIFT, %rax @@ -1573,8 +1573,8 @@ #endif andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -1595,14 +1595,14 @@ addq $32 * 2 * SIZE, AO addq $64 * 2 * SIZE, BO subq $64 * 2, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO leaq (BO, %rax, 4), BO ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1612,10 +1612,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1644,10 +1644,10 @@ addq $ 8 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm3, %xmm3 shufps $0xb1, %xmm5, %xmm5 @@ -2365,10 +2365,10 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2 * COMPSIZE), B @@ -2390,12 +2390,12 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $1, N - je .L999 + je L(999) ALIGN_4 #ifdef LN @@ -2432,10 +2432,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 @@ -2462,10 +2462,10 @@ addq $ 8 * SIZE, B addq $32 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2474,10 +2474,10 @@ #endif andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movsd 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -2489,10 +2489,10 @@ addq $2 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): #if defined(LT) || defined(RN) movq A, AO #else @@ -2510,7 +2510,7 @@ #endif testq $1, M - je .L60 + je L(60) #ifdef LN movq K, %rax @@ -2555,10 +2555,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -2631,10 +2631,10 @@ addq $64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2644,10 +2644,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -2659,10 +2659,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -2806,9 +2806,9 @@ #endif ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) #ifdef LN movq K, %rax @@ -2851,10 +2851,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2918,10 +2918,10 @@ addq $64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2931,10 +2931,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -2945,10 +2945,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -3196,13 +3196,13 @@ #endif ALIGN_4 -.L70: +L(70): movq M, I sarq $2, I # i = (m >> 2) - jle .L79 + jle L(79) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + ZBASE_SHIFT, %rax @@ -3253,10 +3253,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3378,10 +3378,10 @@ decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3391,10 +3391,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -3411,10 +3411,10 @@ addq $ 8 * SIZE, AO # aoffset += 4 addq $ 8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm5, %xmm5 @@ -3957,10 +3957,10 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, COMPSIZE), B @@ -3982,7 +3982,7 @@ #endif ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/ztrsm_kernel_LT_1x4_nehalem.S b/kernel/x86_64/ztrsm_kernel_LT_1x4_nehalem.S index 874d34db34..6756395456 100644 --- a/kernel/x86_64/ztrsm_kernel_LT_1x4_nehalem.S +++ b/kernel/x86_64/ztrsm_kernel_LT_1x4_nehalem.S @@ -168,15 +168,15 @@ #endif testq M, M - jle .L999 + jle L(999) movq N, J sarq $2, J NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -214,7 +214,7 @@ movq M, I ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -264,10 +264,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm12 @@ -391,10 +391,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -403,10 +403,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm1, %xmm12 movaps -16 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm13 @@ -442,10 +442,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -819,7 +819,7 @@ decq I BRANCH - jg .L11 + jg L(11) #ifdef LN movq K, %rax @@ -841,13 +841,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L20: +L(20): testq $2, N BRANCH - jle .L30 + jle L(30) #if defined(LT) || defined(RN) movq A, AO @@ -882,7 +882,7 @@ movq M, I ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -922,10 +922,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -997,10 +997,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1009,10 +1009,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): ADD1 %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -1034,10 +1034,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1239,7 +1239,7 @@ decq I BRANCH - jg .L21 + jg L(21) #ifdef LN movq K, %rax @@ -1260,10 +1260,10 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(LT) || defined(RN) movq A, AO @@ -1295,7 +1295,7 @@ movq M, I ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -1332,10 +1332,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -1375,13 +1375,13 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 ALIGN_3 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1390,10 +1390,10 @@ #endif andq $3, %rax BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): ADD1 %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -1407,10 +1407,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -1536,7 +1536,7 @@ #endif decq I BRANCH - jg .L31 + jg L(31) #ifdef LN movq K, %rax @@ -1557,7 +1557,7 @@ #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_LT_2x1_atom.S b/kernel/x86_64/ztrsm_kernel_LT_2x1_atom.S index 92dc6366b9..90f80829d1 100644 --- a/kernel/x86_64/ztrsm_kernel_LT_2x1_atom.S +++ b/kernel/x86_64/ztrsm_kernel_LT_2x1_atom.S @@ -177,10 +177,10 @@ movq N, J testq N, N - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -214,10 +214,10 @@ movq M, I sarq $1, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -265,10 +265,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): ADDSD2 %xmm2, %xmm13 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps %xmm0, %xmm2 @@ -411,10 +411,10 @@ mulsd %xmm3, %xmm6 movsd 1 * SIZE(BO), %xmm3 - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -424,10 +424,10 @@ andq $3, %rax BRANCH BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): ADDSD2 %xmm2, %xmm13 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -466,10 +466,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): ADDSD2 %xmm2, %xmm13 ADDSD3 %xmm7, %xmm14 ADDSD4 %xmm6, %xmm15 @@ -672,12 +672,12 @@ addq %rax, AORIG #endif decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): testq $1, M - jle .L99 + jle L(99) #ifdef LN movq K, %rax @@ -717,10 +717,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADDSD2 %xmm2, %xmm9 movaps %xmm0, %xmm2 @@ -798,10 +798,10 @@ addq $8 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -811,10 +811,10 @@ andq $3, %rax BRANCH BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): ADDSD2 %xmm2, %xmm9 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -837,10 +837,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): ADDSD2 %xmm2, %xmm9 ADDSD4 %xmm6, %xmm11 @@ -945,7 +945,7 @@ #endif ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -963,10 +963,10 @@ subq $1, KK #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_LT_2x2_core2.S b/kernel/x86_64/ztrsm_kernel_LT_2x2_core2.S index 6cf850609d..87c4f531ee 100644 --- a/kernel/x86_64/ztrsm_kernel_LT_2x2_core2.S +++ b/kernel/x86_64/ztrsm_kernel_LT_2x2_core2.S @@ -196,10 +196,10 @@ movq N, J sarq $1, J # j = (n >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L01: +L(01): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -234,12 +234,12 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): prefetcht0 (PREFETCH_R + 0) * SIZE(B) movddup -16 * SIZE(B), %xmm8 @@ -268,10 +268,10 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -280,10 +280,10 @@ #endif andq $3, %rax BRANCH - jle .L05 + jle L(05) ALIGN_4 -.L04: +L(04): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 movddup -14 * SIZE(B), %xmm10 @@ -298,10 +298,10 @@ addq $ 8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L05: +L(05): #if defined(LT) || defined(RN) movq A, AO #else @@ -322,10 +322,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L30 + jle L(30) ALIGN_4 -.L10: +L(10): leaq (PREFETCH_R + 0) * SIZE(B), BB #ifdef LN @@ -387,10 +387,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -511,10 +511,10 @@ subq $1, %rax BRANCH BRANCH - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -526,10 +526,10 @@ andq $3, %rax BRANCH BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): ADD1 %xmm2, %xmm10 ADD1 %xmm3, %xmm14 ADD2 %xmm4, %xmm11 @@ -566,10 +566,10 @@ addq $8 * SIZE, BO subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): ADD1 %xmm2, %xmm10 ADD1 %xmm3, %xmm14 ADD2 %xmm4, %xmm11 @@ -947,12 +947,12 @@ #endif decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L30: +L(30): testq $1, M - jle .L99 + jle L(99) #ifdef LN movq K, %rax @@ -987,9 +987,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L42 + je L(42) -.L41: +L(41): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1059,9 +1059,9 @@ subq $ -8 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L41 + jne L(41) -.L42: +L(42): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1072,9 +1072,9 @@ andq $3, %rax # if (k & 1) BRANCH - jle .L44 + jle L(44) -.L43: +L(43): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -1094,10 +1094,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1312,7 +1312,7 @@ #endif ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1334,13 +1334,13 @@ #endif decq J # j -- - jg .L01 + jg L(01) -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) -.L101: +L(101): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -1375,10 +1375,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 movddup -14 * SIZE(B), %xmm10 @@ -1400,10 +1400,10 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1412,10 +1412,10 @@ #endif andq $3, %rax BRANCH - jle .L105 + jle L(105) ALIGN_4 -.L104: +L(104): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 @@ -1425,10 +1425,10 @@ addq $4 * SIZE, BO addq $2 * SIZE, B decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq A, AO #else @@ -1446,10 +1446,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L130 + jle L(130) ALIGN_4 -.L110: +L(110): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1484,9 +1484,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L112 + je L(112) -.L111: +L(111): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1564,10 +1564,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1577,9 +1577,9 @@ movapd POSINV, %xmm7 andq $3, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): movapd -16 * SIZE(AO), %xmm0 movapd -14 * SIZE(AO), %xmm1 @@ -1601,10 +1601,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1834,15 +1834,15 @@ #endif decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L130: +L(130): testq $1, M - jle .L199 + jle L(199) ALIGN_4 -.L140: +L(140): #ifdef LN movq K, %rax salq $0 + ZBASE_SHIFT, %rax @@ -1876,9 +1876,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L142 + je L(142) -.L141: +L(141): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1918,9 +1918,9 @@ subq $ -8 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L141 + jne L(141) -.L142: +L(142): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1931,9 +1931,9 @@ andq $3, %rax # if (k & 1) BRANCH - jle .L144 + jle L(144) -.L143: +L(143): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -1947,10 +1947,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L143 + jg L(143) ALIGN_4 -.L144: +L(144): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -2108,7 +2108,7 @@ #endif ALIGN_4 -.L199: +L(199): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2131,7 +2131,7 @@ ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/ztrsm_kernel_LT_2x2_penryn.S b/kernel/x86_64/ztrsm_kernel_LT_2x2_penryn.S index 0078117587..4277220c73 100644 --- a/kernel/x86_64/ztrsm_kernel_LT_2x2_penryn.S +++ b/kernel/x86_64/ztrsm_kernel_LT_2x2_penryn.S @@ -171,10 +171,10 @@ movq N, J sarq $1, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -212,10 +212,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -271,10 +271,10 @@ #endif sarq $3, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 @@ -520,10 +520,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -532,10 +532,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 @@ -571,10 +571,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax subq $2, %rax @@ -918,13 +918,13 @@ decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M BRANCH - jle .L39 + jle L(39) ALIGN_4 #ifdef LN @@ -963,10 +963,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_4 -.L22: +L(22): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -1032,10 +1032,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1044,10 +1044,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -1068,10 +1068,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1269,7 +1269,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1289,13 +1289,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(LT) || defined(RN) movq A, AO @@ -1333,10 +1333,10 @@ movq M, I sarq $1, I # i = (m >> 2) NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1374,10 +1374,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_4 -.L52: +L(52): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -1452,10 +1452,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1464,10 +1464,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -1490,10 +1490,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1706,13 +1706,13 @@ #endif decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L79 + jle L(79) ALIGN_4 #ifdef LN @@ -1746,10 +1746,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x4e, %xmm2, %xmm7 @@ -1792,10 +1792,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1804,10 +1804,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm7 @@ -1822,10 +1822,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -1968,7 +1968,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -1987,7 +1987,7 @@ #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_LT_2x2_sse2.S b/kernel/x86_64/ztrsm_kernel_LT_2x2_sse2.S index d90bfd8a55..0dd1fd3f51 100644 --- a/kernel/x86_64/ztrsm_kernel_LT_2x2_sse2.S +++ b/kernel/x86_64/ztrsm_kernel_LT_2x2_sse2.S @@ -320,10 +320,10 @@ movq N, J sarq $1, J # j = (n >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L01: +L(01): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -358,12 +358,12 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): PREFETCHNTA 56 * SIZE(B) movlpd 0 * SIZE(B), %xmm0 @@ -395,10 +395,10 @@ subq $-16 * SIZE, BO addq $ 8 * SIZE, B decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -407,10 +407,10 @@ #endif andq $3, %rax BRANCH - jle .L05 + jle L(05) ALIGN_4 -.L04: +L(04): movlpd 0 * SIZE(B), %xmm0 movlpd 1 * SIZE(B), %xmm1 movlpd 2 * SIZE(B), %xmm2 @@ -429,10 +429,10 @@ addq $ 8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L05: +L(05): #if defined(LT) || defined(RN) movq A, AO #else @@ -453,10 +453,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L30 + jle L(30) ALIGN_4 -.L10: +L(10): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -507,8 +507,8 @@ #endif andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -526,7 +526,7 @@ KERNEL7(16 * 1) KERNEL8(16 * 1) cmpq $64 * 2, %rax - jle .L12 + jle L(12) KERNEL1(16 * 2) KERNEL2(16 * 2) KERNEL3(16 * 2) @@ -544,7 +544,7 @@ KERNEL7(16 * 3) KERNEL8(16 * 3) cmpq $64 * 4, %rax - jle .L12 + jle L(12) KERNEL1(16 * 4) KERNEL2(16 * 4) KERNEL3(16 * 4) @@ -562,7 +562,7 @@ KERNEL7(16 * 5) KERNEL8(16 * 5) cmpq $64 * 6, %rax - jle .L12 + jle L(12) KERNEL1(16 * 6) KERNEL2(16 * 6) KERNEL3(16 * 6) @@ -583,14 +583,14 @@ addq $16 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $64 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 4), BO # * 64 ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -600,10 +600,10 @@ movapd POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -632,10 +632,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1031,12 +1031,12 @@ #endif decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L30: +L(30): testq $1, M - jle .L99 + jle L(99) #ifdef LN movq K, %rax @@ -1071,9 +1071,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L42 + je L(42) -.L41: +L(41): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 @@ -1149,9 +1149,9 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L41 + jne L(41) -.L42: +L(42): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1161,9 +1161,9 @@ movapd POSINV, %xmm15 andq $3, %rax # if (k & 1) BRANCH - jle .L44 + jle L(44) -.L43: +L(43): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 @@ -1186,10 +1186,10 @@ addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1417,7 +1417,7 @@ #endif ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1439,13 +1439,13 @@ #endif decq J # j -- - jg .L01 + jg L(01) -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) -.L101: +L(101): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -1481,10 +1481,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movlpd 0 * SIZE(B), %xmm0 movlpd 1 * SIZE(B), %xmm1 movlpd 2 * SIZE(B), %xmm2 @@ -1514,10 +1514,10 @@ subq $-16 * SIZE, BO addq $ 8 * SIZE, B decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1526,10 +1526,10 @@ #endif andq $3, %rax BRANCH - jle .L105 + jle L(105) ALIGN_4 -.L104: +L(104): movlpd 0 * SIZE(B), %xmm0 movlpd 1 * SIZE(B), %xmm1 @@ -1541,10 +1541,10 @@ addq $4 * SIZE, BO addq $2 * SIZE, B decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq A, AO #else @@ -1562,10 +1562,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L130 + jle L(130) ALIGN_4 -.L110: +L(110): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1600,9 +1600,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L112 + je L(112) -.L111: +L(111): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -1662,10 +1662,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1675,9 +1675,9 @@ movapd POSINV, %xmm15 andq $3, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -1695,10 +1695,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1943,15 +1943,15 @@ #endif decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L130: +L(130): testq $1, M - jle .L199 + jle L(199) ALIGN_4 -.L140: +L(140): #ifdef LN movq K, %rax salq $0 + ZBASE_SHIFT, %rax @@ -1985,9 +1985,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L142 + je L(142) -.L141: +L(141): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -2019,9 +2019,9 @@ addq $8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L141 + jne L(141) -.L142: +L(142): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -2035,9 +2035,9 @@ #endif andq $3, %rax # if (k & 1) BRANCH - jle .L144 + jle L(144) -.L143: +L(143): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -2048,10 +2048,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L143 + jg L(143) ALIGN_4 -.L144: +L(144): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2213,7 +2213,7 @@ #endif ALIGN_4 -.L199: +L(199): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2236,7 +2236,7 @@ ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/ztrsm_kernel_LT_2x2_sse3.S b/kernel/x86_64/ztrsm_kernel_LT_2x2_sse3.S index c52b058bf6..1ad609874b 100644 --- a/kernel/x86_64/ztrsm_kernel_LT_2x2_sse3.S +++ b/kernel/x86_64/ztrsm_kernel_LT_2x2_sse3.S @@ -408,10 +408,10 @@ movq N, J sarq $1, J # j = (n >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -444,10 +444,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L30 + jle L(30) ALIGN_4 -.L10: +L(10): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -493,9 +493,9 @@ #endif andq $-8, %rax salq $4, %rax - je .L12 + je L(12) -.L1X: +L(1X): KERNEL1 (16 * 0) KERNEL2 (16 * 0) KERNEL3 (16 * 0) @@ -514,7 +514,7 @@ KERNEL16(16 * 0) cmpq $128 * 1, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 1) KERNEL2 (16 * 1) KERNEL3 (16 * 1) @@ -533,7 +533,7 @@ KERNEL16(16 * 1) cmpq $128 * 2, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 2) KERNEL2 (16 * 2) KERNEL3 (16 * 2) @@ -552,7 +552,7 @@ KERNEL16(16 * 2) cmpq $128 * 3, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 3) KERNEL2 (16 * 3) KERNEL3 (16 * 3) @@ -571,7 +571,7 @@ KERNEL16(16 * 3) cmpq $128 * 4, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 4) KERNEL2 (16 * 4) KERNEL3 (16 * 4) @@ -590,7 +590,7 @@ KERNEL16(16 * 4) cmpq $128 * 5, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 5) KERNEL2 (16 * 5) KERNEL3 (16 * 5) @@ -609,7 +609,7 @@ KERNEL16(16 * 5) cmpq $128 * 6, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 6) KERNEL2 (16 * 6) KERNEL3 (16 * 6) @@ -628,7 +628,7 @@ KERNEL16(16 * 6) cmpq $128 * 7, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 7) KERNEL2 (16 * 7) KERNEL3 (16 * 7) @@ -649,14 +649,14 @@ addq $32 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $128 * 8, %rax - jg .L1X + jg L(1X) -.L11: +L(11): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 2), BO # * 64 ALIGN_4 -.L12: +L(12): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -665,10 +665,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L14 + je L(14) ALIGN_4 -.L13: +L(13): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 ADD1 %xmm9, %xmm0 @@ -699,10 +699,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm3, %xmm3 SHUFPD_1 %xmm5, %xmm5 @@ -975,12 +975,12 @@ addq %rax, AORIG #endif decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L30: +L(30): testq $1, M - jle .L99 + jle L(99) #ifdef LN movq K, %rax @@ -1014,9 +1014,9 @@ subq KK, %rax #endif sarq $3, %rax - je .L42 + je L(42) -.L41: +L(41): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -1126,9 +1126,9 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L41 + jne L(41) -.L42: +L(42): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1137,9 +1137,9 @@ #endif andq $7, %rax # if (k & 1) BRANCH - jle .L44 + jle L(44) -.L43: +L(43): mulpd %xmm8, %xmm9 ADD1 %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -1157,10 +1157,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm3, %xmm3 @@ -1400,7 +1400,7 @@ #endif ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1418,13 +1418,13 @@ subq $2, KK #endif decq J # j -- - jg .L01 + jg L(01) -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) -.L101: +L(101): #if defined(LT) || defined(RN) movq A, AO #else @@ -1453,10 +1453,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L130 + jle L(130) ALIGN_4 -.L110: +L(110): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1491,9 +1491,9 @@ subq KK, %rax #endif sarq $3, %rax - je .L112 + je L(112) -.L111: +L(111): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -1612,10 +1612,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1624,9 +1624,9 @@ #endif andq $7, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 ADD1 %xmm9, %xmm0 @@ -1645,10 +1645,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm5, %xmm5 @@ -1863,12 +1863,12 @@ #endif decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L130: +L(130): testq $1, M - jle .L149 + jle L(149) ALIGN_4 #ifdef LN @@ -1903,9 +1903,9 @@ subq KK, %rax #endif sarq $3, %rax - je .L142 + je L(142) -.L141: +L(141): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -1967,9 +1967,9 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L141 + jne L(141) -.L142: +L(142): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1978,9 +1978,9 @@ #endif andq $7, %rax # if (k & 1) BRANCH - jle .L144 + jle L(144) -.L143: +L(143): mulpd %xmm8, %xmm9 ADD1 %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -1992,10 +1992,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L143 + jg L(143) ALIGN_4 -.L144: +L(144): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -2146,7 +2146,7 @@ #endif ALIGN_4 -.L149: +L(149): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2165,7 +2165,7 @@ #endif ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_LT_2x4_nehalem.S b/kernel/x86_64/ztrsm_kernel_LT_2x4_nehalem.S index 0d6531ac85..49aa90673a 100644 --- a/kernel/x86_64/ztrsm_kernel_LT_2x4_nehalem.S +++ b/kernel/x86_64/ztrsm_kernel_LT_2x4_nehalem.S @@ -171,10 +171,10 @@ movq N, J sarq $2, J NOBRANCH - jle .L30 + jle L(30) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -212,10 +212,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -265,10 +265,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm12 @@ -395,10 +395,10 @@ subq $-16 * SIZE, AO subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -407,10 +407,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm1, %xmm12 movaps -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm13 @@ -446,10 +446,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1010,13 +1010,13 @@ decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M BRANCH - jle .L29 + jle L(29) #ifdef LN movq K, %rax @@ -1056,10 +1056,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1129,10 +1129,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1141,10 +1141,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1166,10 +1166,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1587,7 +1587,7 @@ #endif ALIGN_4 -.L29: +L(29): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -1608,13 +1608,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L30: +L(30): testq $2, N BRANCH - jle .L50 + jle L(50) #if defined(LT) || defined(RN) movq A, AO @@ -1649,10 +1649,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1692,10 +1692,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -1759,10 +1759,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1771,10 +1771,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): ADD1 %xmm1, %xmm8 movaps -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -1794,10 +1794,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2108,13 +2108,13 @@ decq I # i -- BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L40: +L(40): testq $1, M BRANCH - jle .L49 + jle L(49) #ifdef LN movq K, %rax @@ -2154,10 +2154,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2199,10 +2199,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2211,10 +2211,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2229,10 +2229,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2446,7 +2446,7 @@ #endif ALIGN_4 -.L49: +L(49): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -2466,10 +2466,10 @@ #endif ALIGN_4 -.L50: +L(50): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(LT) || defined(RN) movq A, AO @@ -2502,10 +2502,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -2544,10 +2544,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -2587,10 +2587,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2599,10 +2599,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): ADD1 %xmm1, %xmm8 movddup -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -2616,10 +2616,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2832,13 +2832,13 @@ decq I # i -- BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L69 + jle L(69) #ifdef LN movq K, %rax @@ -2873,10 +2873,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2918,10 +2918,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2930,10 +2930,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2948,10 +2948,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -3067,7 +3067,7 @@ #endif ALIGN_4 -.L69: +L(69): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -3087,7 +3087,7 @@ #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_LT_4x2_sse.S b/kernel/x86_64/ztrsm_kernel_LT_4x2_sse.S index 0c3a052a17..15850c5542 100644 --- a/kernel/x86_64/ztrsm_kernel_LT_4x2_sse.S +++ b/kernel/x86_64/ztrsm_kernel_LT_4x2_sse.S @@ -332,10 +332,10 @@ movq N, J sarq $1, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -370,10 +370,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 @@ -423,10 +423,10 @@ addq $16 * SIZE, B addq $64 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -435,10 +435,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movaps 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -454,10 +454,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -478,10 +478,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + ZBASE_SHIFT, %rax @@ -532,8 +532,8 @@ #endif andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -554,14 +554,14 @@ addq $32 * 2 * SIZE, AO addq $64 * 2 * SIZE, BO subq $64 * 2, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO leaq (BO, %rax, 4), BO ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -571,10 +571,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -603,10 +603,10 @@ addq $ 8 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm3, %xmm3 shufps $0xb1, %xmm5, %xmm5 @@ -1324,12 +1324,12 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) #ifdef LN movq K, %rax @@ -1374,10 +1374,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -1492,10 +1492,10 @@ addq $128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1505,10 +1505,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1525,10 +1525,10 @@ addq $ 4 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm3, %xmm3 @@ -1887,9 +1887,9 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #ifdef LN movq K, %rax @@ -1934,10 +1934,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -2055,10 +2055,10 @@ addq $128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2068,10 +2068,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -2090,10 +2090,10 @@ addq $ 2 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm3, %xmm3 @@ -2368,7 +2368,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2 * COMPSIZE), B @@ -2390,12 +2390,12 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $1, N - je .L999 + je L(999) ALIGN_4 #ifdef LN @@ -2432,10 +2432,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 @@ -2462,10 +2462,10 @@ addq $ 8 * SIZE, B addq $32 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2474,10 +2474,10 @@ #endif andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movsd 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -2489,10 +2489,10 @@ addq $2 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): #if defined(LT) || defined(RN) movq A, AO #else @@ -2511,10 +2511,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + ZBASE_SHIFT, %rax @@ -2565,10 +2565,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2690,10 +2690,10 @@ decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2703,10 +2703,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -2723,10 +2723,10 @@ addq $ 8 * SIZE, AO # aoffset += 4 addq $ 8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm5, %xmm5 @@ -3269,12 +3269,12 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) #ifdef LN movq K, %rax @@ -3317,10 +3317,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3384,10 +3384,10 @@ addq $64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3397,10 +3397,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -3411,10 +3411,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -3662,9 +3662,9 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) #ifdef LN movq K, %rax @@ -3709,10 +3709,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -3785,10 +3785,10 @@ addq $64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3798,10 +3798,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -3813,10 +3813,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -3960,7 +3960,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, COMPSIZE), B @@ -3982,7 +3982,7 @@ #endif ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/ztrsm_kernel_RT_1x4_nehalem.S b/kernel/x86_64/ztrsm_kernel_RT_1x4_nehalem.S index a65c2718ba..465a02a955 100644 --- a/kernel/x86_64/ztrsm_kernel_RT_1x4_nehalem.S +++ b/kernel/x86_64/ztrsm_kernel_RT_1x4_nehalem.S @@ -168,11 +168,11 @@ #endif testq M, M - jle .L999 + jle L(999) testq $1, N BRANCH - jle .L20 + jle L(20) #if defined(LT) || defined(RN) movq A, AO @@ -204,7 +204,7 @@ movq M, I ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -241,10 +241,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -284,13 +284,13 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 ALIGN_3 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -299,10 +299,10 @@ #endif andq $3, %rax BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): ADD1 %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -316,10 +316,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -445,7 +445,7 @@ #endif decq I BRANCH - jg .L31 + jg L(31) #ifdef LN movq K, %rax @@ -466,10 +466,10 @@ #endif ALIGN_4 -.L20: +L(20): testq $2, N BRANCH - jle .L30 + jle L(30) #if defined(LT) || defined(RN) movq A, AO @@ -504,7 +504,7 @@ movq M, I ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -544,10 +544,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -619,10 +619,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -631,10 +631,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): ADD1 %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -656,10 +656,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -861,7 +861,7 @@ decq I BRANCH - jg .L21 + jg L(21) #ifdef LN movq K, %rax @@ -882,14 +882,14 @@ #endif ALIGN_4 -.L30: +L(30): movq N, J sarq $2, J NOBRANCH - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -927,7 +927,7 @@ movq M, I ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -977,10 +977,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm12 @@ -1104,10 +1104,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1116,10 +1116,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm1, %xmm12 movaps -16 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm13 @@ -1155,10 +1155,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1532,7 +1532,7 @@ decq I BRANCH - jg .L11 + jg L(11) #ifdef LN movq K, %rax @@ -1554,10 +1554,10 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_RT_2x2_core2.S b/kernel/x86_64/ztrsm_kernel_RT_2x2_core2.S index 0702b00887..ce09e346c4 100644 --- a/kernel/x86_64/ztrsm_kernel_RT_2x2_core2.S +++ b/kernel/x86_64/ztrsm_kernel_RT_2x2_core2.S @@ -195,9 +195,9 @@ #endif testq $1, N - jle .L100 + jle L(100) -.L101: +L(101): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -232,10 +232,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 movddup -14 * SIZE(B), %xmm10 @@ -257,10 +257,10 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -269,10 +269,10 @@ #endif andq $3, %rax BRANCH - jle .L105 + jle L(105) ALIGN_4 -.L104: +L(104): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 @@ -282,10 +282,10 @@ addq $4 * SIZE, BO addq $2 * SIZE, B decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq A, AO #else @@ -303,10 +303,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L130 + jle L(130) ALIGN_4 -.L110: +L(110): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -341,9 +341,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L112 + je L(112) -.L111: +L(111): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -421,10 +421,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -434,9 +434,9 @@ movapd POSINV, %xmm7 andq $3, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): movapd -16 * SIZE(AO), %xmm0 movapd -14 * SIZE(AO), %xmm1 @@ -458,10 +458,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -691,15 +691,15 @@ #endif decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L130: +L(130): testq $1, M - jle .L199 + jle L(199) ALIGN_4 -.L140: +L(140): #ifdef LN movq K, %rax salq $0 + ZBASE_SHIFT, %rax @@ -733,9 +733,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L142 + je L(142) -.L141: +L(141): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -775,9 +775,9 @@ subq $ -8 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L141 + jne L(141) -.L142: +L(142): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -788,9 +788,9 @@ andq $3, %rax # if (k & 1) BRANCH - jle .L144 + jle L(144) -.L143: +L(143): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -804,10 +804,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L143 + jg L(143) ALIGN_4 -.L144: +L(144): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -965,7 +965,7 @@ #endif ALIGN_4 -.L199: +L(199): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -987,13 +987,13 @@ #endif ALIGN_4 -.L100: +L(100): movq N, J sarq $1, J # j = (n >> 2) - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -1028,12 +1028,12 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): prefetcht0 (PREFETCH_R + 0) * SIZE(B) movddup -16 * SIZE(B), %xmm8 @@ -1062,10 +1062,10 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1074,10 +1074,10 @@ #endif andq $3, %rax BRANCH - jle .L05 + jle L(05) ALIGN_4 -.L04: +L(04): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 movddup -14 * SIZE(B), %xmm10 @@ -1092,10 +1092,10 @@ addq $ 8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L05: +L(05): #if defined(LT) || defined(RN) movq A, AO #else @@ -1116,10 +1116,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L30 + jle L(30) ALIGN_4 -.L10: +L(10): leaq (PREFETCH_R + 0) * SIZE(B), BB #ifdef LN @@ -1181,10 +1181,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1305,10 +1305,10 @@ subq $1, %rax BRANCH BRANCH - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1320,10 +1320,10 @@ andq $3, %rax BRANCH BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): ADD1 %xmm2, %xmm10 ADD1 %xmm3, %xmm14 ADD2 %xmm4, %xmm11 @@ -1360,10 +1360,10 @@ addq $8 * SIZE, BO subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): ADD1 %xmm2, %xmm10 ADD1 %xmm3, %xmm14 ADD2 %xmm4, %xmm11 @@ -1741,12 +1741,12 @@ #endif decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L30: +L(30): testq $1, M - jle .L99 + jle L(99) #ifdef LN movq K, %rax @@ -1781,9 +1781,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L42 + je L(42) -.L41: +L(41): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1853,9 +1853,9 @@ subq $ -8 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L41 + jne L(41) -.L42: +L(42): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1866,9 +1866,9 @@ andq $3, %rax # if (k & 1) BRANCH - jle .L44 + jle L(44) -.L43: +L(43): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -1888,10 +1888,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2106,7 +2106,7 @@ #endif ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -2128,10 +2128,10 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/ztrsm_kernel_RT_2x2_penryn.S b/kernel/x86_64/ztrsm_kernel_RT_2x2_penryn.S index 7770f5d9cc..1b0db5bca6 100644 --- a/kernel/x86_64/ztrsm_kernel_RT_2x2_penryn.S +++ b/kernel/x86_64/ztrsm_kernel_RT_2x2_penryn.S @@ -170,7 +170,7 @@ testq $1, N BRANCH - jle .L40 + jle L(40) #if defined(LT) || defined(RN) movq A, AO @@ -204,10 +204,10 @@ movq M, I sarq $1, I # i = (m >> 2) NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -242,10 +242,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_4 -.L52: +L(52): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -320,10 +320,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -332,10 +332,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -358,10 +358,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -574,13 +574,13 @@ #endif decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L79 + jle L(79) ALIGN_4 #ifdef LN @@ -614,10 +614,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x4e, %xmm2, %xmm7 @@ -660,10 +660,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -672,10 +672,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm7 @@ -690,10 +690,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -836,7 +836,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -855,14 +855,14 @@ #endif ALIGN_4 -.L40: +L(40): movq N, J sarq $1, J NOBRANCH - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -901,10 +901,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -960,10 +960,10 @@ #endif sarq $3, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 @@ -1209,10 +1209,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1221,10 +1221,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 @@ -1260,10 +1260,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax subq $2, %rax @@ -1607,13 +1607,13 @@ decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M BRANCH - jle .L39 + jle L(39) ALIGN_4 #ifdef LN @@ -1652,10 +1652,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_4 -.L22: +L(22): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -1721,10 +1721,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1733,10 +1733,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -1757,10 +1757,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1958,7 +1958,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1978,10 +1978,10 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_RT_2x2_sse2.S b/kernel/x86_64/ztrsm_kernel_RT_2x2_sse2.S index 2dffe2d327..c27af76346 100644 --- a/kernel/x86_64/ztrsm_kernel_RT_2x2_sse2.S +++ b/kernel/x86_64/ztrsm_kernel_RT_2x2_sse2.S @@ -318,9 +318,9 @@ #endif testq $1, N - jle .L100 + jle L(100) -.L101: +L(101): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -356,10 +356,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movlpd 0 * SIZE(B), %xmm0 movlpd 1 * SIZE(B), %xmm1 movlpd 2 * SIZE(B), %xmm2 @@ -389,10 +389,10 @@ subq $-16 * SIZE, BO addq $ 8 * SIZE, B decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -401,10 +401,10 @@ #endif andq $3, %rax BRANCH - jle .L105 + jle L(105) ALIGN_4 -.L104: +L(104): movlpd 0 * SIZE(B), %xmm0 movlpd 1 * SIZE(B), %xmm1 @@ -416,10 +416,10 @@ addq $4 * SIZE, BO addq $2 * SIZE, B decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq A, AO #else @@ -437,10 +437,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L130 + jle L(130) ALIGN_4 -.L110: +L(110): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -475,9 +475,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L112 + je L(112) -.L111: +L(111): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -537,10 +537,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -550,9 +550,9 @@ movapd POSINV, %xmm15 andq $3, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -570,10 +570,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -818,15 +818,15 @@ #endif decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L130: +L(130): testq $1, M - jle .L199 + jle L(199) ALIGN_4 -.L140: +L(140): #ifdef LN movq K, %rax salq $0 + ZBASE_SHIFT, %rax @@ -860,9 +860,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L142 + je L(142) -.L141: +L(141): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -894,9 +894,9 @@ addq $8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L141 + jne L(141) -.L142: +L(142): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -910,9 +910,9 @@ #endif andq $3, %rax # if (k & 1) BRANCH - jle .L144 + jle L(144) -.L143: +L(143): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -923,10 +923,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L143 + jg L(143) ALIGN_4 -.L144: +L(144): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1088,7 +1088,7 @@ #endif ALIGN_4 -.L199: +L(199): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -1110,13 +1110,13 @@ #endif ALIGN_4 -.L100: +L(100): movq N, J sarq $1, J # j = (n >> 2) - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -1152,12 +1152,12 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): PREFETCHNTA 56 * SIZE(B) movlpd 0 * SIZE(B), %xmm0 @@ -1189,10 +1189,10 @@ subq $-16 * SIZE, BO addq $ 8 * SIZE, B decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1201,10 +1201,10 @@ #endif andq $3, %rax BRANCH - jle .L05 + jle L(05) ALIGN_4 -.L04: +L(04): movlpd 0 * SIZE(B), %xmm0 movlpd 1 * SIZE(B), %xmm1 movlpd 2 * SIZE(B), %xmm2 @@ -1223,10 +1223,10 @@ addq $ 8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L05: +L(05): #if defined(LT) || defined(RN) movq A, AO #else @@ -1247,10 +1247,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L30 + jle L(30) ALIGN_4 -.L10: +L(10): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1301,8 +1301,8 @@ #endif andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -1320,7 +1320,7 @@ KERNEL7(16 * 1) KERNEL8(16 * 1) cmpq $64 * 2, %rax - jle .L12 + jle L(12) KERNEL1(16 * 2) KERNEL2(16 * 2) KERNEL3(16 * 2) @@ -1338,7 +1338,7 @@ KERNEL7(16 * 3) KERNEL8(16 * 3) cmpq $64 * 4, %rax - jle .L12 + jle L(12) KERNEL1(16 * 4) KERNEL2(16 * 4) KERNEL3(16 * 4) @@ -1356,7 +1356,7 @@ KERNEL7(16 * 5) KERNEL8(16 * 5) cmpq $64 * 6, %rax - jle .L12 + jle L(12) KERNEL1(16 * 6) KERNEL2(16 * 6) KERNEL3(16 * 6) @@ -1377,14 +1377,14 @@ addq $16 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $64 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 4), BO # * 64 ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1394,10 +1394,10 @@ movapd POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -1426,10 +1426,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1825,12 +1825,12 @@ #endif decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L30: +L(30): testq $1, M - jle .L99 + jle L(99) #ifdef LN movq K, %rax @@ -1865,9 +1865,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L42 + je L(42) -.L41: +L(41): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 @@ -1943,9 +1943,9 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L41 + jne L(41) -.L42: +L(42): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1955,9 +1955,9 @@ movapd POSINV, %xmm15 andq $3, %rax # if (k & 1) BRANCH - jle .L44 + jle L(44) -.L43: +L(43): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 @@ -1980,10 +1980,10 @@ addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2211,7 +2211,7 @@ #endif ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -2233,10 +2233,10 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_3 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/ztrsm_kernel_RT_2x2_sse3.S b/kernel/x86_64/ztrsm_kernel_RT_2x2_sse3.S index a473df4dcd..fb322a45e8 100644 --- a/kernel/x86_64/ztrsm_kernel_RT_2x2_sse3.S +++ b/kernel/x86_64/ztrsm_kernel_RT_2x2_sse3.S @@ -407,9 +407,9 @@ #endif testq $1, N - jle .L100 + jle L(100) -.L101: +L(101): #if defined(LT) || defined(RN) movq A, AO #else @@ -438,10 +438,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L130 + jle L(130) ALIGN_4 -.L110: +L(110): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -476,9 +476,9 @@ subq KK, %rax #endif sarq $3, %rax - je .L112 + je L(112) -.L111: +L(111): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -597,10 +597,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -609,9 +609,9 @@ #endif andq $7, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 ADD1 %xmm9, %xmm0 @@ -630,10 +630,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm5, %xmm5 @@ -848,12 +848,12 @@ #endif decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L130: +L(130): testq $1, M - jle .L149 + jle L(149) ALIGN_4 #ifdef LN @@ -888,9 +888,9 @@ subq KK, %rax #endif sarq $3, %rax - je .L142 + je L(142) -.L141: +L(141): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -952,9 +952,9 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L141 + jne L(141) -.L142: +L(142): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -963,9 +963,9 @@ #endif andq $7, %rax # if (k & 1) BRANCH - jle .L144 + jle L(144) -.L143: +L(143): mulpd %xmm8, %xmm9 ADD1 %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -977,10 +977,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L143 + jg L(143) ALIGN_4 -.L144: +L(144): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -1131,7 +1131,7 @@ #endif ALIGN_4 -.L149: +L(149): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -1151,13 +1151,13 @@ ALIGN_3 -.L100: +L(100): movq N, J sarq $1, J # j = (n >> 2) - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -1190,10 +1190,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L30 + jle L(30) ALIGN_4 -.L10: +L(10): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1239,9 +1239,9 @@ #endif andq $-8, %rax salq $4, %rax - je .L12 + je L(12) -.L1X: +L(1X): KERNEL1 (16 * 0) KERNEL2 (16 * 0) KERNEL3 (16 * 0) @@ -1260,7 +1260,7 @@ KERNEL16(16 * 0) cmpq $128 * 1, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 1) KERNEL2 (16 * 1) KERNEL3 (16 * 1) @@ -1279,7 +1279,7 @@ KERNEL16(16 * 1) cmpq $128 * 2, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 2) KERNEL2 (16 * 2) KERNEL3 (16 * 2) @@ -1298,7 +1298,7 @@ KERNEL16(16 * 2) cmpq $128 * 3, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 3) KERNEL2 (16 * 3) KERNEL3 (16 * 3) @@ -1317,7 +1317,7 @@ KERNEL16(16 * 3) cmpq $128 * 4, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 4) KERNEL2 (16 * 4) KERNEL3 (16 * 4) @@ -1336,7 +1336,7 @@ KERNEL16(16 * 4) cmpq $128 * 5, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 5) KERNEL2 (16 * 5) KERNEL3 (16 * 5) @@ -1355,7 +1355,7 @@ KERNEL16(16 * 5) cmpq $128 * 6, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 6) KERNEL2 (16 * 6) KERNEL3 (16 * 6) @@ -1374,7 +1374,7 @@ KERNEL16(16 * 6) cmpq $128 * 7, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 7) KERNEL2 (16 * 7) KERNEL3 (16 * 7) @@ -1395,14 +1395,14 @@ addq $32 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $128 * 8, %rax - jg .L1X + jg L(1X) -.L11: +L(11): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 2), BO # * 64 ALIGN_4 -.L12: +L(12): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1411,10 +1411,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L14 + je L(14) ALIGN_4 -.L13: +L(13): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 ADD1 %xmm9, %xmm0 @@ -1445,10 +1445,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm3, %xmm3 SHUFPD_1 %xmm5, %xmm5 @@ -1721,12 +1721,12 @@ addq %rax, AORIG #endif decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L30: +L(30): testq $1, M - jle .L99 + jle L(99) #ifdef LN movq K, %rax @@ -1760,9 +1760,9 @@ subq KK, %rax #endif sarq $3, %rax - je .L42 + je L(42) -.L41: +L(41): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -1872,9 +1872,9 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L41 + jne L(41) -.L42: +L(42): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1883,9 +1883,9 @@ #endif andq $7, %rax # if (k & 1) BRANCH - jle .L44 + jle L(44) -.L43: +L(43): mulpd %xmm8, %xmm9 ADD1 %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -1903,10 +1903,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm3, %xmm3 @@ -2146,7 +2146,7 @@ #endif ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -2164,10 +2164,10 @@ subq $2, KK #endif decq J # j -- - jg .L01 + jg L(01) -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_RT_2x4_nehalem.S b/kernel/x86_64/ztrsm_kernel_RT_2x4_nehalem.S index ddb5fe0982..768a9b2fa6 100644 --- a/kernel/x86_64/ztrsm_kernel_RT_2x4_nehalem.S +++ b/kernel/x86_64/ztrsm_kernel_RT_2x4_nehalem.S @@ -170,7 +170,7 @@ testq $1, N BRANCH - jle .L30 + jle L(30) #if defined(LT) || defined(RN) movq A, AO @@ -203,10 +203,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -245,10 +245,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -288,10 +288,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -300,10 +300,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): ADD1 %xmm1, %xmm8 movddup -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -317,10 +317,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -533,13 +533,13 @@ decq I # i -- BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L69 + jle L(69) #ifdef LN movq K, %rax @@ -574,10 +574,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -619,10 +619,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -631,10 +631,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -649,10 +649,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -768,7 +768,7 @@ #endif ALIGN_4 -.L69: +L(69): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -788,10 +788,10 @@ #endif ALIGN_4 -.L30: +L(30): testq $2, N BRANCH - jle .L50 + jle L(50) #if defined(LT) || defined(RN) movq A, AO @@ -826,10 +826,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -869,10 +869,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -936,10 +936,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -948,10 +948,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): ADD1 %xmm1, %xmm8 movaps -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -971,10 +971,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1285,13 +1285,13 @@ decq I # i -- BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L40: +L(40): testq $1, M BRANCH - jle .L49 + jle L(49) #ifdef LN movq K, %rax @@ -1331,10 +1331,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1376,10 +1376,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1388,10 +1388,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1406,10 +1406,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1623,7 +1623,7 @@ #endif ALIGN_4 -.L49: +L(49): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -1643,14 +1643,14 @@ #endif ALIGN_4 -.L50: +L(50): movq N, J sarq $2, J NOBRANCH - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -1688,10 +1688,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1741,10 +1741,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm12 @@ -1871,10 +1871,10 @@ subq $-16 * SIZE, AO subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1883,10 +1883,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm1, %xmm12 movaps -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm13 @@ -1922,10 +1922,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2486,13 +2486,13 @@ decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M BRANCH - jle .L29 + jle L(29) #ifdef LN movq K, %rax @@ -2532,10 +2532,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2605,10 +2605,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2617,10 +2617,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2642,10 +2642,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3063,7 +3063,7 @@ #endif ALIGN_4 -.L29: +L(29): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -3084,10 +3084,10 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_RT_4x2_sse.S b/kernel/x86_64/ztrsm_kernel_RT_4x2_sse.S index 518e1b4fef..eda16264bb 100644 --- a/kernel/x86_64/ztrsm_kernel_RT_4x2_sse.S +++ b/kernel/x86_64/ztrsm_kernel_RT_4x2_sse.S @@ -332,7 +332,7 @@ #endif testq $1, N - je .L40 + je L(40) ALIGN_4 #ifdef LN @@ -369,10 +369,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 @@ -399,10 +399,10 @@ addq $ 8 * SIZE, B addq $32 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -411,10 +411,10 @@ #endif andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movlps 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -426,10 +426,10 @@ addq $2 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): #if defined(LT) || defined(RN) movq A, AO #else @@ -448,10 +448,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + ZBASE_SHIFT, %rax @@ -502,10 +502,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -627,10 +627,10 @@ decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -640,10 +640,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -660,10 +660,10 @@ addq $ 8 * SIZE, AO # aoffset += 4 addq $ 8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm5, %xmm5 @@ -1206,12 +1206,12 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) #ifdef LN movq K, %rax @@ -1254,10 +1254,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1321,10 +1321,10 @@ addq $64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1334,10 +1334,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -1348,10 +1348,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -1599,9 +1599,9 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) #ifdef LN movq K, %rax @@ -1646,10 +1646,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -1722,10 +1722,10 @@ addq $64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1735,10 +1735,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1750,10 +1750,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -1897,7 +1897,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, COMPSIZE), B @@ -1919,13 +1919,13 @@ #endif ALIGN_4 -.L40: +L(40): movq N, J sarq $1, J # j = (n >> 2) - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -1960,10 +1960,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 @@ -2013,10 +2013,10 @@ addq $16 * SIZE, B addq $64 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2025,10 +2025,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movaps 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -2044,10 +2044,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -2068,10 +2068,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + ZBASE_SHIFT, %rax @@ -2122,8 +2122,8 @@ #endif andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -2144,14 +2144,14 @@ addq $32 * 2 * SIZE, AO addq $64 * 2 * SIZE, BO subq $64 * 2, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO leaq (BO, %rax, 4), BO ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2161,10 +2161,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -2193,10 +2193,10 @@ addq $ 8 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm3, %xmm3 shufps $0xb1, %xmm5, %xmm5 @@ -2914,12 +2914,12 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) #ifdef LN movq K, %rax @@ -2964,10 +2964,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -3082,10 +3082,10 @@ addq $128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3095,10 +3095,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -3115,10 +3115,10 @@ addq $ 4 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm3, %xmm3 @@ -3477,9 +3477,9 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #ifdef LN movq K, %rax @@ -3524,10 +3524,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -3645,10 +3645,10 @@ addq $128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3658,10 +3658,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -3680,10 +3680,10 @@ addq $ 2 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm3, %xmm3 @@ -3958,7 +3958,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2 * COMPSIZE), B @@ -3980,10 +3980,10 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp