From 3f1032ea5ae9e7c7d604a25d85fd69d5422ce4ca Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Thu, 1 Oct 2026 23:06:29 -0400 Subject: [PATCH 1/5] Add L(), the spelling of an assembler-local label, for x86_64 and arm64 Nothing uses it yet. L(name) is .Lname, except on Mach-O, where the assembler's prefix for local labels is "L" and a ".L" label is an ordinary symbol. Co-Authored-By: Claude Fable 5.1 --- common_arm64.h | 6 ++++++ common_x86_64.h | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/common_arm64.h b/common_arm64.h index d37a4a9cba..511b884854 100644 --- a/common_arm64.h +++ b/common_arm64.h @@ -151,6 +151,12 @@ static inline int blas_quickdivide(blasint x, blasint y){ #if defined(ASSEMBLER) && !defined(NEEDPARAM) +#if defined(__APPLE__) +#define L(x) L##x +#else +#define L(x) .L##x +#endif + .macro PROLOGUE .text ; .p2align 2 ; diff --git a/common_x86_64.h b/common_x86_64.h index 425bef4e5f..3d38ee3ebb 100644 --- a/common_x86_64.h +++ b/common_x86_64.h @@ -405,6 +405,12 @@ static __inline unsigned int blas_quickdivide(unsigned int x, unsigned int y){ #define REALNAME ASMFNAME #endif +#ifdef OS_DARWIN +#define L(x) L##x +#else +#define L(x) .L##x +#endif + #ifdef OS_DARWIN #define PROLOGUE .text;.align 5; .globl REALNAME; REALNAME: #define PROLOGUE_EXPORT .text;.align 5; .globl REALNAME; REALNAME: From d95a7223b2baf9695d9cd70dc3c994084435b292 Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Thu, 1 Oct 2026 23:14:10 -0400 Subject: [PATCH 2/5] x86_64: spell the local labels of the assembly kernels L(name) No change to any object except on Mach-O. The kernels name their labels .Lxxx, which makes them local to the assembler on ELF and COFF, but not on Mach-O, where that prefix is "L": there every label is a symbol. Since EPILOGUE is .subsections_via_symbols on Darwin, the linker takes each such symbol for the start of a block of its own. It does not see that one block falls through into the next, and * folds a block that is byte for byte the same as one of another kernel into that one (on by default when linking at -O2), and * with -dead_strip, removes a block that no branch refers to, either of which leaves the code before the block running on into whatever comes next. With Apple clang 21 (ld-1267), libopenblas.dylib built for NEHALEM returns garbage from cblas_sasum(5, {3,-1,2,-4,1}, 1) (sasum_k has lost its final "ret"), and openblas_utest crashes in its first test. With L(name) the labels are local on Mach-O as well, and a kernel is one block. Generated by asm-labels/use_local_label_macro.py, which rewrites every ".Lname" token of kernel/x86_64/*.S and nothing else: 23906 labels in 249 files. For an ELF target, all 255 files preprocess to the same text as before, and the 2619 kernel objects of a DYNAMIC_ARCH build are byte for byte the same. Generated-by: asm-labels/use_local_label_macro.py Co-Authored-By: Claude Fable 5.1 --- kernel/x86_64/amax.S | 42 +- kernel/x86_64/amax_atom.S | 60 +- kernel/x86_64/amax_sse.S | 72 +- kernel/x86_64/amax_sse2.S | 70 +- kernel/x86_64/asum.S | 42 +- kernel/x86_64/asum_atom.S | 72 +- kernel/x86_64/asum_sse.S | 72 +- kernel/x86_64/asum_sse2.S | 62 +- kernel/x86_64/axpy.S | 40 +- kernel/x86_64/axpy_atom.S | 56 +- kernel/x86_64/axpy_sse.S | 214 +-- kernel/x86_64/axpy_sse2.S | 122 +- kernel/x86_64/builtin_stinit.S | 8 +- kernel/x86_64/cgemm_kernel_4x2_bulldozer.S | 200 +-- kernel/x86_64/cgemm_kernel_4x2_piledriver.S | 200 +-- kernel/x86_64/cgemm_kernel_4x8_sandy.S | 352 ++--- kernel/x86_64/cgemm_kernel_8x2_haswell.S | 758 +++++----- kernel/x86_64/cgemm_kernel_8x2_sandy.S | 256 ++-- kernel/x86_64/cgemv_n.S | 400 +++--- kernel/x86_64/cgemv_t.S | 392 +++--- kernel/x86_64/copy.S | 40 +- kernel/x86_64/copy_sse.S | 174 +-- kernel/x86_64/copy_sse2.S | 116 +- kernel/x86_64/daxpy_bulldozer.S | 62 +- kernel/x86_64/dcopy_bulldozer.S | 56 +- kernel/x86_64/ddot_bulldozer.S | 58 +- kernel/x86_64/dgemm_kernel_16x2_haswell.S | 896 ++++++------ kernel/x86_64/dgemm_kernel_16x2_skylakex.S | 896 ++++++------ kernel/x86_64/dgemm_kernel_4x4_haswell.S | 626 ++++----- kernel/x86_64/dgemm_kernel_4x8_haswell.S | 898 ++++++------ kernel/x86_64/dgemm_kernel_4x8_sandy.S | 398 +++--- kernel/x86_64/dgemm_kernel_8x2_bulldozer.S | 762 +++++----- kernel/x86_64/dgemm_kernel_8x2_piledriver.S | 790 +++++------ kernel/x86_64/dgemm_ncopy_2.S | 118 +- kernel/x86_64/dgemm_ncopy_4.S | 212 +-- kernel/x86_64/dgemm_ncopy_8.S | 264 ++-- kernel/x86_64/dgemm_ncopy_8_bulldozer.S | 264 ++-- kernel/x86_64/dgemm_tcopy_2.S | 50 +- kernel/x86_64/dgemm_tcopy_4.S | 74 +- kernel/x86_64/dgemm_tcopy_8.S | 96 +- kernel/x86_64/dgemm_tcopy_8_bulldozer.S | 96 +- kernel/x86_64/dgemv_n.S | 362 ++--- kernel/x86_64/dgemv_n_atom.S | 88 +- kernel/x86_64/dgemv_n_bulldozer.S | 350 ++--- kernel/x86_64/dgemv_t.S | 310 ++--- kernel/x86_64/dgemv_t_atom.S | 82 +- kernel/x86_64/dgemv_t_bulldozer.S | 172 +-- kernel/x86_64/dot.S | 38 +- kernel/x86_64/dot_atom.S | 50 +- kernel/x86_64/dot_sse.S | 220 +-- kernel/x86_64/dot_sse2.S | 122 +- kernel/x86_64/dtrsm_kernel_LT_8x2_bulldozer.S | 190 +-- kernel/x86_64/dtrsm_kernel_RN_8x2_bulldozer.S | 190 +-- kernel/x86_64/gemm_beta.S | 50 +- kernel/x86_64/gemm_kernel_2x8_nehalem.S | 202 +-- kernel/x86_64/gemm_kernel_4x2_atom.S | 138 +- kernel/x86_64/gemm_kernel_4x4_barcelona.S | 242 ++-- kernel/x86_64/gemm_kernel_4x4_core2.S | 266 ++-- kernel/x86_64/gemm_kernel_4x4_penryn.S | 214 +-- kernel/x86_64/gemm_kernel_4x4_sse2.S | 298 ++-- kernel/x86_64/gemm_kernel_4x4_sse3.S | 250 ++-- kernel/x86_64/gemm_kernel_4x8_nano.S | 334 ++--- kernel/x86_64/gemm_kernel_4x8_nehalem.S | 274 ++-- kernel/x86_64/gemm_kernel_8x4_barcelona.S | 336 ++--- kernel/x86_64/gemm_kernel_8x4_core2.S | 318 ++--- kernel/x86_64/gemm_kernel_8x4_penryn.S | 266 ++-- kernel/x86_64/gemm_kernel_8x4_sse.S | 342 ++--- kernel/x86_64/gemm_kernel_8x4_sse3.S | 340 ++--- kernel/x86_64/gemm_ncopy_2.S | 42 +- kernel/x86_64/gemm_ncopy_2_bulldozer.S | 48 +- kernel/x86_64/gemm_ncopy_4.S | 60 +- kernel/x86_64/gemm_ncopy_4_opteron.S | 68 +- kernel/x86_64/gemm_tcopy_2.S | 36 +- kernel/x86_64/gemm_tcopy_2_bulldozer.S | 44 +- kernel/x86_64/gemm_tcopy_4.S | 64 +- kernel/x86_64/gemm_tcopy_4_opteron.S | 72 +- kernel/x86_64/iamax.S | 42 +- kernel/x86_64/iamax_sse.S | 268 ++-- kernel/x86_64/iamax_sse2.S | 258 ++-- kernel/x86_64/izamax.S | 42 +- kernel/x86_64/izamax_sse.S | 118 +- kernel/x86_64/izamax_sse2.S | 116 +- kernel/x86_64/nrm2.S | 42 +- kernel/x86_64/nrm2_sse.S | 60 +- kernel/x86_64/qdot.S | 38 +- kernel/x86_64/qgemm_kernel_2x2.S | 104 +- kernel/x86_64/qgemv_n.S | 72 +- kernel/x86_64/qgemv_t.S | 76 +- kernel/x86_64/qtrsm_kernel_LN_2x2.S | 104 +- kernel/x86_64/qtrsm_kernel_LT_2x2.S | 104 +- kernel/x86_64/qtrsm_kernel_RT_2x2.S | 104 +- kernel/x86_64/rot.S | 40 +- kernel/x86_64/rot_sse.S | 106 +- kernel/x86_64/rot_sse2.S | 94 +- kernel/x86_64/scal.S | 76 +- kernel/x86_64/scal_atom.S | 106 +- kernel/x86_64/scal_sse.S | 156 +-- kernel/x86_64/scal_sse2.S | 138 +- kernel/x86_64/sgemm_kernel_16x2_bulldozer.S | 916 ++++++------ kernel/x86_64/sgemm_kernel_16x2_piledriver.S | 916 ++++++------ kernel/x86_64/sgemm_kernel_16x4_haswell.S | 1234 ++++++++--------- kernel/x86_64/sgemm_kernel_16x4_sandy.S | 472 +++---- kernel/x86_64/sgemm_kernel_16x4_skylakex.S | 1234 ++++++++--------- kernel/x86_64/sgemm_kernel_8x4_bulldozer.S | 336 ++--- kernel/x86_64/sgemm_kernel_8x8_sandy.S | 470 +++---- kernel/x86_64/sgemv_n.S | 800 +++++------ kernel/x86_64/sgemv_t.S | 806 +++++------ kernel/x86_64/sum.S | 42 +- kernel/x86_64/swap.S | 38 +- kernel/x86_64/swap_sse.S | 160 +-- kernel/x86_64/swap_sse2.S | 82 +- kernel/x86_64/symv_L_sse.S | 98 +- kernel/x86_64/symv_L_sse2.S | 94 +- kernel/x86_64/symv_U_sse.S | 102 +- kernel/x86_64/symv_U_sse2.S | 102 +- kernel/x86_64/trsm_kernel_LN_2x8_nehalem.S | 196 +-- kernel/x86_64/trsm_kernel_LN_4x2_atom.S | 140 +- kernel/x86_64/trsm_kernel_LN_4x4_barcelona.S | 216 +-- kernel/x86_64/trsm_kernel_LN_4x4_core2.S | 274 ++-- kernel/x86_64/trsm_kernel_LN_4x4_penryn.S | 208 +-- kernel/x86_64/trsm_kernel_LN_4x4_sse2.S | 282 ++-- kernel/x86_64/trsm_kernel_LN_4x4_sse3.S | 242 ++-- kernel/x86_64/trsm_kernel_LN_4x8_nehalem.S | 276 ++-- kernel/x86_64/trsm_kernel_LN_8x4_sse.S | 316 ++--- kernel/x86_64/trsm_kernel_LT_2x8_nehalem.S | 196 +-- kernel/x86_64/trsm_kernel_LT_4x2_atom.S | 140 +- kernel/x86_64/trsm_kernel_LT_4x4_barcelona.S | 220 +-- kernel/x86_64/trsm_kernel_LT_4x4_core2.S | 276 ++-- kernel/x86_64/trsm_kernel_LT_4x4_penryn.S | 208 +-- kernel/x86_64/trsm_kernel_LT_4x4_sse2.S | 290 ++-- kernel/x86_64/trsm_kernel_LT_4x4_sse3.S | 242 ++-- kernel/x86_64/trsm_kernel_LT_4x8_nehalem.S | 276 ++-- kernel/x86_64/trsm_kernel_LT_8x4_sse.S | 316 ++--- kernel/x86_64/trsm_kernel_RT_2x8_nehalem.S | 196 +-- kernel/x86_64/trsm_kernel_RT_4x2_atom.S | 140 +- kernel/x86_64/trsm_kernel_RT_4x4_barcelona.S | 218 +-- kernel/x86_64/trsm_kernel_RT_4x4_core2.S | 274 ++-- kernel/x86_64/trsm_kernel_RT_4x4_penryn.S | 208 +-- kernel/x86_64/trsm_kernel_RT_4x4_sse2.S | 282 ++-- kernel/x86_64/trsm_kernel_RT_4x4_sse3.S | 242 ++-- kernel/x86_64/trsm_kernel_RT_4x8_nehalem.S | 276 ++-- kernel/x86_64/trsm_kernel_RT_8x4_sse.S | 316 ++--- kernel/x86_64/xdot.S | 38 +- kernel/x86_64/xgemm3m_kernel_2x2.S | 104 +- kernel/x86_64/xgemm_kernel_1x1.S | 30 +- kernel/x86_64/xgemv_n.S | 44 +- kernel/x86_64/xgemv_t.S | 44 +- kernel/x86_64/xtrsm_kernel_LT_1x1.S | 30 +- kernel/x86_64/zamax.S | 42 +- kernel/x86_64/zamax_atom.S | 52 +- kernel/x86_64/zamax_sse.S | 56 +- kernel/x86_64/zamax_sse2.S | 58 +- kernel/x86_64/zasum.S | 42 +- kernel/x86_64/zasum_atom.S | 64 +- kernel/x86_64/zasum_sse.S | 76 +- kernel/x86_64/zasum_sse2.S | 62 +- kernel/x86_64/zaxpy.S | 40 +- kernel/x86_64/zaxpy_atom.S | 48 +- kernel/x86_64/zaxpy_sse.S | 316 ++--- kernel/x86_64/zaxpy_sse2.S | 158 +-- kernel/x86_64/zcopy.S | 40 +- kernel/x86_64/zcopy_sse.S | 178 +-- kernel/x86_64/zcopy_sse2.S | 116 +- kernel/x86_64/zdot.S | 38 +- kernel/x86_64/zdot_atom.S | 46 +- kernel/x86_64/zdot_sse.S | 344 ++--- kernel/x86_64/zdot_sse2.S | 144 +- kernel/x86_64/zgemm3m_kernel_2x8_nehalem.S | 194 +-- kernel/x86_64/zgemm3m_kernel_4x2_atom.S | 138 +- kernel/x86_64/zgemm3m_kernel_4x4_barcelona.S | 290 ++-- kernel/x86_64/zgemm3m_kernel_4x4_core2.S | 258 ++-- kernel/x86_64/zgemm3m_kernel_4x4_penryn.S | 206 +-- kernel/x86_64/zgemm3m_kernel_4x4_sse2.S | 298 ++-- kernel/x86_64/zgemm3m_kernel_4x4_sse3.S | 242 ++-- kernel/x86_64/zgemm3m_kernel_4x8_nehalem.S | 274 ++-- kernel/x86_64/zgemm3m_kernel_8x4_barcelona.S | 336 ++--- kernel/x86_64/zgemm3m_kernel_8x4_core2.S | 318 ++--- kernel/x86_64/zgemm3m_kernel_8x4_penryn.S | 266 ++-- kernel/x86_64/zgemm3m_kernel_8x4_sse.S | 342 ++--- kernel/x86_64/zgemm3m_kernel_8x4_sse3.S | 340 ++--- kernel/x86_64/zgemm_beta.S | 54 +- kernel/x86_64/zgemm_kernel_1x4_nehalem.S | 88 +- kernel/x86_64/zgemm_kernel_2x1_atom.S | 52 +- kernel/x86_64/zgemm_kernel_2x2_barcelona.S | 118 +- kernel/x86_64/zgemm_kernel_2x2_bulldozer.S | 144 +- kernel/x86_64/zgemm_kernel_2x2_core2.S | 136 +- kernel/x86_64/zgemm_kernel_2x2_penryn.S | 100 +- kernel/x86_64/zgemm_kernel_2x2_piledriver.S | 144 +- kernel/x86_64/zgemm_kernel_2x2_sse2.S | 164 +-- kernel/x86_64/zgemm_kernel_2x2_sse3.S | 120 +- kernel/x86_64/zgemm_kernel_2x4_nehalem.S | 146 +- kernel/x86_64/zgemm_kernel_4x2_barcelona.S | 190 +-- kernel/x86_64/zgemm_kernel_4x2_core2.S | 172 +-- kernel/x86_64/zgemm_kernel_4x2_haswell.S | 590 ++++---- kernel/x86_64/zgemm_kernel_4x2_penryn.S | 138 +- kernel/x86_64/zgemm_kernel_4x2_sse.S | 196 +-- kernel/x86_64/zgemm_kernel_4x2_sse3.S | 194 +-- kernel/x86_64/zgemm_kernel_4x4_sandy.S | 286 ++-- kernel/x86_64/zgemm_ncopy_1.S | 24 +- kernel/x86_64/zgemm_ncopy_2.S | 42 +- kernel/x86_64/zgemm_tcopy_1.S | 20 +- kernel/x86_64/zgemm_tcopy_2.S | 44 +- kernel/x86_64/zgemv_n.S | 250 ++-- kernel/x86_64/zgemv_n_atom.S | 82 +- kernel/x86_64/zgemv_n_dup.S | 146 +- kernel/x86_64/zgemv_t.S | 204 +-- kernel/x86_64/zgemv_t_atom.S | 72 +- kernel/x86_64/zgemv_t_dup.S | 104 +- kernel/x86_64/znrm2.S | 42 +- kernel/x86_64/znrm2_sse.S | 64 +- kernel/x86_64/zrot.S | 36 +- kernel/x86_64/zrot_sse.S | 132 +- kernel/x86_64/zrot_sse2.S | 136 +- kernel/x86_64/zscal.S | 48 +- kernel/x86_64/zscal_atom.S | 66 +- kernel/x86_64/zscal_sse.S | 202 +-- kernel/x86_64/zscal_sse2.S | 226 +-- kernel/x86_64/zsum.S | 42 +- kernel/x86_64/zsum_sse.S | 76 +- kernel/x86_64/zsum_sse2.S | 62 +- kernel/x86_64/zswap.S | 38 +- kernel/x86_64/zswap_sse.S | 162 +-- kernel/x86_64/zswap_sse2.S | 144 +- kernel/x86_64/zsymv_L_sse.S | 82 +- kernel/x86_64/zsymv_L_sse2.S | 92 +- kernel/x86_64/zsymv_U_sse.S | 78 +- kernel/x86_64/zsymv_U_sse2.S | 86 +- kernel/x86_64/ztrsm_kernel_LN_2x1_atom.S | 52 +- kernel/x86_64/ztrsm_kernel_LN_2x2_core2.S | 136 +- kernel/x86_64/ztrsm_kernel_LN_2x2_penryn.S | 100 +- kernel/x86_64/ztrsm_kernel_LN_2x2_sse2.S | 144 +- kernel/x86_64/ztrsm_kernel_LN_2x2_sse3.S | 118 +- kernel/x86_64/ztrsm_kernel_LN_2x4_nehalem.S | 148 +- kernel/x86_64/ztrsm_kernel_LN_4x2_sse.S | 174 +-- kernel/x86_64/ztrsm_kernel_LT_1x4_nehalem.S | 78 +- kernel/x86_64/ztrsm_kernel_LT_2x1_atom.S | 52 +- kernel/x86_64/ztrsm_kernel_LT_2x2_core2.S | 136 +- kernel/x86_64/ztrsm_kernel_LT_2x2_penryn.S | 100 +- kernel/x86_64/ztrsm_kernel_LT_2x2_sse2.S | 144 +- kernel/x86_64/ztrsm_kernel_LT_2x2_sse3.S | 118 +- kernel/x86_64/ztrsm_kernel_LT_2x4_nehalem.S | 148 +- kernel/x86_64/ztrsm_kernel_LT_4x2_sse.S | 174 +-- kernel/x86_64/ztrsm_kernel_RT_1x4_nehalem.S | 78 +- kernel/x86_64/ztrsm_kernel_RT_2x2_core2.S | 136 +- kernel/x86_64/ztrsm_kernel_RT_2x2_penryn.S | 100 +- kernel/x86_64/ztrsm_kernel_RT_2x2_sse2.S | 144 +- kernel/x86_64/ztrsm_kernel_RT_2x2_sse3.S | 118 +- kernel/x86_64/ztrsm_kernel_RT_2x4_nehalem.S | 148 +- kernel/x86_64/ztrsm_kernel_RT_4x2_sse.S | 174 +-- 249 files changed, 23906 insertions(+), 23906 deletions(-) diff --git a/kernel/x86_64/amax.S b/kernel/x86_64/amax.S index 1498bb226c..62cc7d3a29 100644 --- a/kernel/x86_64/amax.S +++ b/kernel/x86_64/amax.S @@ -64,9 +64,9 @@ fldz testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) ffreep %st @@ -76,17 +76,17 @@ #endif addq INCX, X decq M - jle .L999 + jle L(999) cmpq $SIZE, INCX - jne .L40 + jne L(40) movq M, I sarq $3, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -166,17 +166,17 @@ addq $8 * SIZE, X decq I - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): movq M, I andq $7, I - jle .L999 + jle L(999) ALIGN_4 -.L21: +L(21): FLD 0 * SIZE(X) #ifdef USE_ABS fabs @@ -188,17 +188,17 @@ addq $1 * SIZE, X decq I - jg .L21 - jmp .L999 + jg L(21) + jmp L(999) ALIGN_4 -.L40: +L(40): movq M, I sarq $3, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): FLD 0 * SIZE(X) addq INCX, X #ifdef USE_ABS @@ -280,17 +280,17 @@ ffreep %st decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): movq M, I andq $7, I - jle .L999 + jle L(999) ALIGN_4 -.L61: +L(61): FLD 0 * SIZE(X) #ifdef USE_ABS fabs @@ -302,10 +302,10 @@ addq INCX, X decq I - jg .L61 + jg L(61) ALIGN_4 -.L999: +L(999): ret EPILOGUE diff --git a/kernel/x86_64/amax_atom.S b/kernel/x86_64/amax_atom.S index 6164cb3e33..d2c128b0fe 100644 --- a/kernel/x86_64/amax_atom.S +++ b/kernel/x86_64/amax_atom.S @@ -60,10 +60,10 @@ leaq (, INCX, SIZE), INCX testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) #ifdef USE_ABS pcmpeqb %xmm15, %xmm15 @@ -77,18 +77,18 @@ andps %xmm15, %xmm0 #endif decq M - jle .L999 + jle L(999) movaps %xmm0, %xmm1 movaps %xmm0, %xmm2 movaps %xmm0, %xmm3 cmpq $SIZE, INCX - jne .L20 + jne L(20) movq M, I sarq $3, I - jle .L15 + jle L(15) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -101,10 +101,10 @@ movsd 7 * SIZE(X), %xmm11 decq I - jle .L13 + jle L(13) ALIGN_4 -.L12: +L(12): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -159,10 +159,10 @@ addq $8 * SIZE, X decq I - jg .L12 + jg L(12) ALIGN_4 -.L13: +L(13): #ifdef USE_ABS andps %xmm15, %xmm4 #endif @@ -200,9 +200,9 @@ addq $8 * SIZE, X ALIGN_4 -.L15: +L(15): testq $4, M - jle .L17 + jle L(17) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -229,9 +229,9 @@ addq $4 * SIZE, X ALIGN_3 -.L17: +L(17): testq $2, M - jle .L18 + jle L(18) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -247,22 +247,22 @@ addq $2 * SIZE, X ALIGN_3 -.L18: +L(18): testq $1, M - jle .L998 + jle L(998) movsd 0 * SIZE(X), %xmm4 #ifdef USE_ABS andps %xmm15, %xmm4 #endif maxsd %xmm4, %xmm3 - jmp .L998 + jmp L(998) ALIGN_3 -.L20: +L(20): movq M, I sarq $3, I - jle .L25 + jle L(25) movsd (X), %xmm4 addq INCX, X @@ -282,10 +282,10 @@ movsd (X), %xmm11 decq I - jle .L23 + jle L(23) ALIGN_4 -.L22: +L(22): #ifdef USE_ABS andps %xmm15, %xmm4 #endif @@ -343,10 +343,10 @@ movsd (X), %xmm11 decq I - jg .L22 + jg L(22) ALIGN_4 -.L23: +L(23): #ifdef USE_ABS andps %xmm15, %xmm4 #endif @@ -383,9 +383,9 @@ maxsd %xmm11, %xmm3 ALIGN_4 -.L25: +L(25): testq $4, M - jle .L27 + jle L(27) movsd (X), %xmm4 addq INCX, X @@ -414,9 +414,9 @@ maxsd %xmm7, %xmm3 ALIGN_3 -.L27: +L(27): testq $2, M - jle .L28 + jle L(28) movsd (X), %xmm4 addq INCX, X @@ -433,9 +433,9 @@ maxsd %xmm5, %xmm2 ALIGN_3 -.L28: +L(28): testq $1, M - jle .L998 + jle L(998) movsd (X), %xmm4 addq INCX, X @@ -445,13 +445,13 @@ maxsd %xmm4, %xmm3 ALIGN_3 -.L998: +L(998): maxsd %xmm1, %xmm0 maxsd %xmm3, %xmm2 maxsd %xmm2, %xmm0 ALIGN_4 -.L999: +L(999): RESTOREREGISTERS diff --git a/kernel/x86_64/amax_sse.S b/kernel/x86_64/amax_sse.S index 2349905d39..c4e7471711 100644 --- a/kernel/x86_64/amax_sse.S +++ b/kernel/x86_64/amax_sse.S @@ -61,7 +61,7 @@ leaq (, INCX, SIZE), INCX testq M, M - jle .L999 + jle L(999) #ifdef USE_ABS pcmpeqb %xmm15, %xmm15 @@ -78,18 +78,18 @@ movaps %xmm0, %xmm3 addq INCX, X decq M - jle .L999 + jle L(999) cmpq $SIZE, INCX - jne .L40 + jne L(40) subq $-32 * SIZE, X cmpq $3, M - jle .L17 + jle L(17) testq $SIZE, X - je .L05 + je L(05) movss -32 * SIZE(X), %xmm1 shufps $0, %xmm1, %xmm1 @@ -100,9 +100,9 @@ addq $SIZE, X ALIGN_3 -.L05: +L(05): testq $2 * SIZE, X - je .L06 + je L(06) movsd -32 * SIZE(X), %xmm2 unpcklps %xmm2, %xmm2 @@ -113,10 +113,10 @@ addq $2 * SIZE, X ALIGN_3 -.L06: +L(06): movq M, I sarq $5, I - jle .L15 + jle L(15) movaps -32 * SIZE(X), %xmm4 movaps -28 * SIZE(X), %xmm5 @@ -124,10 +124,10 @@ movaps -20 * SIZE(X), %xmm7 decq I - jle .L12 + jle L(12) ALIGN_4 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -186,10 +186,10 @@ subq $-32 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_4 -.L12: +L(12): #ifdef USE_ABS andps %xmm15, %xmm4 #endif @@ -238,9 +238,9 @@ ALIGN_3 -.L15: +L(15): testq $16, M - je .L16 + je L(16) movaps -32 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -269,9 +269,9 @@ addq $16 * SIZE, X ALIGN_3 -.L16: +L(16): testq $8, M - je .L17 + je L(17) movaps -32 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -287,9 +287,9 @@ addq $8 * SIZE, X ALIGN_3 -.L17: +L(17): testq $4, M - je .L18 + je L(18) movaps -32 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -299,9 +299,9 @@ addq $4 * SIZE, X ALIGN_3 -.L18: +L(18): testq $2, M - je .L19 + je L(19) movsd -32 * SIZE(X), %xmm4 unpcklps %xmm4, %xmm4 @@ -312,25 +312,25 @@ addq $2 * SIZE, X ALIGN_3 -.L19: +L(19): testq $1, M - je .L998 + je L(998) movss -32 * SIZE(X), %xmm4 #ifdef USE_ABS andps %xmm15, %xmm4 #endif maxss %xmm4, %xmm0 - jmp .L998 + jmp L(998) ALIGN_3 -.L40: +L(40): movq M, I sarq $3, I - jle .L45 + jle L(45) ALIGN_4 -.L41: +L(41): movss (X), %xmm4 addq INCX, X #ifdef USE_ABS @@ -388,12 +388,12 @@ maxss %xmm7, %xmm3 decq I - jg .L41 + jg L(41) ALIGN_4 -.L45: +L(45): testq $4, M - je .L46 + je L(46) movss (X), %xmm4 addq INCX, X @@ -424,9 +424,9 @@ maxss %xmm7, %xmm3 ALIGN_3 -.L46: +L(46): testq $2, M - je .L47 + je L(47) movss (X), %xmm4 addq INCX, X @@ -443,9 +443,9 @@ maxss %xmm5, %xmm1 ALIGN_3 -.L47: +L(47): testq $1, M - je .L998 + je L(998) movss (X), %xmm4 addq INCX, X @@ -455,7 +455,7 @@ maxss %xmm4, %xmm2 ALIGN_4 -.L998: +L(998): maxps %xmm1, %xmm0 maxps %xmm3, %xmm2 maxps %xmm2, %xmm0 @@ -467,7 +467,7 @@ maxss %xmm1, %xmm0 ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/amax_sse2.S b/kernel/x86_64/amax_sse2.S index 44ddaba778..5893b6a292 100644 --- a/kernel/x86_64/amax_sse2.S +++ b/kernel/x86_64/amax_sse2.S @@ -61,7 +61,7 @@ leaq (, INCX, SIZE), INCX testq M, M - jle .L999 + jle L(999) #ifdef USE_ABS pcmpeqb %xmm15, %xmm15 @@ -78,15 +78,15 @@ movaps %xmm0, %xmm2 movaps %xmm0, %xmm3 decq M - jle .L999 + jle L(999) cmpq $SIZE, INCX - jne .L40 + jne L(40) subq $-16 * SIZE, X testq $SIZE, X - je .L05 + je L(05) movsd -16 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -96,13 +96,13 @@ maxpd %xmm4, %xmm3 addq $SIZE, X decq M - jle .L998 + jle L(998) ALIGN_3 -.L05: +L(05): movq M, I sarq $4, I - jle .L15 + jle L(15) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -110,10 +110,10 @@ movaps -10 * SIZE(X), %xmm7 decq I - jle .L12 + jle L(12) ALIGN_4 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -172,10 +172,10 @@ subq $-16 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_4 -.L12: +L(12): #ifdef USE_ABS andps %xmm15, %xmm4 #endif @@ -223,9 +223,9 @@ subq $-16 * SIZE, X ALIGN_4 -.L15: +L(15): testq $8, M - jle .L16 + jle L(16) movaps -16 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -253,9 +253,9 @@ addq $8 * SIZE, X ALIGN_3 -.L16: +L(16): testq $4, M - jle .L17 + jle L(17) movaps -16 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -272,9 +272,9 @@ addq $4 * SIZE, X ALIGN_3 -.L17: +L(17): testq $2, M - jle .L18 + jle L(18) movaps -16 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -284,9 +284,9 @@ addq $2 * SIZE, X ALIGN_3 -.L18: +L(18): testq $1, M - jle .L998 + jle L(998) movsd -16 * SIZE(X), %xmm4 unpcklpd %xmm4, %xmm4 @@ -294,16 +294,16 @@ andps %xmm15, %xmm4 #endif maxpd %xmm4, %xmm3 - jmp .L998 + jmp L(998) ALIGN_3 -.L40: +L(40): movq M, I sarq $4, I - jle .L45 + jle L(45) ALIGN_4 -.L41: +L(41): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -385,15 +385,15 @@ maxpd %xmm7, %xmm3 decq I - jg .L41 + jg L(41) ALIGN_4 -.L45: +L(45): andq $15, M - jle .L998 + jle L(998) testq $8, M - je .L46 + je L(46) movsd (X), %xmm4 addq INCX, X @@ -432,9 +432,9 @@ maxpd %xmm7, %xmm3 ALIGN_3 -.L46: +L(46): testq $4, M - je .L47 + je L(47) movsd (X), %xmm4 addq INCX, X @@ -455,9 +455,9 @@ maxpd %xmm5, %xmm1 ALIGN_3 -.L47: +L(47): testq $2, M - je .L48 + je L(48) movsd (X), %xmm6 addq INCX, X @@ -469,9 +469,9 @@ maxpd %xmm6, %xmm2 ALIGN_3 -.L48: +L(48): testq $1, M - je .L998 + je L(998) movsd (X), %xmm7 unpcklpd %xmm7, %xmm7 @@ -481,7 +481,7 @@ maxpd %xmm7, %xmm3 ALIGN_4 -.L998: +L(998): maxpd %xmm1, %xmm0 maxpd %xmm3, %xmm2 maxpd %xmm2, %xmm0 @@ -490,7 +490,7 @@ maxsd %xmm1, %xmm0 ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/asum.S b/kernel/x86_64/asum.S index a2cbfd4804..11d5239c45 100644 --- a/kernel/x86_64/asum.S +++ b/kernel/x86_64/asum.S @@ -56,9 +56,9 @@ fldz testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) salq $BASE_SHIFT, INCX @@ -66,14 +66,14 @@ fldz fldz cmpq $SIZE, INCX - jne .L40 + jne L(40) movq M, I sarq $3, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -109,31 +109,31 @@ faddp %st, %st(1) decq I - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): andq $7, M - jle .L998 + jle L(998) ALIGN_4 -.L21: +L(21): FLD (X) fabs faddp %st,%st(1) addq $1 * SIZE, X decq M - jg .L21 - jmp .L998 + jg L(21) + jmp L(998) ALIGN_4 -.L40: +L(40): movq M, I sarq $3, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): FLD (X) addq INCX, X fabs @@ -171,31 +171,31 @@ faddp %st, %st(1) decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): andq $7, M - jle .L998 + jle L(998) ALIGN_4 -.L61: +L(61): FLD (X) addq INCX, X fabs faddp %st,%st(1) decq M - jg .L61 + jg L(61) ALIGN_4 -.L998: +L(998): faddp %st,%st(2) faddp %st,%st(1) faddp %st,%st(1) ALIGN_4 -.L999: +L(999): ret EPILOGUE diff --git a/kernel/x86_64/asum_atom.S b/kernel/x86_64/asum_atom.S index 910a48f09e..7dde619511 100644 --- a/kernel/x86_64/asum_atom.S +++ b/kernel/x86_64/asum_atom.S @@ -54,9 +54,9 @@ xorps %xmm0, %xmm0 testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) xorps %xmm1, %xmm1 xorps %xmm2, %xmm2 @@ -69,24 +69,24 @@ xorps %xmm13, %xmm13 cmpq $SIZE, INCX - jne .L20 + jne L(20) testq $SIZE, X - je .L05 + je L(05) movsd (X), %xmm0 addq $SIZE, X andps %xmm15, %xmm0 decq M - jle .L999 + jle L(999) ALIGN_3 -.L05: +L(05): subq $-16 * SIZE, X movq M, I sarq $4, I - jle .L12 + jle L(12) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -99,10 +99,10 @@ movaps -2 * SIZE(X), %xmm11 decq I - jle .L11 + jle L(11) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -161,10 +161,10 @@ subq $-16 * SIZE, X decq I - jg .L10 + jg L(10) ALIGN_4 -.L11: +L(11): andps %xmm15, %xmm4 addsd %xmm13, %xmm3 pshufd $0x4e, %xmm4, %xmm12 @@ -209,12 +209,12 @@ subq $-16 * SIZE, X ALIGN_3 -.L12: +L(12): andq $15, M - jle .L998 + jle L(998) testq $8, M - je .L13 + je L(13) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -240,9 +240,9 @@ addsd %xmm13, %xmm3 ALIGN_3 -.L13: +L(13): testq $4, M - je .L14 + je L(14) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -258,9 +258,9 @@ addsd %xmm13, %xmm3 ALIGN_3 -.L14: +L(14): testq $2, M - je .L15 + je L(15) movaps -16 * SIZE(X), %xmm4 addq $2 * SIZE, X @@ -271,20 +271,20 @@ addsd %xmm5, %xmm3 ALIGN_3 -.L15: +L(15): testq $1, M - je .L998 + je L(998) movsd -16 * SIZE(X), %xmm4 andps %xmm15, %xmm4 addsd %xmm4, %xmm0 - jmp .L998 + jmp L(998) ALIGN_3 -.L20: +L(20): movq M, I sarq $3, I - jle .L25 + jle L(25) movsd (X), %xmm4 addq INCX, X @@ -304,10 +304,10 @@ movsd (X), %xmm11 decq I - jle .L23 + jle L(23) ALIGN_4 -.L22: +L(22): andps %xmm15, %xmm4 addq INCX, X addsd %xmm4, %xmm0 @@ -343,10 +343,10 @@ movsd (X), %xmm11 decq I - jg .L22 + jg L(22) ALIGN_4 -.L23: +L(23): andps %xmm15, %xmm4 addq INCX, X addsd %xmm4, %xmm0 @@ -367,12 +367,12 @@ addsd %xmm11, %xmm3 ALIGN_3 -.L25: +L(25): andq $7, M - jle .L998 + jle L(998) testq $4, M - je .L26 + je L(26) movsd (X), %xmm4 addq INCX, X @@ -393,9 +393,9 @@ addsd %xmm7, %xmm3 ALIGN_3 -.L26: +L(26): testq $2, M - je .L27 + je L(27) movsd (X), %xmm4 addq INCX, X @@ -409,22 +409,22 @@ addsd %xmm5, %xmm1 ALIGN_3 -.L27: +L(27): testq $1, M - je .L998 + je L(998) movsd (X), %xmm4 andps %xmm15, %xmm4 addsd %xmm4, %xmm0 ALIGN_3 -.L998: +L(998): addsd %xmm1, %xmm0 addsd %xmm3, %xmm2 addsd %xmm2, %xmm0 ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/asum_sse.S b/kernel/x86_64/asum_sse.S index 7d7004d486..5519db6183 100644 --- a/kernel/x86_64/asum_sse.S +++ b/kernel/x86_64/asum_sse.S @@ -54,9 +54,9 @@ xorps %xmm0, %xmm0 testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) xorps %xmm1, %xmm1 xorps %xmm2, %xmm2 @@ -68,37 +68,37 @@ leaq (, INCX, SIZE), INCX cmpq $SIZE, INCX - jne .L100 + jne L(100) subq $-32 * SIZE, X cmpq $3, M - jle .L18 + jle L(18) testq $4, X - je .L05 + je L(05) movss -32 * SIZE(X), %xmm0 andps %xmm15, %xmm0 addq $SIZE, X decq M - jle .L998 + jle L(998) ALIGN_3 -.L05: +L(05): testq $8, X - je .L10 + je L(10) movsd -32 * SIZE(X), %xmm1 andps %xmm15, %xmm1 addq $2 * SIZE, X subq $2, M - jle .L998 + jle L(998) ALIGN_3 -.L10: +L(10): movq M, I sarq $5, I - jle .L14 + jle L(14) movaps -32 * SIZE(X), %xmm4 movaps -28 * SIZE(X), %xmm5 @@ -110,10 +110,10 @@ movaps -8 * SIZE(X), %xmm10 movaps -4 * SIZE(X), %xmm11 decq I - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -156,10 +156,10 @@ subq $-32 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): andps %xmm15, %xmm4 addps %xmm4, %xmm0 andps %xmm15, %xmm5 @@ -183,9 +183,9 @@ subq $-32 * SIZE, X ALIGN_3 -.L14: +L(14): testq $16, M - je .L16 + je L(16) movaps -32 * SIZE(X), %xmm4 andps %xmm15, %xmm4 @@ -206,9 +206,9 @@ addq $16 * SIZE, X ALIGN_3 -.L16: +L(16): testq $8, M - je .L17 + je L(17) movaps -32 * SIZE(X), %xmm4 andps %xmm15, %xmm4 @@ -221,9 +221,9 @@ addq $8 * SIZE, X ALIGN_3 -.L17: +L(17): testq $4, M - je .L18 + je L(18) movaps -32 * SIZE(X), %xmm6 andps %xmm15, %xmm6 @@ -231,9 +231,9 @@ addq $4 * SIZE, X ALIGN_3 -.L18: +L(18): testq $2, M - je .L19 + je L(19) #ifdef movsd xorps %xmm7, %xmm7 @@ -244,23 +244,23 @@ addq $2 * SIZE, X ALIGN_3 -.L19: +L(19): testq $1, M - je .L998 + je L(998) movss -32 * SIZE(X), %xmm6 andps %xmm15, %xmm6 addps %xmm6, %xmm2 - jmp .L998 + jmp L(998) ALIGN_4 -.L100: +L(100): movq M, I sarq $3, I - jle .L105 + jle L(105) ALIGN_4 -.L101: +L(101): movss 0 * SIZE(X), %xmm4 addq INCX, X andps %xmm15, %xmm4 @@ -302,24 +302,24 @@ addss %xmm6, %xmm3 decq I - jg .L101 + jg L(101) ALIGN_4 -.L105: +L(105): andq $7, M - jle .L998 + jle L(998) ALIGN_4 -.L106: +L(106): movss 0 * SIZE(X), %xmm4 andps %xmm15, %xmm4 addps %xmm4, %xmm0 addq INCX, X decq M - jg .L106 + jg L(106) ALIGN_4 -.L998: +L(998): addps %xmm1, %xmm0 addps %xmm3, %xmm2 addps %xmm2, %xmm0 @@ -337,7 +337,7 @@ #endif ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/asum_sse2.S b/kernel/x86_64/asum_sse2.S index e75ebde137..35bcc1d8f7 100644 --- a/kernel/x86_64/asum_sse2.S +++ b/kernel/x86_64/asum_sse2.S @@ -54,9 +54,9 @@ xorps %xmm0, %xmm0 testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) xorps %xmm1, %xmm1 xorps %xmm2, %xmm2 @@ -70,23 +70,23 @@ subq $-16 * SIZE, X cmpq $SIZE, INCX - jne .L40 + jne L(40) testq $SIZE, X - je .L05 + je L(05) movsd -16 * SIZE(X), %xmm0 addq $SIZE, X andps %xmm15, %xmm0 subq $1, M - jle .L999 + jle L(999) ALIGN_3 -.L05: +L(05): movq M, I sarq $4, I - jle .L20 + jle L(20) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -99,10 +99,10 @@ movaps -2 * SIZE(X), %xmm11 decq I - jle .L11 + jle L(11) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -145,10 +145,10 @@ subq $-16 * SIZE, X decq I - jg .L10 + jg L(10) ALIGN_4 -.L11: +L(11): andps %xmm15, %xmm4 andps %xmm15, %xmm5 andps %xmm15, %xmm6 @@ -172,12 +172,12 @@ subq $-16 * SIZE, X ALIGN_3 -.L20: +L(20): andq $15, M - jle .L998 + jle L(998) testq $8, M - je .L21 + je L(21) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -196,9 +196,9 @@ addq $8 * SIZE, X ALIGN_3 -.L21: +L(21): testq $4, M - je .L22 + je L(22) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -211,18 +211,18 @@ addq $4 * SIZE, X ALIGN_3 -.L22: +L(22): testq $2, M - je .L23 + je L(23) movaps -16 * SIZE(X), %xmm6 andps %xmm15, %xmm6 addpd %xmm6, %xmm3 addq $2 * SIZE, X -.L23: +L(23): testq $1, M - je .L998 + je L(998) #ifdef movsd xorps %xmm4, %xmm4 @@ -230,16 +230,16 @@ movsd -16 * SIZE(X), %xmm4 andps %xmm15, %xmm4 addsd %xmm4, %xmm0 - jmp .L998 + jmp L(998) ALIGN_3 -.L40: +L(40): movq M, I sarq $3, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): movsd -16 * SIZE(X), %xmm4 addq INCX, X movhpd -16 * SIZE(X), %xmm4 @@ -269,33 +269,33 @@ addpd %xmm7, %xmm3 decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): #ifdef movsd xorps %xmm4, %xmm4 #endif andq $7, M - jle .L998 + jle L(998) ALIGN_4 -.L61: +L(61): movsd -16 * SIZE(X), %xmm4 andps %xmm15, %xmm4 addpd %xmm4, %xmm0 addq INCX, X decq M - jg .L61 + jg L(61) ALIGN_4 -.L998: +L(998): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 ALIGN_4 -.L999: +L(999): #ifndef HAVE_SSE3 movhlps %xmm0, %xmm1 addsd %xmm1, %xmm0 diff --git a/kernel/x86_64/axpy.S b/kernel/x86_64/axpy.S index 0ad6591b0f..6884122513 100644 --- a/kernel/x86_64/axpy.S +++ b/kernel/x86_64/axpy.S @@ -60,21 +60,21 @@ salq $BASE_SHIFT, INCY testq M, M - jle .L40 + jle L(40) cmpq $SIZE, INCX - jne .L14 + jne L(14) cmpq $SIZE, INCY - jne .L14 + jne L(14) movq M, %rax sarq $3, %rax - jle .L15 + jle L(15) ALIGN_3 #define PRESIZE 33 -.L16: +L(16): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -134,16 +134,16 @@ addq $8 * SIZE, X addq $8 * SIZE, Y decq %rax - jg .L16 + jg L(16) ALIGN_3 -.L15: +L(15): movq M, %rax andq $7, %rax - jle .L40 + jle L(40) ALIGN_3 -.L22: +L(22): FLD 0 * SIZE(X) fmul %st(1),%st FLD 0 * SIZE(Y) @@ -152,17 +152,17 @@ addq $SIZE, X addq $SIZE, Y decq %rax - jg .L22 - jmp .L40 + jg L(22) + jmp L(40) ALIGN_3 -.L14: +L(14): movq M, %rax sarq $2, %rax - jle .L28 + jle L(28) ALIGN_3 -.L29: +L(29): FLD (X) fmul %st(1),%st FLD (Y) @@ -196,16 +196,16 @@ addq INCY, Y decq %rax - jg .L29 + jg L(29) ALIGN_3 -.L28: +L(28): movq M, %rax andq $3, %rax - jle .L40 + jle L(40) ALIGN_3 -.L35: +L(35): FLD (X) fmul %st(1),%st FLD (Y) @@ -215,9 +215,9 @@ addq INCY, Y decq %rax - jg .L35 + jg L(35) -.L40: +L(40): ffreep %st(0) ret diff --git a/kernel/x86_64/axpy_atom.S b/kernel/x86_64/axpy_atom.S index adfd691641..565b19adc9 100644 --- a/kernel/x86_64/axpy_atom.S +++ b/kernel/x86_64/axpy_atom.S @@ -83,16 +83,16 @@ leaq (, INCY, SIZE), INCY testq M, M - jle .L29 + jle L(29) cmpq $SIZE, INCX - jne .L20 + jne L(20) cmpq $SIZE, INCY - jne .L20 + jne L(20) movq M, %rax sarq $3, %rax - jle .L13 + jle L(13) movsd 0 * SIZE(X), %xmm0 movsd 1 * SIZE(X), %xmm1 @@ -114,10 +114,10 @@ mulsd ALPHA, %xmm3 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -179,10 +179,10 @@ addq $8 * SIZE, Y addq $8 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): addsd %xmm4, %xmm0 movsd 4 * SIZE(Y), %xmm4 addsd %xmm5, %xmm1 @@ -215,10 +215,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L13: +L(13): movq M, %rax andq $4, %rax - jle .L15 + jle L(15) ALIGN_3 movsd 0 * SIZE(X), %xmm0 @@ -249,10 +249,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L15: +L(15): movq M, %rax andq $2, %rax - jle .L16 + jle L(16) ALIGN_3 movsd 0 * SIZE(X), %xmm0 @@ -272,10 +272,10 @@ addq $2 * SIZE, Y ALIGN_3 -.L16: +L(16): movq M, %rax andq $1, %rax - jle .L19 + jle L(19) ALIGN_3 movsd 0 * SIZE(X), %xmm0 @@ -286,7 +286,7 @@ addq $SIZE, Y ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -294,12 +294,12 @@ ret ALIGN_3 -.L20: +L(20): movq Y, YY movq M, %rax sarq $3, %rax - jle .L23 + jle L(23) movsd (X), %xmm0 addq INCX, X @@ -333,10 +333,10 @@ mulsd ALPHA, %xmm3 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): addsd %xmm4, %xmm0 movsd (Y), %xmm4 addq INCY, Y @@ -414,10 +414,10 @@ mulsd ALPHA, %xmm3 decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): addsd %xmm4, %xmm0 movsd (Y), %xmm4 addq INCY, Y @@ -462,10 +462,10 @@ addq INCY, YY ALIGN_3 -.L23: +L(23): movq M, %rax andq $4, %rax - jle .L25 + jle L(25) ALIGN_3 movsd (X), %xmm0 @@ -505,10 +505,10 @@ addq INCY, YY ALIGN_3 -.L25: +L(25): movq M, %rax andq $2, %rax - jle .L26 + jle L(26) ALIGN_3 movsd (X), %xmm0 @@ -531,10 +531,10 @@ addq INCY, YY ALIGN_3 -.L26: +L(26): movq M, %rax andq $1, %rax - jle .L29 + jle L(29) ALIGN_3 movsd (X), %xmm0 @@ -545,7 +545,7 @@ addq $SIZE, Y ALIGN_3 -.L29: +L(29): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/axpy_sse.S b/kernel/x86_64/axpy_sse.S index dd52a7c830..52367954ab 100644 --- a/kernel/x86_64/axpy_sse.S +++ b/kernel/x86_64/axpy_sse.S @@ -89,21 +89,21 @@ leaq (, INCY, SIZE), INCY testq M, M - jle .L19 + jle L(19) cmpq $SIZE, INCX - jne .L50 + jne L(50) cmpq $SIZE, INCY - jne .L50 + jne L(50) subq $-32 * SIZE, X subq $-32 * SIZE, Y cmpq $3, M - jle .L16 + jle L(16) testq $SIZE, Y - je .L00 + je L(00) movss -32 * SIZE(X), %xmm0 mulss ALPHA, %xmm0 @@ -112,12 +112,12 @@ addq $1 * SIZE, X addq $1 * SIZE, Y decq M - jle .L19 + jle L(19) ALIGN_3 -.L00: +L(00): testq $SIZE * 2, Y - je .L10 + je L(10) movsd -32 * SIZE(X), %xmm0 movsd -32 * SIZE(Y), %xmm4 @@ -128,16 +128,16 @@ addq $2 * SIZE, X addq $2 * SIZE, Y subq $2, M - jle .L19 + jle L(19) ALIGN_3 -.L10: +L(10): testq $SIZE * 3, X - jne .L20 + jne L(20) movq M, %rax sarq $5, %rax - jle .L13 + jle L(13) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -145,10 +145,10 @@ movaps -20 * SIZE(X), %xmm3 decq %rax - jle .L12 + jle L(12) ALIGN_4 -.L11: +L(11): movaps -16 * SIZE(X), %xmm4 movaps -12 * SIZE(X), %xmm5 @@ -212,10 +212,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movaps -16 * SIZE(X), %xmm4 movaps -12 * SIZE(X), %xmm5 @@ -258,10 +258,10 @@ subq $-32 * SIZE, Y ALIGN_3 -.L13: +L(13): movq M, %rax andq $16, %rax - jle .L14 + jle L(14) ALIGN_3 movaps -32 * SIZE(X), %xmm0 @@ -287,10 +287,10 @@ addq $16 * SIZE, Y ALIGN_3 -.L14: +L(14): movq M, %rax andq $8, %rax - jle .L15 + jle L(15) ALIGN_3 movaps -32 * SIZE(X), %xmm0 @@ -308,10 +308,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L15: +L(15): movq M, %rax andq $4, %rax - jle .L16 + jle L(16) ALIGN_3 movaps -32 * SIZE(X), %xmm0 @@ -326,10 +326,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L16: +L(16): movq M, %rax andq $2, %rax - jle .L17 + jle L(17) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -344,10 +344,10 @@ addq $2 * SIZE, Y ALIGN_3 -.L17: +L(17): movq M, %rax andq $1, %rax - jle .L19 + jle L(19) ALIGN_3 movss -32 * SIZE(X), %xmm0 @@ -357,7 +357,7 @@ movss %xmm0, -32 * SIZE(Y) ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -365,18 +365,18 @@ ret ALIGN_3 -.L20: +L(20): #ifdef ALIGNED_ACCESS testq $SIZE, X - jne .L30 + jne L(30) movhps -32 * SIZE(X), %xmm0 movq M, %rax sarq $5, %rax - jle .L23 + jle L(23) movaps -30 * SIZE(X), %xmm1 movaps -26 * SIZE(X), %xmm2 @@ -384,10 +384,10 @@ movaps -18 * SIZE(X), %xmm4 decq %rax - jle .L22 + jle L(22) ALIGN_4 -.L21: +L(21): movaps -14 * SIZE(X), %xmm5 movaps -10 * SIZE(X), %xmm6 @@ -459,10 +459,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): movaps -14 * SIZE(X), %xmm5 movaps -10 * SIZE(X), %xmm6 @@ -513,10 +513,10 @@ subq $-32 * SIZE, Y ALIGN_3 -.L23: +L(23): movq M, %rax andq $16, %rax - jle .L24 + jle L(24) ALIGN_3 movaps -30 * SIZE(X), %xmm1 @@ -549,10 +549,10 @@ addq $16 * SIZE, Y ALIGN_3 -.L24: +L(24): movq M, %rax andq $8, %rax - jle .L25 + jle L(25) ALIGN_3 movaps -30 * SIZE(X), %xmm1 @@ -573,10 +573,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L25: +L(25): movq M, %rax andq $4, %rax - jle .L26 + jle L(26) ALIGN_3 movaps -30 * SIZE(X), %xmm1 @@ -592,10 +592,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L26: +L(26): movq M, %rax andq $2, %rax - jle .L27 + jle L(27) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -610,10 +610,10 @@ addq $2 * SIZE, Y ALIGN_3 -.L27: +L(27): movq M, %rax andq $1, %rax - jle .L29 + jle L(29) ALIGN_3 movss -32 * SIZE(X), %xmm0 @@ -624,7 +624,7 @@ addq $SIZE, Y ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -632,15 +632,15 @@ ret ALIGN_3 -.L30: +L(30): testq $2 * SIZE, X - jne .L40 + jne L(40) movaps -33 * SIZE(X), %xmm0 movq M, %rax sarq $5, %rax - jle .L33 + jle L(33) movaps -29 * SIZE(X), %xmm1 movaps -25 * SIZE(X), %xmm2 @@ -648,10 +648,10 @@ movaps -17 * SIZE(X), %xmm4 decq %rax - jle .L32 + jle L(32) ALIGN_4 -.L31: +L(31): movaps -13 * SIZE(X), %xmm5 movaps -9 * SIZE(X), %xmm6 @@ -731,10 +731,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L32: +L(32): movaps -13 * SIZE(X), %xmm5 movaps -9 * SIZE(X), %xmm6 @@ -793,10 +793,10 @@ subq $-32 * SIZE, Y ALIGN_3 -.L33: +L(33): movq M, %rax andq $16, %rax - jle .L34 + jle L(34) ALIGN_3 movaps -29 * SIZE(X), %xmm1 @@ -835,10 +835,10 @@ addq $16 * SIZE, Y ALIGN_3 -.L34: +L(34): movq M, %rax andq $8, %rax - jle .L35 + jle L(35) ALIGN_3 movaps -29 * SIZE(X), %xmm1 @@ -862,10 +862,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L35: +L(35): movq M, %rax andq $4, %rax - jle .L36 + jle L(36) ALIGN_3 movaps -29 * SIZE(X), %xmm1 @@ -882,10 +882,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L36: +L(36): movq M, %rax andq $2, %rax - jle .L37 + jle L(37) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -900,10 +900,10 @@ addq $2 * SIZE, Y ALIGN_3 -.L37: +L(37): movq M, %rax andq $1, %rax - jle .L39 + jle L(39) ALIGN_3 movss -32 * SIZE(X), %xmm0 @@ -914,7 +914,7 @@ addq $SIZE, Y ALIGN_3 -.L39: +L(39): xorq %rax,%rax RESTOREREGISTERS @@ -922,12 +922,12 @@ ret ALIGN_3 -.L40: +L(40): movaps -35 * SIZE(X), %xmm0 movq M, %rax sarq $5, %rax - jle .L43 + jle L(43) movaps -31 * SIZE(X), %xmm1 movaps -27 * SIZE(X), %xmm2 @@ -935,10 +935,10 @@ movaps -19 * SIZE(X), %xmm4 decq %rax - jle .L42 + jle L(42) ALIGN_4 -.L41: +L(41): movaps -15 * SIZE(X), %xmm5 movaps -11 * SIZE(X), %xmm6 @@ -1018,10 +1018,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L42: +L(42): movaps -15 * SIZE(X), %xmm5 movaps -11 * SIZE(X), %xmm6 @@ -1080,10 +1080,10 @@ subq $-32 * SIZE, Y ALIGN_3 -.L43: +L(43): movq M, %rax andq $16, %rax - jle .L44 + jle L(44) ALIGN_3 movaps -31 * SIZE(X), %xmm1 @@ -1120,10 +1120,10 @@ addq $16 * SIZE, Y ALIGN_3 -.L44: +L(44): movq M, %rax andq $8, %rax - jle .L45 + jle L(45) ALIGN_3 movaps -31 * SIZE(X), %xmm1 @@ -1146,10 +1146,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L45: +L(45): movq M, %rax andq $4, %rax - jle .L46 + jle L(46) ALIGN_3 movaps -31 * SIZE(X), %xmm1 @@ -1166,10 +1166,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L46: +L(46): movq M, %rax andq $2, %rax - jle .L47 + jle L(47) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -1184,10 +1184,10 @@ addq $2 * SIZE, Y ALIGN_3 -.L47: +L(47): movq M, %rax andq $1, %rax - jle .L49 + jle L(49) ALIGN_3 movss -32 * SIZE(X), %xmm0 @@ -1198,7 +1198,7 @@ addq $SIZE, Y ALIGN_3 -.L49: +L(49): xorq %rax,%rax RESTOREREGISTERS @@ -1209,7 +1209,7 @@ movq M, %rax sarq $5, %rax - jle .L23 + jle L(23) movsd -32 * SIZE(X), %xmm0 movhps -30 * SIZE(X), %xmm0 @@ -1221,10 +1221,10 @@ movhps -18 * SIZE(X), %xmm3 decq %rax - jle .L22 + jle L(22) ALIGN_4 -.L21: +L(21): movsd -16 * SIZE(X), %xmm4 movhps -14 * SIZE(X), %xmm4 movsd -12 * SIZE(X), %xmm5 @@ -1296,10 +1296,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): movsd -16 * SIZE(X), %xmm4 movhps -14 * SIZE(X), %xmm4 movsd -12 * SIZE(X), %xmm5 @@ -1346,10 +1346,10 @@ subq $-32 * SIZE, Y ALIGN_3 -.L23: +L(23): movq M, %rax andq $16, %rax - jle .L24 + jle L(24) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -1380,10 +1380,10 @@ addq $16 * SIZE, Y ALIGN_3 -.L24: +L(24): movq M, %rax andq $8, %rax - jle .L25 + jle L(25) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -1403,10 +1403,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L25: +L(25): movq M, %rax andq $4, %rax - jle .L26 + jle L(26) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -1422,10 +1422,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L26: +L(26): movq M, %rax andq $2, %rax - jle .L27 + jle L(27) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -1440,10 +1440,10 @@ addq $2 * SIZE, Y ALIGN_3 -.L27: +L(27): movq M, %rax andq $1, %rax - jle .L29 + jle L(29) ALIGN_3 movss -32 * SIZE(X), %xmm0 @@ -1454,7 +1454,7 @@ addq $SIZE, Y ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -1464,20 +1464,20 @@ ALIGN_3 -.L50: +L(50): movq M, %rax movq Y, YY //If incx==0 || incy==0, avoid unloop. cmpq $0, INCX - je .L56 + je L(56) cmpq $0, INCY - je .L56 + je L(56) sarq $3, %rax - jle .L55 + jle L(55) ALIGN_3 -.L51: +L(51): movss (X), %xmm0 addq INCX, X mulss ALPHA, %xmm0 @@ -1553,16 +1553,16 @@ addq INCY, Y decq %rax - jg .L51 + jg L(51) ALIGN_3 -.L55: +L(55): movq M, %rax andq $7, %rax - jle .L59 + jle L(59) ALIGN_3 -.L56: +L(56): movss (X), %xmm0 addq INCX, X mulss ALPHA, %xmm0 @@ -1571,10 +1571,10 @@ movss %xmm0, (Y) addq INCY, Y decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L59: +L(59): xorq %rax,%rax RESTOREREGISTERS diff --git a/kernel/x86_64/axpy_sse2.S b/kernel/x86_64/axpy_sse2.S index 9b07b90204..19c6980d4a 100644 --- a/kernel/x86_64/axpy_sse2.S +++ b/kernel/x86_64/axpy_sse2.S @@ -88,15 +88,15 @@ leaq (, INCY, SIZE), INCY testq M, M - jle .L47 + jle L(47) cmpq $SIZE, INCX - jne .L40 + jne L(40) cmpq $SIZE, INCY - jne .L40 + jne L(40) testq $SIZE, Y - je .L10 + je L(10) movsd (X), %xmm0 mulsd ALPHA, %xmm0 @@ -105,19 +105,19 @@ addq $1 * SIZE, X addq $1 * SIZE, Y decq M - jle .L19 + jle L(19) ALIGN_4 -.L10: +L(10): subq $-16 * SIZE, X subq $-16 * SIZE, Y testq $SIZE, X - jne .L20 + jne L(20) movq M, %rax sarq $4, %rax - jle .L13 + jle L(13) movaps -16 * SIZE(X), %xmm0 movaps -14 * SIZE(X), %xmm1 @@ -125,10 +125,10 @@ movaps -10 * SIZE(X), %xmm3 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): movaps -8 * SIZE(X), %xmm4 movaps -6 * SIZE(X), %xmm5 @@ -192,10 +192,10 @@ subq $-16 * SIZE, Y subq $-16 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movaps -8 * SIZE(X), %xmm4 movaps -6 * SIZE(X), %xmm5 @@ -238,10 +238,10 @@ subq $-16 * SIZE, X ALIGN_3 -.L13: +L(13): movq M, %rax andq $8, %rax - jle .L14 + jle L(14) ALIGN_3 movaps -16 * SIZE(X), %xmm0 @@ -267,10 +267,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L14: +L(14): movq M, %rax andq $4, %rax - jle .L15 + jle L(15) ALIGN_3 movaps -16 * SIZE(X), %xmm0 @@ -289,10 +289,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L15: +L(15): movq M, %rax andq $2, %rax - jle .L16 + jle L(16) ALIGN_3 movaps -16 * SIZE(X), %xmm0 @@ -304,10 +304,10 @@ addq $2 * SIZE, Y ALIGN_3 -.L16: +L(16): movq M, %rax andq $1, %rax - jle .L19 + jle L(19) ALIGN_3 movsd -16 * SIZE(X), %xmm0 @@ -317,7 +317,7 @@ movsd %xmm0, -16 * SIZE(Y) ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -325,24 +325,24 @@ ret ALIGN_3 -.L20: +L(20): #ifdef ALIGNED_ACCESS movhps -16 * SIZE(X), %xmm0 movq M, %rax sarq $4, %rax - jle .L23 + jle L(23) movaps -15 * SIZE(X), %xmm1 movaps -13 * SIZE(X), %xmm2 movaps -11 * SIZE(X), %xmm3 decq %rax - jle .L22 + jle L(22) ALIGN_4 -.L21: +L(21): movaps -9 * SIZE(X), %xmm4 movaps -7 * SIZE(X), %xmm5 @@ -414,10 +414,10 @@ subq $-16 * SIZE, X subq $-16 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): movaps -9 * SIZE(X), %xmm4 movaps -7 * SIZE(X), %xmm5 @@ -469,10 +469,10 @@ subq $-16 * SIZE, Y ALIGN_3 -.L23: +L(23): movq M, %rax andq $8, %rax - jle .L24 + jle L(24) ALIGN_3 movaps -15 * SIZE(X), %xmm1 @@ -506,10 +506,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L24: +L(24): movq M, %rax andq $4, %rax - jle .L25 + jle L(25) ALIGN_3 movaps -15 * SIZE(X), %xmm1 @@ -532,10 +532,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L25: +L(25): movq M, %rax andq $2, %rax - jle .L26 + jle L(26) ALIGN_3 movaps -15 * SIZE(X), %xmm1 @@ -549,10 +549,10 @@ addq $2 * SIZE, Y ALIGN_3 -.L26: +L(26): movq M, %rax andq $1, %rax - jle .L29 + jle L(29) ALIGN_3 movsd -16 * SIZE(X), %xmm0 @@ -562,7 +562,7 @@ movsd %xmm0, -16 * SIZE(Y) ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -573,7 +573,7 @@ #else movq M, %rax sarq $4, %rax - jle .L23 + jle L(23) movsd -16 * SIZE(X), %xmm0 movhps -15 * SIZE(X), %xmm0 @@ -585,10 +585,10 @@ movhps -9 * SIZE(X), %xmm3 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): movsd -8 * SIZE(X), %xmm4 movhps -7 * SIZE(X), %xmm4 movsd -6 * SIZE(X), %xmm5 @@ -660,10 +660,10 @@ subq $-16 * SIZE, Y subq $-16 * SIZE, X decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): movsd -8 * SIZE(X), %xmm4 movhps -7 * SIZE(X), %xmm4 movsd -6 * SIZE(X), %xmm5 @@ -710,10 +710,10 @@ subq $-16 * SIZE, X ALIGN_3 -.L23: +L(23): movq M, %rax andq $8, %rax - jle .L24 + jle L(24) ALIGN_3 movsd -16 * SIZE(X), %xmm0 @@ -743,10 +743,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L24: +L(24): movq M, %rax andq $4, %rax - jle .L25 + jle L(25) ALIGN_3 movsd -16 * SIZE(X), %xmm0 @@ -767,10 +767,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L25: +L(25): movq M, %rax andq $2, %rax - jle .L26 + jle L(26) ALIGN_3 movsd -16 * SIZE(X), %xmm0 @@ -783,10 +783,10 @@ addq $2 * SIZE, Y ALIGN_3 -.L26: +L(26): movq M, %rax andq $1, %rax - jle .L29 + jle L(29) ALIGN_3 movsd -16 * SIZE(X), %xmm0 @@ -796,7 +796,7 @@ movsd %xmm0, -16 * SIZE(Y) ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -805,20 +805,20 @@ ALIGN_3 #endif -.L40: +L(40): movq Y, YY movq M, %rax //If incx==0 || incy==0, avoid unloop. cmpq $0, INCX - je .L46 + je L(46) cmpq $0, INCY - je .L46 + je L(46) sarq $3, %rax - jle .L45 + jle L(45) ALIGN_3 -.L41: +L(41): movsd 0 * SIZE(X), %xmm0 addq INCX, X movhpd 0 * SIZE(X), %xmm0 @@ -885,16 +885,16 @@ addq INCY, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L45: +L(45): movq M, %rax andq $7, %rax - jle .L47 + jle L(47) ALIGN_3 -.L46: +L(46): movsd (X), %xmm0 addq INCX, X mulsd %xmm15, %xmm0 @@ -902,10 +902,10 @@ movsd %xmm0, (Y) addq INCY, Y decq %rax - jg .L46 + jg L(46) ALIGN_3 -.L47: +L(47): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/builtin_stinit.S b/kernel/x86_64/builtin_stinit.S index cb3a288878..b33c04d64f 100644 --- a/kernel/x86_64/builtin_stinit.S +++ b/kernel/x86_64/builtin_stinit.S @@ -43,18 +43,18 @@ PROFCODE cmpq $4096, %rax - jle .L999 + jle L(999) ALIGN_3 -.L01: +L(01): subq $4096, %rax subq $4096, %rsp movq $0, (%rsp) cmpq $4096, %rax - jg .L01 + jg L(01) ALIGN_3 -.L999: +L(999): subq %rax, %rsp ret diff --git a/kernel/x86_64/cgemm_kernel_4x2_bulldozer.S b/kernel/x86_64/cgemm_kernel_4x2_bulldozer.S index 2675f71fb9..00ddb33135 100644 --- a/kernel/x86_64/cgemm_kernel_4x2_bulldozer.S +++ b/kernel/x86_64/cgemm_kernel_4x2_bulldozer.S @@ -561,13 +561,13 @@ STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -595,36 +595,36 @@ #endif #endif -.L2_0: +L(2_0): movq Ndiv6, J cmpq $0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $4*SIZE,BO1 addq $4*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -638,11 +638,11 @@ movq M, I sarq $2, I // i = (m >> 2) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -680,7 +680,7 @@ andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -691,7 +691,7 @@ negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL4x2_1(xxx) @@ -705,7 +705,7 @@ KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL4x2_1(xxx) @@ -719,12 +719,12 @@ KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -732,7 +732,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -744,14 +744,14 @@ negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL4x2_SUB(xxx) - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -847,7 +847,7 @@ addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 @@ -856,15 +856,15 @@ * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): testq $3, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -902,7 +902,7 @@ andq $-8, %rax // K = K - ( K % 8 ) - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -913,7 +913,7 @@ negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -927,7 +927,7 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -941,12 +941,12 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -954,7 +954,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -966,14 +966,14 @@ negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL2x2_SUB(xxx) - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -1053,13 +1053,13 @@ /**************************************************************************/ -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1097,7 +1097,7 @@ andq $-8, %rax // K = K - ( K % 8 ) - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1108,7 +1108,7 @@ negq %rax ALIGN_4 -.L2_42: +L(2_42): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x2_1(xxx) @@ -1122,7 +1122,7 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x2_1(xxx) @@ -1136,12 +1136,12 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -1149,7 +1149,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1161,14 +1161,14 @@ negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -1250,17 +1250,17 @@ -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -1268,30 +1268,30 @@ movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -1305,11 +1305,11 @@ movq M, I sarq $2, I // i = (m >> 2) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1347,7 +1347,7 @@ andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1358,7 +1358,7 @@ negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL4x1_1(xxx) @@ -1371,7 +1371,7 @@ KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_16 + je L(1_16) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL4x1_1(xxx) @@ -1384,12 +1384,12 @@ KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -1397,7 +1397,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 4 ; number of values @@ -1409,14 +1409,14 @@ negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL4x1_SUB(xxx) - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -1489,7 +1489,7 @@ addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 @@ -1498,15 +1498,15 @@ * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): testq $3, M - jz .L999 + jz L(999) testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1544,7 +1544,7 @@ andq $-8, %rax // K = K - ( K % 8 ) - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1555,7 +1555,7 @@ negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -1568,7 +1568,7 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_26 + je L(1_26) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -1581,12 +1581,12 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -1594,7 +1594,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2; number of values @@ -1606,14 +1606,14 @@ negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL2x1_SUB(xxx) - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -1680,13 +1680,13 @@ /**************************************************************************/ -.L1_40: +L(1_40): testq $1, M - jz .L999 // to next 2 lines of N + jz L(999) // to next 2 lines of N ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1724,7 +1724,7 @@ andq $-8, %rax // K = K - ( K % 8 ) - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1735,7 +1735,7 @@ negq %rax ALIGN_4 -.L1_42: +L(1_42): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x1_1(xxx) @@ -1748,7 +1748,7 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x1_1(xxx) @@ -1761,12 +1761,12 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -1774,7 +1774,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1786,14 +1786,14 @@ negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -1865,7 +1865,7 @@ -.L999: +L(999): vzeroupper movq SP, %rsp diff --git a/kernel/x86_64/cgemm_kernel_4x2_piledriver.S b/kernel/x86_64/cgemm_kernel_4x2_piledriver.S index bf7f91ee9e..9d1ea86cab 100644 --- a/kernel/x86_64/cgemm_kernel_4x2_piledriver.S +++ b/kernel/x86_64/cgemm_kernel_4x2_piledriver.S @@ -585,13 +585,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -619,36 +619,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif #endif -.L2_0: +L(2_0): movq Ndiv6, J cmpq $0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $4*SIZE,BO1 addq $4*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -662,11 +662,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = (m >> 2) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -704,7 +704,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -715,7 +715,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL4x2_1(xxx) @@ -729,7 +729,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL4x2_1(xxx) @@ -743,12 +743,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -756,7 +756,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -768,14 +768,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL4x2_SUB(xxx) - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -871,7 +871,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 @@ -880,15 +880,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): testq $3, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -926,7 +926,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax // K = K - ( K % 8 ) - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -937,7 +937,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -951,7 +951,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -965,12 +965,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -978,7 +978,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -990,14 +990,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL2x2_SUB(xxx) - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -1077,13 +1077,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1121,7 +1121,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax // K = K - ( K % 8 ) - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1132,7 +1132,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_42: +L(2_42): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x2_1(xxx) @@ -1146,7 +1146,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x2_1(xxx) @@ -1160,12 +1160,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -1173,7 +1173,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1185,14 +1185,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -1274,17 +1274,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -1292,30 +1292,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -1329,11 +1329,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = (m >> 2) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1371,7 +1371,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1382,7 +1382,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL4x1_1(xxx) @@ -1395,7 +1395,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_16 + je L(1_16) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL4x1_1(xxx) @@ -1408,12 +1408,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -1421,7 +1421,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 4 ; number of values @@ -1433,14 +1433,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL4x1_SUB(xxx) - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -1513,7 +1513,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 @@ -1522,15 +1522,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): testq $3, M - jz .L999 + jz L(999) testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1568,7 +1568,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax // K = K - ( K % 8 ) - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1579,7 +1579,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -1592,7 +1592,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_26 + je L(1_26) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -1605,12 +1605,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -1618,7 +1618,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2; number of values @@ -1630,14 +1630,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL2x1_SUB(xxx) - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -1704,13 +1704,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L1_40: +L(1_40): testq $1, M - jz .L999 // to next 2 lines of N + jz L(999) // to next 2 lines of N ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1748,7 +1748,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax // K = K - ( K % 8 ) - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1759,7 +1759,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_42: +L(1_42): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x1_1(xxx) @@ -1772,7 +1772,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x1_1(xxx) @@ -1785,12 +1785,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -1798,7 +1798,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1810,14 +1810,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -1889,7 +1889,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L999: +L(999): vzeroupper movq SP, %rsp diff --git a/kernel/x86_64/cgemm_kernel_4x8_sandy.S b/kernel/x86_64/cgemm_kernel_4x8_sandy.S index f64f199b7e..810d1d5946 100644 --- a/kernel/x86_64/cgemm_kernel_4x8_sandy.S +++ b/kernel/x86_64/cgemm_kernel_4x8_sandy.S @@ -283,9 +283,9 @@ movq %r11, kk; MOVQ bn,j; SARQ $2,j; # Rn = 4 -JLE .L0_loopE; +JLE L(0_loopE); ALIGN_5; -.L0_bodyB:; +L(0_bodyB):; #if defined(TRMMKERNEL) && defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk; @@ -298,9 +298,9 @@ LEAQ (bb, k, 1), prebb; # Rn=4, SIZE=4 COMPLEX=2 MOVQ ba,ptrba; MOVQ bm,i; SARQ $3,i; # Rm = 8 -JLE .L1_loopE; +JLE L(1_loopE); ALIGN_5; -.L1_bodyB:; +L(1_bodyB):; #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -345,9 +345,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2,k; # Unroll 4 times -JLE .L2_loopE; +JLE L(2_loopE); ALIGN_5; -.L2_bodyB:; +L(2_bodyB):; # Computing kernel ######### Unroll 1 ################## @@ -567,19 +567,19 @@ MUL_SY yvec1, yvec4, yvec6; MUL_SY yvec1, yvec5, yvec7; ADD2_SY yvec6, yvec10, yvec10; ADD2_SY yvec7, yvec8, yvec8; -.L2_bodyE:; +L(2_bodyE):; DECQ k; -JG .L2_bodyB; +JG L(2_bodyB); ALIGN_5 -.L2_loopE:; +L(2_loopE):; #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L3_loopE; +JLE L(3_loopE); ALIGN_5 -.L3_loopB: +L(3_loopB): ######### Unroll 1 ################## PREFETCH0 PRESIZE*SIZE(ptrba) LD_SY 8*SIZE(ptrba), yvec1; # Ar4, Ai4, Ar5, Ai5.. @@ -689,15 +689,15 @@ MUL_SY yvec1, yvec4, yvec6; MUL_SY yvec1, yvec5, yvec7; ADD2_SY yvec6, yvec10, yvec10; ADD2_SY yvec7, yvec8, yvec8; -.L3_loopE: +L(3_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L4_loopE; +JLE L(4_loopE); ALIGN_5 -.L4_loopB:; +L(4_loopB):; ######### Unroll 1 ################## PREFETCH0 PRESIZE*SIZE(ptrba) LD_SY 8*SIZE(ptrba), yvec1; # Ar4, Ai4, Ar5, Ai5.. @@ -752,7 +752,7 @@ MUL_SY yvec1, yvec5, yvec7; ADD2_SY yvec6, yvec10, yvec10; ADD2_SY yvec7, yvec8, yvec8; -.L4_loopE:; +L(4_loopE):; #### Handle #### XOR_SY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -853,7 +853,7 @@ REVS_SY $0xe4,yvec7,yvec8,yvec8; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L4_loopEx; +JNE L(4_loopEx); ALIGN_5 EXTRA_SY $1,yvec15,xvec7; EXTRA_SY $1,yvec14,xvec6; @@ -909,12 +909,12 @@ ADDQ $8, kk; #endif ADDQ $16*SIZE,C0; ADDQ $16*SIZE,C1; -.L1_bodyE:; +L(1_bodyE):; DECQ i; -JG .L1_bodyB; -JMP .L1_loopE; +JG L(1_bodyB); +JMP L(1_loopE); ALIGN_5 -.L4_loopEx: +L(4_loopEx): EXTRA_SY $1, yvec15, xvec7; #ifndef TRMMKERNEL LDL_SY 0*SIZE(C0), xvec6, xvec6; @@ -1055,13 +1055,13 @@ ADDQ $8, kk; ADDQ $16*SIZE, C0; ADDQ $16*SIZE, C1; DECQ i; -JG .L1_bodyB; +JG L(1_bodyB); ALIGN_5; -.L1_loopE:; +L(1_loopE):; TEST $4, bm; -JLE .L5_loopE; +JLE L(5_loopE); ALIGN_5 -.L5_bodyB: +L(5_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -1091,9 +1091,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L8_loopE; +JLE L(8_loopE); ALIGN_5 -.L8_bodyB: +L(8_bodyB): #### Unroll times 1 #### LD_SY 0*SIZE(ptrba), yvec0; VPERMILP_SY $0xb1, yvec0, yvec1; @@ -1220,17 +1220,17 @@ ADD2_SY yvec7, yvec9, yvec9; ADDQ $32*SIZE, ptrba; ADDQ $32*SIZE, ptrbb; DECQ k; -JG .L8_bodyB; +JG L(8_bodyB); ALIGN_5 -.L8_loopE: +L(8_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L9_loopE; +JLE L(9_loopE); ALIGN_5 -.L9_bodyB: +L(9_bodyB): #### Unroll times 1 #### LD_SY 0*SIZE(ptrba), yvec0; VPERMILP_SY $0xb1, yvec0, yvec1; @@ -1295,15 +1295,15 @@ ADD2_SY yvec7, yvec9, yvec9; ADDQ $16*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; -.L9_loopE: +L(9_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L10_loopE; +JLE L(10_loopE); ALIGN_5 -.L10_bodyB: +L(10_bodyB): #### Unroll times 1 #### LD_SY 0*SIZE(ptrba), yvec0; VPERMILP_SY $0xb1, yvec0, yvec1; @@ -1337,7 +1337,7 @@ ADD2_SY yvec7, yvec9, yvec9; ADDQ $8*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L10_loopE: +L(10_loopE): #### Handle #### XOR_SY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -1470,11 +1470,11 @@ ADDQ $4, kk; ADDQ $8*SIZE, C0; ADDQ $8*SIZE, C1; -.L5_loopE: +L(5_loopE): TEST $2, bm; -JLE .L6_loopE; +JLE L(6_loopE); ALIGN_5 -.L6_bodyB: +L(6_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -1505,9 +1505,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L11_loopE; +JLE L(11_loopE); ALIGN_5 -.L11_bodyB: +L(11_bodyB): LD_SX 0*SIZE(ptrba), xvec0; # ar1, ai1, ar2, ai2 EDUP_SX 0*SIZE(ptrbb), xvec2; # br1, br1, br2, br2 SHUF_SX $0x4e, xvec2, xvec3; # br3, br3, br4, br4 @@ -1630,17 +1630,17 @@ ADD2_SX xvec5, xvec12, xvec12; ADDQ $16*SIZE, ptrba; ADDQ $32*SIZE, ptrbb; DECQ k; -JG .L11_bodyB; +JG L(11_bodyB); ALIGN_5 -.L11_loopE: +L(11_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L12_loopE; +JLE L(12_loopE); ALIGN_5 -.L12_bodyB: +L(12_bodyB): LD_SX 0*SIZE(ptrba), xvec0; # ar1, ai1, ar2, ai2 EDUP_SX 0*SIZE(ptrbb), xvec2; # br1, br1, br2, br2 SHUF_SX $0x4e, xvec2, xvec3; # br3, br3, br4, br4 @@ -1703,15 +1703,15 @@ ADD2_SX xvec5, xvec12, xvec12; ADDQ $8*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; -.L12_loopE: +L(12_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L13_loopE; +JLE L(13_loopE); ALIGN_5 -.L13_bodyB: +L(13_bodyB): LD_SX 0*SIZE(ptrba), xvec0; # ar1, ai1, ar2, ai2 EDUP_SX 0*SIZE(ptrbb), xvec2; # br1, br1, br2, br2 SHUF_SX $0x4e, xvec2, xvec3; # br3, br3, br4, br4 @@ -1744,7 +1744,7 @@ ADD2_SX xvec5, xvec12, xvec12; ADDQ $4*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L13_loopE: +L(13_loopE): #### Handle #### #if defined(RN) || defined(RT) || defined(CN) || defined(CT) XOR_SY yvec7, yvec7, yvec7; @@ -1851,11 +1851,11 @@ ADDQ $2, kk; ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -.L6_loopE: +L(6_loopE): TEST $1, bm; -JLE .L7_loopE; +JLE L(7_loopE); ALIGN_5 -.L7_bodyB: +L(7_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -1883,9 +1883,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L14_loopE; +JLE L(14_loopE); ALIGN_5 -.L14_bodyB: +L(14_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0xb1, xvec2, xvec3; @@ -1956,17 +1956,17 @@ ADD2_SX xvec5, xvec14, xvec14; ADDQ $8*SIZE, ptrba; ADDQ $32*SIZE, ptrbb; DECQ k; -JG .L14_bodyB; +JG L(14_bodyB); ALIGN_5 -.L14_loopE: +L(14_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L15_loopE; +JLE L(15_loopE); ALIGN_5 -.L15_bodyB: +L(15_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0xb1, xvec2, xvec3; @@ -2003,15 +2003,15 @@ ADD2_SX xvec5, xvec14, xvec14; ADDQ $4*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; -.L15_loopE: +L(15_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L16_loopE; +JLE L(16_loopE); ALIGN_5 -.L16_bodyB: +L(16_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0xb1, xvec2, xvec3; @@ -2031,7 +2031,7 @@ ADD2_SX xvec5, xvec14, xvec14; ADDQ $2*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L16_loopE: +L(16_loopE): #### Handle #### #if defined(NR) || defined(NC) || defined(TR) || defined(TC) XOR_SY yvec7, yvec7, yvec7; @@ -2097,7 +2097,7 @@ ADDQ $1, kk; ADDQ $2*SIZE, C0; ADDQ $2*SIZE, C1; -.L7_loopE: +L(7_loopE): #if defined(TRMMKERNEL) && !defined(LEFT) ADDQ $4, kk; #endif @@ -2105,15 +2105,15 @@ MOVQ bk,k; SALQ $5,k; ADDQ k,bb; LEAQ (C,ldc,4),C; -.L0_bodyE:; +L(0_bodyE):; DECQ j; -JG .L0_bodyB; +JG L(0_bodyB); ALIGN_5; -.L0_loopE:; +L(0_loopE):; TEST $2, bn; -JLE .L20_loopE; +JLE L(20_loopE); ALIGN_5 -.L20_bodyB: +L(20_bodyB): #if defined(TRMMKERNEL) && defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk; @@ -2123,9 +2123,9 @@ LEAQ (C, ldc, 1), C1; MOVQ ba, ptrba; MOVQ bm, i; SARQ $3, i; -JLE .L21_loopE; +JLE L(21_loopE); ALIGN_5 -.L21_bodyB: +L(21_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -2159,9 +2159,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L211_loopE; +JLE L(211_loopE); ALIGN_5 -.L211_bodyB: +L(211_bodyB): EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; SHUF_SX $0x4e, xvec4, xvec6; @@ -2408,17 +2408,17 @@ ADD2_SX xvec3, xvec8, xvec8; ADDQ $64*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; DECQ k; -JG .L211_bodyB; +JG L(211_bodyB); ALIGN_5 -.L211_loopE: +L(211_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L212_loopE; +JLE L(212_loopE); ALIGN_5 -.L212_bodyB: +L(212_bodyB): EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; SHUF_SX $0x4e, xvec4, xvec6; @@ -2543,15 +2543,15 @@ ADD2_SX xvec3, xvec8, xvec8; ADDQ $32*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L212_loopE: +L(212_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L213_loopE; +JLE L(213_loopE); ALIGN_5 -.L213_bodyB: +L(213_bodyB): EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; SHUF_SX $0x4e, xvec4, xvec6; @@ -2615,7 +2615,7 @@ ADD2_SX xvec3, xvec8, xvec8; ADDQ $16*SIZE, ptrba; ADDQ $4*SIZE, ptrbb -.L213_loopE: +L(213_loopE): #### Handle #### #if defined(RN) || defined(RT) || defined(CN) || defined(CT) XOR_SY yvec7, yvec7, yvec7; @@ -2803,13 +2803,13 @@ ADDQ $8, kk; ADDQ $16*SIZE, C0; ADDQ $16*SIZE, C1; DECQ i; -JG .L21_bodyB; +JG L(21_bodyB); ALIGN_5 -.L21_loopE: +L(21_loopE): TEST $4, bm; -JLE .L22_loopE; +JLE L(22_loopE); ALIGN_5 -.L22_bodyB: +L(22_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -2840,9 +2840,9 @@ MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L221_loopE; +JLE L(221_loopE); ALIGN_5 -.L221_bodyB: +L(221_bodyB): EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; SHUF_SX $0x4e, xvec4, xvec6; @@ -2980,17 +2980,17 @@ ADD2_SX xvec3, xvec10, xvec10; ADDQ $32*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; DECQ k; -JG .L221_bodyB; +JG L(221_bodyB); ALIGN_5 -.L221_loopE: +L(221_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L222_loopE; +JLE L(222_loopE); ALIGN_5 -.L222_bodyB: +L(222_bodyB): EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; SHUF_SX $0x4e, xvec4, xvec6; @@ -3061,15 +3061,15 @@ ADDQ $16*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L222_loopE: +L(222_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L223_loopE; +JLE L(223_loopE); ALIGN_5 -.L223_bodyB: +L(223_bodyB): EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; SHUF_SX $0x4e, xvec4, xvec6; @@ -3105,7 +3105,7 @@ ADD2_SX xvec3, xvec10, xvec10; ADDQ $8*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L223_loopE: +L(223_loopE): #### Handle #### #if defined(RN) || defined(RT) || defined(CN) || defined(CT) XOR_SY yvec7, yvec7, yvec7; @@ -3213,11 +3213,11 @@ ADDQ $4, kk; ADDQ $8*SIZE, C0; ADDQ $8*SIZE, C1; -.L22_loopE: +L(22_loopE): TEST $2, bm; -JLE .L23_loopE; +JLE L(23_loopE); ALIGN_5 -.L23_bodyB: +L(23_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -3245,9 +3245,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L231_loopE; +JLE L(231_loopE); ALIGN_5 -.L231_bodyB: +L(231_bodyB): EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; SHUF_SX $0x4e, xvec4, xvec6; @@ -3329,17 +3329,17 @@ ADD2_SX xvec3, xvec11, xvec11; ADDQ $16*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; DECQ k; -JG .L231_bodyB; +JG L(231_bodyB); ALIGN_5 -.L231_loopE: +L(231_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L232_loopE; +JLE L(232_loopE); ALIGN_5 -.L232_bodyB: +L(232_bodyB): EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; SHUF_SX $0x4e, xvec4, xvec6; @@ -3381,15 +3381,15 @@ ADD2_SX xvec3, xvec11, xvec11; ADDQ $8*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L232_loopE: +L(232_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L233_loopE; +JLE L(233_loopE); ALIGN_5 -.L233_bodyB: +L(233_bodyB): EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; SHUF_SX $0x4e, xvec4, xvec6; @@ -3411,7 +3411,7 @@ ADD2_SX xvec3, xvec11, xvec11; ADDQ $4*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L233_loopE: +L(233_loopE): #### Handle #### #if defined(RN) || defined(RT) || defined(CN) || defined(CT) XOR_SY yvec7, yvec7, yvec7; @@ -3479,11 +3479,11 @@ ADDQ $2, kk; ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -.L23_loopE: +L(23_loopE): TEST $1, bm; -JLE .L24_loopE; +JLE L(24_loopE); ALIGN_5 -.L24_bodyB: +L(24_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -3510,9 +3510,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L241_loopE; +JLE L(241_loopE); ALIGN_5 -.L241_bodyB: +L(241_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0xb1, xvec2, xvec3; @@ -3555,17 +3555,17 @@ ADD2_SX xvec3, xvec15, xvec15; ADDQ $8*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; DECQ k; -JG .L241_bodyB; +JG L(241_bodyB); ALIGN_5 -.L241_loopE: +L(241_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L242_loopE; +JLE L(242_loopE); ALIGN_5 -.L242_bodyB: +L(242_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0xb1, xvec2, xvec3; @@ -3588,15 +3588,15 @@ ADD2_SX xvec3, xvec15, xvec15; ADDQ $4*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L242_loopE: +L(242_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L243_loopE; +JLE L(243_loopE); ALIGN_5 -.L243_bodyB: +L(243_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0xb1, xvec2, xvec3; @@ -3609,7 +3609,7 @@ ADD2_SX xvec3, xvec15, xvec15; ADDQ $2*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L243_loopE: +L(243_loopE): #### Handle #### XOR_SY yvec7, yvec7, yvec7; #if defined(NR) || defined(NC) || defined(TR) || defined(TC) @@ -3652,7 +3652,7 @@ ADDQ $1, kk; #endif ADDQ $2*SIZE, C0; ADDQ $2*SIZE, C1; -.L24_loopE: +L(24_loopE): #if defined(TRMMKERNEL) && !defined(LEFT) ADDQ $2, kk; #endif @@ -3660,11 +3660,11 @@ MOVQ bk, k; SALQ $4, k; ADDQ k, bb; LEAQ (C, ldc, 2), C; -.L20_loopE: +L(20_loopE): TEST $1, bn; -JLE .L30_loopE; +JLE L(30_loopE); ALIGN_5 -.L30_bodyB: +L(30_bodyB): #if defined(TRMMKERNEL) && defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk; @@ -3673,9 +3673,9 @@ MOVQ C, C0; MOVQ ba, ptrba; MOVQ bm, i; SARQ $3, i; -JLE .L31_loopE; +JLE L(31_loopE); ALIGN_5 -.L31_bodyB: +L(31_bodyB): MOVQ bb, ptrbb; #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; @@ -3705,9 +3705,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L311_loopE; +JLE L(311_loopE); ALIGN_5 -.L311_bodyB: +L(311_bodyB): #### Unroll 1 #### LD_SY 0*SIZE(ptrba), yvec0; LD_SY 8*SIZE(ptrba), yvec1; @@ -3778,17 +3778,17 @@ ADD2_SY yvec7, yvec14, yvec14; ADDQ $64*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L311_bodyB; +JG L(311_bodyB); ALIGN_5 -.L311_loopE: +L(311_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L312_loopE; +JLE L(312_loopE); ALIGN_5 -.L312_bodyB: +L(312_bodyB): #### Unroll 1 #### LD_SY 0*SIZE(ptrba), yvec0; LD_SY 8*SIZE(ptrba), yvec1; @@ -3825,15 +3825,15 @@ ADD2_SY yvec7, yvec14, yvec14; ADDQ $32*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L312_loopE: +L(312_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L313_loopE; +JLE L(313_loopE); ALIGN_5 -.L313_bodyB: +L(313_bodyB): #### Unroll 1 #### LD_SY 0*SIZE(ptrba), yvec0; LD_SY 8*SIZE(ptrba), yvec1; @@ -3853,7 +3853,7 @@ ADD2_SY yvec7, yvec14, yvec14; ADDQ $16*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L313_loopE: +L(313_loopE): #### Handle #### XOR_SY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -3919,13 +3919,13 @@ ADDQ $8, kk; #endif ADDQ $16*SIZE, C0; DECQ i; -JG .L31_bodyB; +JG L(31_bodyB); ALIGN_5 -.L31_loopE: +L(31_loopE): TEST $4, bm; -JLE .L32_loopE; +JLE L(32_loopE); ALIGN_5 -.L32_bodyB: +L(32_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -3952,9 +3952,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L321_loopE; +JLE L(321_loopE); ALIGN_5 -.L321_bodyB: +L(321_bodyB): #### Unroll 1 #### LD_SY 0*SIZE(ptrba), yvec0; BROAD_SY 0*SIZE(ptrbb), yvec2; @@ -4001,17 +4001,17 @@ ADD2_SY yvec6, yvec15, yvec15; ADDQ $32*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L321_bodyB; +JG L(321_bodyB); ALIGN_5 -.L321_loopE: +L(321_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L322_loopE; +JLE L(322_loopE); ALIGN_5 -.L322_bodyB: +L(322_bodyB): #### Unroll 1 #### LD_SY 0*SIZE(ptrba), yvec0; BROAD_SY 0*SIZE(ptrbb), yvec2; @@ -4036,15 +4036,15 @@ ADD2_SY yvec6, yvec15, yvec15; ADDQ $16*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L322_loopE: +L(322_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L323_loopE; +JLE L(323_loopE); ALIGN_5 -.L323_bodyB: +L(323_bodyB): #### Unroll 1 #### LD_SY 0*SIZE(ptrba), yvec0; BROAD_SY 0*SIZE(ptrbb), yvec2; @@ -4058,7 +4058,7 @@ ADD2_SY yvec6, yvec15, yvec15; ADDQ $8*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L323_loopE: +L(323_loopE): #### Handle #### XOR_SY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -4104,11 +4104,11 @@ ADDQ $4, kk; #endif ADDQ $8*SIZE, C0; -.L32_loopE: +L(32_loopE): TEST $2, bm; -JLE .L33_loopE; +JLE L(33_loopE); ALIGN_5 -.L33_bodyB: +L(33_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -4135,9 +4135,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L331_loopE; +JLE L(331_loopE); ALIGN_5 -.L331_bodyB: +L(331_bodyB): #### Unroll 1 #### LD_SX 0*SIZE(ptrba), xvec0; BROAD_SX 0*SIZE(ptrbb), xvec2; @@ -4180,17 +4180,17 @@ ADD2_SX xvec3, xvec15, xvec15; ADDQ $16*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L331_bodyB; +JG L(331_bodyB); ALIGN_5 -.L331_loopE: +L(331_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L332_loopE; +JLE L(332_loopE); ALIGN_5 -.L332_bodyB: +L(332_bodyB): #### Unroll 1 #### LD_SX 0*SIZE(ptrba), xvec0; BROAD_SX 0*SIZE(ptrbb), xvec2; @@ -4213,15 +4213,15 @@ ADD2_SX xvec3, xvec15, xvec15; ADDQ $8*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L332_loopE: +L(332_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L333_loopE; +JLE L(333_loopE); ALIGN_5 -.L333_bodyB: +L(333_bodyB): #### Unroll 1 #### LD_SX 0*SIZE(ptrba), xvec0; BROAD_SX 0*SIZE(ptrbb), xvec2; @@ -4234,7 +4234,7 @@ ADD2_SX xvec3, xvec15, xvec15; ADDQ $4*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L333_loopE: +L(333_loopE): #### Handle #### XOR_SY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -4276,11 +4276,11 @@ ADDQ $2, kk; #endif ADDQ $4*SIZE, C0; -.L33_loopE: +L(33_loopE): TEST $1, bm; -JLE .L34_loopE; +JLE L(34_loopE); ALIGN_5 -.L34_bodyB: +L(34_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -4307,9 +4307,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L341_loopE; +JLE L(341_loopE); ALIGN_5 -.L341_bodyB: +L(341_bodyB): LD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0xa0, xvec2, xvec3; @@ -4332,17 +4332,17 @@ ADD2_SX xvec4, xvec15, xvec15; ADDQ $8*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L341_bodyB; +JG L(341_bodyB); ALIGN_5 -.L341_loopE: +L(341_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L342_loopE; +JLE L(342_loopE); ALIGN_5 -.L342_bodyB: +L(342_bodyB): LD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0xa0, xvec2, xvec3; @@ -4355,15 +4355,15 @@ ADD2_SX xvec4, xvec15, xvec15; ADDQ $4*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L342_loopE: +L(342_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L343_loopE; +JLE L(343_loopE); ALIGN_5 -.L343_bodyB: +L(343_bodyB): XOR_SY yvec0, yvec0, yvec0; XOR_SY yvec2, yvec2, yvec2; LDL_SX 0*SIZE(ptrba), xvec0, xvec0; @@ -4378,7 +4378,7 @@ ADD2_SX xvec4, xvec15, xvec15; ADDQ $2*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L343_loopE: +L(343_loopE): #### Handle #### XOR_SY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -4419,7 +4419,7 @@ ADDQ $1, kk; #endif ADDQ $2*SIZE, C0; -.L34_loopE: +L(34_loopE): #if defined(TRMMKERNEL) && !defined(LEFT) ADDQ $1, kk; #endif @@ -4428,7 +4428,7 @@ MOVQ bk, k; SALQ $3, k; ADDQ k, bb; ADDQ ldc, C; -.L30_loopE: +L(30_loopE): movq 0(%rsp), %rbx; movq 8(%rsp), %rbp; movq 16(%rsp), %r12; diff --git a/kernel/x86_64/cgemm_kernel_8x2_haswell.S b/kernel/x86_64/cgemm_kernel_8x2_haswell.S index a608071dbe..92243e10b0 100644 --- a/kernel/x86_64/cgemm_kernel_8x2_haswell.S +++ b/kernel/x86_64/cgemm_kernel_8x2_haswell.S @@ -1268,13 +1268,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $ 0, OLD_M - je .L999 + je L(999) cmpq $ 0, OLD_N - je .L999 + je L(999) cmpq $ 0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -1294,16 +1294,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /************************************************************************************************/ -.L6_0: +L(6_0): movq Ndiv6, J cmpq $ 0, J - je .L2_00 + je L(2_00) ALIGN_4 -.L6_01: +L(6_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO @@ -1315,7 +1315,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L6_02b: +L(6_02b): vmovups (BO1), %xmm0 vmovsd (BO2), %xmm1 @@ -1325,10 +1325,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4*SIZE,BO2 addq $ 6*SIZE,BO decq %rax - jnz .L6_02b + jnz L(6_02b) -.L6_10: +L(6_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc leaq (C, LDC, 1), C // c += 1 * ldc @@ -1338,12 +1338,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 3, I // i = (m >> 3) - je .L6_4_10 + je L(6_4_10) ALIGN_4 /**********************************************************************************************************/ -.L6_8_11: +L(6_8_11): leaq BUFFER1, BO // first buffer to BO addq $ 8 * SIZE, BO @@ -1353,11 +1353,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L6_8_16 + je L(6_8_16) ALIGN_4 -.L6_8_12: +L(6_8_12): KERNEL8x3_SUB KERNEL8x3_SUB @@ -1369,7 +1369,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_SUB KERNEL8x3_SUB - je .L6_8_16 + je L(6_8_16) KERNEL8x3_SUB KERNEL8x3_SUB @@ -1381,50 +1381,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_SUB KERNEL8x3_SUB - je .L6_8_16 + je L(6_8_16) - jmp .L6_8_12 + jmp L(6_8_12) ALIGN_4 -.L6_8_16: +L(6_8_16): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L6_8_19 + je L(6_8_19) ALIGN_4 -.L6_8_17: +L(6_8_17): KERNEL8x3_SUB - jnz .L6_8_17 + jnz L(6_8_17) ALIGN_4 -.L6_8_19: +L(6_8_19): SAVE8x3 addq $ 16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L6_8_11 + jg L(6_8_11) ALIGN_4 /**********************************************************************************************************/ -.L6_4_10: +L(6_4_10): testq $ 7, M - jz .L6_4_60 // to next 2 lines of N + jz L(6_4_60) // to next 2 lines of N testq $ 4, M - jz .L6_4_20 + jz L(6_4_20) ALIGN_4 -.L6_4_11: +L(6_4_11): leaq BUFFER1, BO // first buffer to BO addq $ 8 * SIZE, BO @@ -1434,11 +1434,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L6_4_16 + je L(6_4_16) ALIGN_4 -.L6_4_12: +L(6_4_12): prefetcht0 A_PR1(AO) KERNEL4x3_SUB @@ -1454,7 +1454,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_SUB KERNEL4x3_SUB - je .L6_4_16 + je L(6_4_16) prefetcht0 A_PR1(AO) KERNEL4x3_SUB @@ -1470,28 +1470,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_SUB KERNEL4x3_SUB - je .L6_4_16 + je L(6_4_16) - jmp .L6_4_12 + jmp L(6_4_12) ALIGN_4 -.L6_4_16: +L(6_4_16): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L6_4_19 + je L(6_4_19) ALIGN_4 -.L6_4_17: +L(6_4_17): KERNEL4x3_SUB - jnz .L6_4_17 + jnz L(6_4_17) ALIGN_4 -.L6_4_19: +L(6_4_19): SAVE4x3 @@ -1504,13 +1504,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Rest of M ***************************************************************************/ -.L6_4_20: +L(6_4_20): testq $ 2, M - jz .L6_4_40 + jz L(6_4_40) ALIGN_4 -.L6_4_21: +L(6_4_21): leaq BUFFER1, BO // first buffer to BO addq $ 8 * SIZE, BO @@ -1520,11 +1520,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L6_4_26 + je L(6_4_26) ALIGN_4 -.L6_4_22: +L(6_4_22): prefetcht0 A_PR1(AO) KERNEL2x3_SUB @@ -1538,7 +1538,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_SUB KERNEL2x3_SUB - je .L6_4_26 + je L(6_4_26) prefetcht0 A_PR1(AO) KERNEL2x3_SUB @@ -1552,46 +1552,46 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_SUB KERNEL2x3_SUB - je .L6_4_26 + je L(6_4_26) - jmp .L6_4_22 + jmp L(6_4_22) ALIGN_4 -.L6_4_26: +L(6_4_26): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L6_4_29 + je L(6_4_29) ALIGN_4 -.L6_4_27: +L(6_4_27): KERNEL2x3_SUB - jnz .L6_4_27 + jnz L(6_4_27) ALIGN_4 -.L6_4_29: +L(6_4_29): SAVE2x3 addq $ 4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L6_4_21 + jg L(6_4_21) ALIGN_4 /**************************************************************************/ -.L6_4_40: +L(6_4_40): testq $ 1, M - jz .L6_4_60 // to next 2 lines of N + jz L(6_4_60) // to next 2 lines of N ALIGN_4 -.L6_4_41: +L(6_4_41): leaq BUFFER1, BO // first buffer to BO addq $ 8 * SIZE, BO @@ -1601,11 +1601,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L6_4_46 + je L(6_4_46) ALIGN_4 -.L6_4_42: +L(6_4_42): prefetcht0 A_PR1(AO) KERNEL1x3_SUB @@ -1618,7 +1618,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_SUB KERNEL1x3_SUB - je .L6_4_46 + je L(6_4_46) prefetcht0 A_PR1(AO) KERNEL1x3_SUB @@ -1631,44 +1631,44 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_SUB KERNEL1x3_SUB - je .L6_4_46 + je L(6_4_46) - jmp .L6_4_42 + jmp L(6_4_42) ALIGN_4 -.L6_4_46: +L(6_4_46): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L6_4_49 + je L(6_4_49) ALIGN_4 -.L6_4_47: +L(6_4_47): KERNEL1x3_SUB - jnz .L6_4_47 + jnz L(6_4_47) ALIGN_4 -.L6_4_49: +L(6_4_49): SAVE1x3 addq $ 2 * SIZE, CO1 # coffset += 2 decq I # i -- - jg .L6_4_41 + jg L(6_4_41) ALIGN_4 -.L6_4_60: +L(6_4_60): /*******************************************************************************************/ -.L7_01: +L(7_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO @@ -1679,7 +1679,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L7_02b: +L(7_02b): vmovsd 2*SIZE(BO1), %xmm0 vmovups (BO2), %xmm1 @@ -1689,11 +1689,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4*SIZE,BO2 addq $ 6*SIZE,BO decq %rax - jnz .L7_02b + jnz L(7_02b) movq BO2, B // next offset of B -.L7_10: +L(7_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc leaq (C, LDC, 1), C // c += 1 * ldc @@ -1703,12 +1703,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 3, I // i = (m >> 3) - je .L7_4_10 + je L(7_4_10) ALIGN_4 /**********************************************************************************************************/ -.L7_8_11: +L(7_8_11): leaq BUFFER1, BO // first buffer to BO addq $ 8 * SIZE, BO @@ -1718,11 +1718,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L7_8_16 + je L(7_8_16) ALIGN_4 -.L7_8_12: +L(7_8_12): KERNEL8x3_SUB KERNEL8x3_SUB @@ -1734,7 +1734,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_SUB KERNEL8x3_SUB - je .L7_8_16 + je L(7_8_16) KERNEL8x3_SUB KERNEL8x3_SUB @@ -1746,50 +1746,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_SUB KERNEL8x3_SUB - je .L7_8_16 + je L(7_8_16) - jmp .L7_8_12 + jmp L(7_8_12) ALIGN_4 -.L7_8_16: +L(7_8_16): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L7_8_19 + je L(7_8_19) ALIGN_4 -.L7_8_17: +L(7_8_17): KERNEL8x3_SUB - jnz .L7_8_17 + jnz L(7_8_17) ALIGN_4 -.L7_8_19: +L(7_8_19): SAVE8x3 addq $ 16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L7_8_11 + jg L(7_8_11) ALIGN_4 /**********************************************************************************************************/ -.L7_4_10: +L(7_4_10): testq $ 7, M - jz .L7_4_60 // to next 2 lines of N + jz L(7_4_60) // to next 2 lines of N testq $ 4, M - jz .L7_4_20 + jz L(7_4_20) ALIGN_4 -.L7_4_11: +L(7_4_11): leaq BUFFER1, BO // first buffer to BO addq $ 8 * SIZE, BO @@ -1799,11 +1799,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L7_4_16 + je L(7_4_16) ALIGN_4 -.L7_4_12: +L(7_4_12): prefetcht0 A_PR1(AO) KERNEL4x3_SUB @@ -1819,7 +1819,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_SUB KERNEL4x3_SUB - je .L7_4_16 + je L(7_4_16) prefetcht0 A_PR1(AO) KERNEL4x3_SUB @@ -1835,28 +1835,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_SUB KERNEL4x3_SUB - je .L7_4_16 + je L(7_4_16) - jmp .L7_4_12 + jmp L(7_4_12) ALIGN_4 -.L7_4_16: +L(7_4_16): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L7_4_19 + je L(7_4_19) ALIGN_4 -.L7_4_17: +L(7_4_17): KERNEL4x3_SUB - jnz .L7_4_17 + jnz L(7_4_17) ALIGN_4 -.L7_4_19: +L(7_4_19): SAVE4x3 @@ -1869,13 +1869,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Rest of M ***************************************************************************/ -.L7_4_20: +L(7_4_20): testq $ 2, M - jz .L7_4_40 + jz L(7_4_40) ALIGN_4 -.L7_4_21: +L(7_4_21): leaq BUFFER1, BO // first buffer to BO addq $ 8 * SIZE, BO @@ -1885,11 +1885,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L7_4_26 + je L(7_4_26) ALIGN_4 -.L7_4_22: +L(7_4_22): prefetcht0 A_PR1(AO) KERNEL2x3_SUB @@ -1903,7 +1903,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_SUB KERNEL2x3_SUB - je .L7_4_26 + je L(7_4_26) prefetcht0 A_PR1(AO) KERNEL2x3_SUB @@ -1917,46 +1917,46 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_SUB KERNEL2x3_SUB - je .L7_4_26 + je L(7_4_26) - jmp .L7_4_22 + jmp L(7_4_22) ALIGN_4 -.L7_4_26: +L(7_4_26): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L7_4_29 + je L(7_4_29) ALIGN_4 -.L7_4_27: +L(7_4_27): KERNEL2x3_SUB - jnz .L7_4_27 + jnz L(7_4_27) ALIGN_4 -.L7_4_29: +L(7_4_29): SAVE2x3 addq $ 4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L7_4_21 + jg L(7_4_21) ALIGN_4 /**************************************************************************/ -.L7_4_40: +L(7_4_40): testq $ 1, M - jz .L7_4_60 // to next 2 lines of N + jz L(7_4_60) // to next 2 lines of N ALIGN_4 -.L7_4_41: +L(7_4_41): leaq BUFFER1, BO // first buffer to BO addq $ 8 * SIZE, BO @@ -1966,11 +1966,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L7_4_46 + je L(7_4_46) ALIGN_4 -.L7_4_42: +L(7_4_42): prefetcht0 A_PR1(AO) KERNEL1x3_SUB @@ -1983,7 +1983,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_SUB KERNEL1x3_SUB - je .L7_4_46 + je L(7_4_46) prefetcht0 A_PR1(AO) KERNEL1x3_SUB @@ -1996,78 +1996,78 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_SUB KERNEL1x3_SUB - je .L7_4_46 + je L(7_4_46) - jmp .L7_4_42 + jmp L(7_4_42) ALIGN_4 -.L7_4_46: +L(7_4_46): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L7_4_49 + je L(7_4_49) ALIGN_4 -.L7_4_47: +L(7_4_47): KERNEL1x3_SUB - jnz .L7_4_47 + jnz L(7_4_47) ALIGN_4 -.L7_4_49: +L(7_4_49): SAVE1x3 addq $ 2 * SIZE, CO1 # coffset += 2 decq I # i -- - jg .L7_4_41 + jg L(7_4_41) ALIGN_4 -.L7_4_60: +L(7_4_60): decq J // j -- - jg .L6_01 // next 6 lines of N + jg L(6_01) // next 6 lines of N /************************************************************************************************/ -.L2_00: +L(2_00): movq Nmod6, J sarq $1, J // j = j / 2 cmpq $ 0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $ 4*SIZE,BO1 addq $ 4*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -2081,12 +2081,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 3, I // i = (m >> 3) - je .L2_4_10 + je L(2_4_10) ALIGN_4 /**********************************************************************************************************/ -.L2_8_11: +L(2_8_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2124,7 +2124,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_8_16 + je L(2_8_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2135,7 +2135,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_8_12: +L(2_8_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2157,7 +2157,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL8x2_SUB - je .L2_8_16 + je L(2_8_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2179,12 +2179,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL8x2_SUB - je .L2_8_16 + je L(2_8_16) - jmp .L2_8_12 + jmp L(2_8_12) ALIGN_4 -.L2_8_16: +L(2_8_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -2192,7 +2192,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_8_19 + je L(2_8_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2204,15 +2204,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_8_17: +L(2_8_17): KERNEL8x2_SUB - jl .L2_8_17 + jl L(2_8_17) ALIGN_4 -.L2_8_19: +L(2_8_19): SAVE8x2 @@ -2235,7 +2235,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_8_11 + jg L(2_8_11) ALIGN_4 @@ -2244,16 +2244,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_4_10: +L(2_4_10): testq $ 7, M - jz .L2_4_60 // to next 2 lines of N + jz L(2_4_60) // to next 2 lines of N testq $ 4, M - jz .L2_4_20 + jz L(2_4_20) ALIGN_4 -.L2_4_11: +L(2_4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2291,7 +2291,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_4_16 + je L(2_4_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2302,7 +2302,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_12: +L(2_4_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2320,7 +2320,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_4_16 + je L(2_4_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2338,12 +2338,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_4_16 + je L(2_4_16) - jmp .L2_4_12 + jmp L(2_4_12) ALIGN_4 -.L2_4_16: +L(2_4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -2351,7 +2351,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_4_19 + je L(2_4_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2363,15 +2363,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_17: +L(2_4_17): KERNEL4x2_SUB - jl .L2_4_17 + jl L(2_4_17) ALIGN_4 -.L2_4_19: +L(2_4_19): SAVE4x2 @@ -2400,13 +2400,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Rest of M ***************************************************************************/ -.L2_4_20: +L(2_4_20): testq $ 2, M - jz .L2_4_40 + jz L(2_4_40) ALIGN_4 -.L2_4_21: +L(2_4_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2444,7 +2444,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_4_26 + je L(2_4_26) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2455,7 +2455,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_22: +L(2_4_22): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2471,7 +2471,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_4_26 + je L(2_4_26) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2487,12 +2487,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_4_26 + je L(2_4_26) - jmp .L2_4_22 + jmp L(2_4_22) ALIGN_4 -.L2_4_26: +L(2_4_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -2500,7 +2500,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_4_29 + je L(2_4_29) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2512,15 +2512,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_27: +L(2_4_27): KERNEL2x2_SUB - jl .L2_4_27 + jl L(2_4_27) ALIGN_4 -.L2_4_29: +L(2_4_29): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -2596,19 +2596,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L2_4_21 + jg L(2_4_21) ALIGN_4 /**************************************************************************/ -.L2_4_40: +L(2_4_40): testq $ 1, M - jz .L2_4_60 // to next 2 lines of N + jz L(2_4_60) // to next 2 lines of N ALIGN_4 -.L2_4_41: +L(2_4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2646,7 +2646,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_4_46 + je L(2_4_46) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2657,7 +2657,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_42: +L(2_4_42): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2672,7 +2672,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_4_46 + je L(2_4_46) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2687,12 +2687,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_4_46 + je L(2_4_46) - jmp .L2_4_42 + jmp L(2_4_42) ALIGN_4 -.L2_4_46: +L(2_4_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -2700,7 +2700,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_4_49 + je L(2_4_49) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2712,15 +2712,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_47: +L(2_4_47): KERNEL1x2_SUB - jl .L2_4_47 + jl L(2_4_47) ALIGN_4 -.L2_4_49: +L(2_4_49): SAVE1x2 @@ -2742,23 +2742,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 2 * SIZE, CO1 # coffset += 2 decq I # i -- - jg .L2_4_41 + jg L(2_4_41) ALIGN_4 -.L2_4_60: +L(2_4_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $ 2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -2766,30 +2766,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $ 1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $ 2*SIZE,BO1 addq $ 2*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -2803,13 +2803,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 3, I // i = (m >> 3) - je .L1_4_10 + je L(1_4_10) ALIGN_4 /**************************************************************************************************/ -.L1_8_11: +L(1_8_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2847,7 +2847,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_8_16 + je L(1_8_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -2858,7 +2858,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_8_12: +L(1_8_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2879,7 +2879,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL8x1_SUB - je .L1_8_16 + je L(1_8_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2900,12 +2900,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL8x1_SUB - je .L1_8_16 + je L(1_8_16) - jmp .L1_8_12 + jmp L(1_8_12) ALIGN_4 -.L1_8_16: +L(1_8_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -2913,7 +2913,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_8_19 + je L(1_8_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 4 ; number of values @@ -2925,15 +2925,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_8_17: +L(1_8_17): KERNEL8x1_SUB - jl .L1_8_17 + jl L(1_8_17) ALIGN_4 -.L1_8_19: +L(1_8_19): SAVE8x1 @@ -2956,22 +2956,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_8_11 + jg L(1_8_11) ALIGN_4 /**************************************************************************************************/ -.L1_4_10: +L(1_4_10): testq $ 7, M - jz .L999 + jz L(999) testq $ 4, M - jz .L1_4_20 + jz L(1_4_20) -.L1_4_11: +L(1_4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3009,7 +3009,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_4_16 + je L(1_4_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -3020,7 +3020,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_12: +L(1_4_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3037,7 +3037,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_4_16 + je L(1_4_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3054,12 +3054,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_4_16 + je L(1_4_16) - jmp .L1_4_12 + jmp L(1_4_12) ALIGN_4 -.L1_4_16: +L(1_4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3067,7 +3067,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_4_19 + je L(1_4_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 4 ; number of values @@ -3079,15 +3079,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_17: +L(1_4_17): KERNEL4x1_SUB - jl .L1_4_17 + jl L(1_4_17) ALIGN_4 -.L1_4_19: +L(1_4_19): SAVE4x1 @@ -3116,13 +3116,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Rest of M ***************************************************************************/ -.L1_4_20: +L(1_4_20): testq $ 2, M - jz .L1_4_40 + jz L(1_4_40) ALIGN_4 -.L1_4_21: +L(1_4_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3160,7 +3160,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_4_26 + je L(1_4_26) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -3171,7 +3171,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_22: +L(1_4_22): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3186,7 +3186,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_4_26 + je L(1_4_26) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3201,12 +3201,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_4_26 + je L(1_4_26) - jmp .L1_4_22 + jmp L(1_4_22) ALIGN_4 -.L1_4_26: +L(1_4_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -3214,7 +3214,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_4_29 + je L(1_4_29) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2; number of values @@ -3226,15 +3226,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_27: +L(1_4_27): KERNEL2x1_SUB - jl .L1_4_27 + jl L(1_4_27) ALIGN_4 -.L1_4_29: +L(1_4_29): SAVE2x1 @@ -3260,13 +3260,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L1_4_40: +L(1_4_40): testq $ 1, M - jz .L999 // to next 2 lines of N + jz L(999) // to next 2 lines of N ALIGN_4 -.L1_4_41: +L(1_4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3304,7 +3304,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_4_46 + je L(1_4_46) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -3315,7 +3315,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_42: +L(1_4_42): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3329,7 +3329,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_4_46 + je L(1_4_46) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3343,12 +3343,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_4_46 + je L(1_4_46) - jmp .L1_4_42 + jmp L(1_4_42) ALIGN_4 -.L1_4_46: +L(1_4_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -3356,7 +3356,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_4_49 + je L(1_4_49) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -3368,15 +3368,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_47: +L(1_4_47): KERNEL1x1_SUB - jl .L1_4_47 + jl L(1_4_47) ALIGN_4 -.L1_4_49: +L(1_4_49): SAVE1x1 @@ -3402,7 +3402,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): vzeroupper movq SP, %rsp @@ -3493,13 +3493,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $ 0, OLD_M - je .L999 + je L(999) cmpq $ 0, OLD_N - je .L999 + je L(999) cmpq $ 0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -3527,36 +3527,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif #endif -.L2_0: +L(2_0): movq Ndiv6, J cmpq $ 0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $ 4*SIZE,BO1 addq $ 4*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -3570,12 +3570,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 3, I // i = (m >> 3) - je .L2_4_10 + je L(2_4_10) ALIGN_4 /**********************************************************************************************************/ -.L2_8_11: +L(2_8_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3613,7 +3613,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_8_16 + je L(2_8_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -3624,7 +3624,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_8_12: +L(2_8_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3646,7 +3646,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL8x2_SUB - je .L2_8_16 + je L(2_8_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3668,12 +3668,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL8x2_SUB - je .L2_8_16 + je L(2_8_16) - jmp .L2_8_12 + jmp L(2_8_12) ALIGN_4 -.L2_8_16: +L(2_8_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3681,7 +3681,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_8_19 + je L(2_8_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -3693,15 +3693,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_8_17: +L(2_8_17): KERNEL8x2_SUB - jl .L2_8_17 + jl L(2_8_17) ALIGN_4 -.L2_8_19: +L(2_8_19): SAVE8x2 @@ -3724,7 +3724,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_8_11 + jg L(2_8_11) ALIGN_4 @@ -3733,16 +3733,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_4_10: +L(2_4_10): testq $ 7, M - jz .L2_4_60 // to next 2 lines of N + jz L(2_4_60) // to next 2 lines of N testq $ 4, M - jz .L2_4_20 + jz L(2_4_20) ALIGN_4 -.L2_4_11: +L(2_4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3780,7 +3780,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_4_16 + je L(2_4_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -3791,7 +3791,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_12: +L(2_4_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3809,7 +3809,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_4_16 + je L(2_4_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3827,12 +3827,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_4_16 + je L(2_4_16) - jmp .L2_4_12 + jmp L(2_4_12) ALIGN_4 -.L2_4_16: +L(2_4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3840,7 +3840,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_4_19 + je L(2_4_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -3852,15 +3852,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_17: +L(2_4_17): KERNEL4x2_SUB - jl .L2_4_17 + jl L(2_4_17) ALIGN_4 -.L2_4_19: +L(2_4_19): SAVE4x2 @@ -3889,13 +3889,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Rest of M ***************************************************************************/ -.L2_4_20: +L(2_4_20): testq $ 2, M - jz .L2_4_40 + jz L(2_4_40) ALIGN_4 -.L2_4_21: +L(2_4_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3933,7 +3933,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_4_26 + je L(2_4_26) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -3944,7 +3944,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_22: +L(2_4_22): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3960,7 +3960,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_4_26 + je L(2_4_26) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3976,12 +3976,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_4_26 + je L(2_4_26) - jmp .L2_4_22 + jmp L(2_4_22) ALIGN_4 -.L2_4_26: +L(2_4_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -3989,7 +3989,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_4_29 + je L(2_4_29) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -4001,15 +4001,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_27: +L(2_4_27): KERNEL2x2_SUB - jl .L2_4_27 + jl L(2_4_27) ALIGN_4 -.L2_4_29: +L(2_4_29): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -4085,19 +4085,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L2_4_21 + jg L(2_4_21) ALIGN_4 /**************************************************************************/ -.L2_4_40: +L(2_4_40): testq $ 1, M - jz .L2_4_60 // to next 2 lines of N + jz L(2_4_60) // to next 2 lines of N ALIGN_4 -.L2_4_41: +L(2_4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4135,7 +4135,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_4_46 + je L(2_4_46) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -4146,7 +4146,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_42: +L(2_4_42): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -4161,7 +4161,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_4_46 + je L(2_4_46) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -4176,12 +4176,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_4_46 + je L(2_4_46) - jmp .L2_4_42 + jmp L(2_4_42) ALIGN_4 -.L2_4_46: +L(2_4_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -4189,7 +4189,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_4_49 + je L(2_4_49) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -4201,15 +4201,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_47: +L(2_4_47): KERNEL1x2_SUB - jl .L2_4_47 + jl L(2_4_47) ALIGN_4 -.L2_4_49: +L(2_4_49): SAVE1x2 @@ -4231,23 +4231,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 2 * SIZE, CO1 # coffset += 2 decq I # i -- - jg .L2_4_41 + jg L(2_4_41) ALIGN_4 -.L2_4_60: +L(2_4_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $ 2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -4255,30 +4255,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $ 1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $ 2*SIZE,BO1 addq $ 2*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -4292,13 +4292,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 3, I // i = (m >> 3) - je .L1_4_10 + je L(1_4_10) ALIGN_4 /**************************************************************************************************/ -.L1_8_11: +L(1_8_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4336,7 +4336,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_8_16 + je L(1_8_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -4347,7 +4347,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_8_12: +L(1_8_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -4368,7 +4368,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL8x1_SUB - je .L1_8_16 + je L(1_8_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -4389,12 +4389,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL8x1_SUB - je .L1_8_16 + je L(1_8_16) - jmp .L1_8_12 + jmp L(1_8_12) ALIGN_4 -.L1_8_16: +L(1_8_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -4402,7 +4402,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_8_19 + je L(1_8_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 4 ; number of values @@ -4414,15 +4414,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_8_17: +L(1_8_17): KERNEL8x1_SUB - jl .L1_8_17 + jl L(1_8_17) ALIGN_4 -.L1_8_19: +L(1_8_19): SAVE8x1 @@ -4445,22 +4445,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_8_11 + jg L(1_8_11) ALIGN_4 /**************************************************************************************************/ -.L1_4_10: +L(1_4_10): testq $ 7, M - jz .L999 + jz L(999) testq $ 4, M - jz .L1_4_20 + jz L(1_4_20) -.L1_4_11: +L(1_4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4498,7 +4498,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_4_16 + je L(1_4_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -4509,7 +4509,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_12: +L(1_4_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -4526,7 +4526,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_4_16 + je L(1_4_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -4543,12 +4543,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_4_16 + je L(1_4_16) - jmp .L1_4_12 + jmp L(1_4_12) ALIGN_4 -.L1_4_16: +L(1_4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -4556,7 +4556,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_4_19 + je L(1_4_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 4 ; number of values @@ -4568,15 +4568,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_17: +L(1_4_17): KERNEL4x1_SUB - jl .L1_4_17 + jl L(1_4_17) ALIGN_4 -.L1_4_19: +L(1_4_19): SAVE4x1 @@ -4605,13 +4605,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Rest of M ***************************************************************************/ -.L1_4_20: +L(1_4_20): testq $ 2, M - jz .L1_4_40 + jz L(1_4_40) ALIGN_4 -.L1_4_21: +L(1_4_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4649,7 +4649,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_4_26 + je L(1_4_26) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -4660,7 +4660,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_22: +L(1_4_22): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -4675,7 +4675,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_4_26 + je L(1_4_26) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -4690,12 +4690,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_4_26 + je L(1_4_26) - jmp .L1_4_22 + jmp L(1_4_22) ALIGN_4 -.L1_4_26: +L(1_4_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4703,7 +4703,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_4_29 + je L(1_4_29) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2; number of values @@ -4715,15 +4715,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_27: +L(1_4_27): KERNEL2x1_SUB - jl .L1_4_27 + jl L(1_4_27) ALIGN_4 -.L1_4_29: +L(1_4_29): SAVE2x1 @@ -4749,13 +4749,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L1_4_40: +L(1_4_40): testq $ 1, M - jz .L999 // to next 2 lines of N + jz L(999) // to next 2 lines of N ALIGN_4 -.L1_4_41: +L(1_4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4793,7 +4793,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_4_46 + je L(1_4_46) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -4804,7 +4804,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_42: +L(1_4_42): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -4818,7 +4818,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_4_46 + je L(1_4_46) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -4832,12 +4832,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_4_46 + je L(1_4_46) - jmp .L1_4_42 + jmp L(1_4_42) ALIGN_4 -.L1_4_46: +L(1_4_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -4845,7 +4845,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_4_49 + je L(1_4_49) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -4857,15 +4857,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_47: +L(1_4_47): KERNEL1x1_SUB - jl .L1_4_47 + jl L(1_4_47) ALIGN_4 -.L1_4_49: +L(1_4_49): SAVE1x1 @@ -4891,7 +4891,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): vzeroupper movq SP, %rsp diff --git a/kernel/x86_64/cgemm_kernel_8x2_sandy.S b/kernel/x86_64/cgemm_kernel_8x2_sandy.S index 988913591c..4c887ea9b1 100644 --- a/kernel/x86_64/cgemm_kernel_8x2_sandy.S +++ b/kernel/x86_64/cgemm_kernel_8x2_sandy.S @@ -963,13 +963,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $ 0, OLD_M - je .L999 + je L(999) cmpq $ 0, OLD_N - je .L999 + je L(999) cmpq $ 0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -997,36 +997,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif #endif -.L2_0: +L(2_0): movq Ndiv6, J cmpq $ 0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $ 4*SIZE,BO1 addq $ 4*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -1040,12 +1040,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 3, I // i = (m >> 3) - je .L2_4_10 + je L(2_4_10) ALIGN_4 /**********************************************************************************************************/ -.L2_8_11: +L(2_8_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1083,7 +1083,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_8_16 + je L(2_8_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1094,7 +1094,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_8_12: +L(2_8_12): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL8x2_1 @@ -1102,7 +1102,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 B_PR1(BO,BI,SIZE) KERNEL8x2_1 - je .L2_8_16 + je L(2_8_16) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL8x2_1 @@ -1110,12 +1110,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 B_PR1(BO,BI,SIZE) KERNEL8x2_1 - je .L2_8_16 + je L(2_8_16) - jmp .L2_8_12 + jmp L(2_8_12) ALIGN_4 -.L2_8_16: +L(2_8_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -1123,7 +1123,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_8_19 + je L(2_8_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1135,15 +1135,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_8_17: +L(2_8_17): KERNEL8x2_SUB - jl .L2_8_17 + jl L(2_8_17) ALIGN_4 -.L2_8_19: +L(2_8_19): SAVE8x2 @@ -1166,7 +1166,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_8_11 + jg L(2_8_11) ALIGN_4 @@ -1175,16 +1175,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_4_10: +L(2_4_10): testq $ 7, M - jz .L2_4_60 // to next 2 lines of N + jz L(2_4_60) // to next 2 lines of N testq $ 4, M - jz .L2_4_20 + jz L(2_4_20) ALIGN_4 -.L2_4_11: +L(2_4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1222,7 +1222,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_4_16 + je L(2_4_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1233,7 +1233,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_12: +L(2_4_12): prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL4x2_SUB @@ -1249,7 +1249,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_4_16 + je L(2_4_16) prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL4x2_SUB @@ -1265,12 +1265,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_4_16 + je L(2_4_16) - jmp .L2_4_12 + jmp L(2_4_12) ALIGN_4 -.L2_4_16: +L(2_4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -1278,7 +1278,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_4_19 + je L(2_4_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1290,15 +1290,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_17: +L(2_4_17): KERNEL4x2_SUB - jl .L2_4_17 + jl L(2_4_17) ALIGN_4 -.L2_4_19: +L(2_4_19): SAVE4x2 @@ -1327,13 +1327,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Rest of M ***************************************************************************/ -.L2_4_20: +L(2_4_20): testq $ 2, M - jz .L2_4_40 + jz L(2_4_40) ALIGN_4 -.L2_4_21: +L(2_4_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1371,7 +1371,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_4_26 + je L(2_4_26) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1382,7 +1382,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_22: +L(2_4_22): prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL2x2_SUB @@ -1396,7 +1396,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_4_26 + je L(2_4_26) prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL2x2_SUB @@ -1410,12 +1410,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_4_26 + je L(2_4_26) - jmp .L2_4_22 + jmp L(2_4_22) ALIGN_4 -.L2_4_26: +L(2_4_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -1423,7 +1423,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_4_29 + je L(2_4_29) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1435,15 +1435,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_27: +L(2_4_27): KERNEL2x2_SUB - jl .L2_4_27 + jl L(2_4_27) ALIGN_4 -.L2_4_29: +L(2_4_29): vbroadcastss ALPHA_R, %xmm0 vbroadcastss ALPHA_I, %xmm1 @@ -1519,19 +1519,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L2_4_21 + jg L(2_4_21) ALIGN_4 /**************************************************************************/ -.L2_4_40: +L(2_4_40): testq $ 1, M - jz .L2_4_60 // to next 2 lines of N + jz L(2_4_60) // to next 2 lines of N ALIGN_4 -.L2_4_41: +L(2_4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1569,7 +1569,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_4_46 + je L(2_4_46) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1580,7 +1580,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_42: +L(2_4_42): prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL1x2_SUB @@ -1593,7 +1593,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_4_46 + je L(2_4_46) prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL1x2_SUB @@ -1606,12 +1606,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_4_46 + je L(2_4_46) - jmp .L2_4_42 + jmp L(2_4_42) ALIGN_4 -.L2_4_46: +L(2_4_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -1619,7 +1619,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_4_49 + je L(2_4_49) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1631,15 +1631,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_47: +L(2_4_47): KERNEL1x2_SUB - jl .L2_4_47 + jl L(2_4_47) ALIGN_4 -.L2_4_49: +L(2_4_49): SAVE1x2 @@ -1661,23 +1661,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 2 * SIZE, CO1 # coffset += 2 decq I # i -- - jg .L2_4_41 + jg L(2_4_41) ALIGN_4 -.L2_4_60: +L(2_4_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $ 2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -1685,30 +1685,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $ 1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $ 2*SIZE,BO1 addq $ 2*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -1722,13 +1722,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 3, I // i = (m >> 3) - je .L1_4_10 + je L(1_4_10) ALIGN_4 /**************************************************************************************************/ -.L1_8_11: +L(1_8_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1766,7 +1766,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_8_16 + je L(1_8_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1777,7 +1777,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_8_12: +L(1_8_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -1798,7 +1798,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL8x1_SUB - je .L1_8_16 + je L(1_8_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -1819,12 +1819,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL8x1_SUB - je .L1_8_16 + je L(1_8_16) - jmp .L1_8_12 + jmp L(1_8_12) ALIGN_4 -.L1_8_16: +L(1_8_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -1832,7 +1832,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_8_19 + je L(1_8_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 4 ; number of values @@ -1844,15 +1844,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_8_17: +L(1_8_17): KERNEL8x1_SUB - jl .L1_8_17 + jl L(1_8_17) ALIGN_4 -.L1_8_19: +L(1_8_19): SAVE8x1 @@ -1875,22 +1875,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_8_11 + jg L(1_8_11) ALIGN_4 /**************************************************************************************************/ -.L1_4_10: +L(1_4_10): testq $ 7, M - jz .L999 + jz L(999) testq $ 4, M - jz .L1_4_20 + jz L(1_4_20) -.L1_4_11: +L(1_4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1928,7 +1928,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_4_16 + je L(1_4_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1939,7 +1939,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_12: +L(1_4_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -1956,7 +1956,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_4_16 + je L(1_4_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -1973,12 +1973,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_4_16 + je L(1_4_16) - jmp .L1_4_12 + jmp L(1_4_12) ALIGN_4 -.L1_4_16: +L(1_4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -1986,7 +1986,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_4_19 + je L(1_4_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 4 ; number of values @@ -1998,15 +1998,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_17: +L(1_4_17): KERNEL4x1_SUB - jl .L1_4_17 + jl L(1_4_17) ALIGN_4 -.L1_4_19: +L(1_4_19): SAVE4x1 @@ -2035,13 +2035,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Rest of M ***************************************************************************/ -.L1_4_20: +L(1_4_20): testq $ 2, M - jz .L1_4_40 + jz L(1_4_40) ALIGN_4 -.L1_4_21: +L(1_4_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2079,7 +2079,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_4_26 + je L(1_4_26) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -2090,7 +2090,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_22: +L(1_4_22): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2105,7 +2105,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_4_26 + je L(1_4_26) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2120,12 +2120,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_4_26 + je L(1_4_26) - jmp .L1_4_22 + jmp L(1_4_22) ALIGN_4 -.L1_4_26: +L(1_4_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -2133,7 +2133,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_4_29 + je L(1_4_29) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2; number of values @@ -2145,15 +2145,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_27: +L(1_4_27): KERNEL2x1_SUB - jl .L1_4_27 + jl L(1_4_27) ALIGN_4 -.L1_4_29: +L(1_4_29): SAVE2x1 @@ -2179,13 +2179,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L1_4_40: +L(1_4_40): testq $ 1, M - jz .L999 // to next 2 lines of N + jz L(999) // to next 2 lines of N ALIGN_4 -.L1_4_41: +L(1_4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2223,7 +2223,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_4_46 + je L(1_4_46) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -2234,7 +2234,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_42: +L(1_4_42): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2248,7 +2248,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_4_46 + je L(1_4_46) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2262,12 +2262,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_4_46 + je L(1_4_46) - jmp .L1_4_42 + jmp L(1_4_42) ALIGN_4 -.L1_4_46: +L(1_4_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -2275,7 +2275,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_4_49 + je L(1_4_49) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -2287,15 +2287,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_47: +L(1_4_47): KERNEL1x1_SUB - jl .L1_4_47 + jl L(1_4_47) ALIGN_4 -.L1_4_49: +L(1_4_49): SAVE1x1 @@ -2321,7 +2321,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): vzeroupper movq SP, %rsp diff --git a/kernel/x86_64/cgemv_n.S b/kernel/x86_64/cgemv_n.S index 206beb6735..1b90b5deec 100644 --- a/kernel/x86_64/cgemv_n.S +++ b/kernel/x86_64/cgemv_n.S @@ -167,7 +167,7 @@ movss %xmm0,ALPHAR movss %xmm1,ALPHAI -.L0t: +L(0t): xorq I,I addq $1,I salq $20,I @@ -175,13 +175,13 @@ movq I,M movss ALPHAR,%xmm0 movss ALPHAI,%xmm1 - jge .L00t + jge L(00t) movq MMM,M addq I,M - jle .L999x + jle L(999x) -.L00t: +L(00t): movq AA, A movq NN, N movq LDAX, LDA @@ -201,9 +201,9 @@ movlps %xmm0, ALPHA testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) ALIGN_3 subq $-32 * SIZE, A @@ -217,7 +217,7 @@ sarq $3, %rax ALIGN_3 -.L01: +L(01): movaps %xmm4, 0 * SIZE(Y1) movaps %xmm4, 4 * SIZE(Y1) movaps %xmm4, 8 * SIZE(Y1) @@ -225,10 +225,10 @@ subq $-16 * SIZE, Y1 decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L10: +L(10): #ifdef ALIGNED_ACCESS movq M, MM @@ -242,18 +242,18 @@ cmovge A2, MM testq $SIZE, A - jne .L200 + jne L(200) testq $2 * SIZE, LDA - jne .L100 + jne L(100) #endif #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $4, N leaq 32 * SIZE(BUFFER), Y1 @@ -347,7 +347,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L1X + je L(1X) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA), %xmm6 @@ -386,7 +386,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, Y1 ALIGN_3 -.L1X: +L(1X): #endif movaps -32 * SIZE(Y1), %xmm0 @@ -396,16 +396,16 @@ movq MM, I sarq $3, I - jle .L15 + jle L(15) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) decq I - jle .L14 + jle L(14) ALIGN_3 -.L13: +L(13): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -554,10 +554,10 @@ subq $1, I BRANCH - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): pshufd $0xb1, %xmm4, %xmm5 mulps %xmm8, %xmm4 addps %xmm4, %xmm0 @@ -683,9 +683,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L15: +L(15): testq $4, MM - je .L17 + je L(17) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) @@ -758,9 +758,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L17: +L(17): testq $2, MM - je .L18 + je L(18) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A2(-32 * SIZE, A1, LDA, 1, %xmm6) @@ -799,9 +799,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L18: +L(18): testq $1, MM - je .L19 + je L(19) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA), %xmm6 @@ -835,21 +835,21 @@ movlps %xmm0, -32 * SIZE(Y1) ALIGN_3 -.L19: +L(19): cmpq $4, N - jge .L11 + jge L(11) ALIGN_3 -.L20: +L(20): #endif cmpq $2, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 2 ALIGN_3 -.L21: +L(21): #endif subq $2, N @@ -917,7 +917,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L2X + je L(2X) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A2), %xmm6 @@ -943,7 +943,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, Y1 ALIGN_3 -.L2X: +L(2X): #endif movaps -32 * SIZE(Y1), %xmm0 @@ -954,7 +954,7 @@ movq MM, I sarq $3, I - jle .L25 + jle L(25) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) @@ -962,10 +962,10 @@ MOVUPS_A1(-20 * SIZE, A1, %xmm10) decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A2) #endif @@ -1050,10 +1050,10 @@ subq $1, I BRANCH - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): pshufd $0xb1, %xmm4, %xmm5 mulps %xmm12, %xmm4 addps %xmm4, %xmm0 @@ -1121,9 +1121,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L25: +L(25): testq $4, MM - je .L27 + je L(27) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) @@ -1166,9 +1166,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L27: +L(27): testq $2, MM - je .L28 + je L(28) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-32 * SIZE, A2, %xmm6) @@ -1193,12 +1193,12 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L28: +L(28): testq $1, MM #if GEMV_UNROLL == 2 - je .L29 + je L(29) #else - je .L30 + je L(30) #endif movsd -32 * SIZE(A1), %xmm4 @@ -1222,15 +1222,15 @@ #if GEMV_UNROLL == 2 ALIGN_3 -.L29: +L(29): cmpq $2, N - jge .L21 + jge L(21) #endif ALIGN_3 -.L30: +L(30): cmpq $1, N - jl .L990 + jl L(990) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -1279,7 +1279,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L3X + je L(3X) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(Y1), %xmm0 @@ -1296,7 +1296,7 @@ addq $2 * SIZE, A1 addq $2 * SIZE, Y1 ALIGN_3 -.L3X: +L(3X): #endif movaps -32 * SIZE(Y1), %xmm0 @@ -1307,7 +1307,7 @@ movq MM, I sarq $3, I - jle .L35 + jle L(35) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) @@ -1315,10 +1315,10 @@ MOVUPS_A1(-20 * SIZE, A1, %xmm10) decq I - jle .L34 + jle L(34) ALIGN_3 -.L33: +L(33): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -1371,10 +1371,10 @@ subq $1, I BRANCH - jg .L33 + jg L(33) ALIGN_3 -.L34: +L(34): pshufd $0xb1, %xmm4, %xmm5 mulps %xmm12, %xmm4 addps %xmm4, %xmm0 @@ -1413,9 +1413,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L35: +L(35): testq $4, MM - je .L37 + je L(37) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) @@ -1442,9 +1442,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L37: +L(37): testq $2, MM - je .L38 + je L(38) MOVUPS_A1(-32 * SIZE, A1, %xmm4) @@ -1462,9 +1462,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L38: +L(38): testq $1, MM - je .L990 + je L(990) movsd -32 * SIZE(A1), %xmm4 @@ -1478,16 +1478,16 @@ movlps %xmm0, -32 * SIZE(Y1) #ifdef ALIGNED_ACCESS - jmp .L990 + jmp L(990) ALIGN_3 -.L100: +L(100): #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L110 + jl L(110) ALIGN_3 -.L101: +L(101): subq $4, N leaq 32 * SIZE(BUFFER), Y1 @@ -1581,7 +1581,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L10X + je L(10X) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA), %xmm6 @@ -1620,7 +1620,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, Y1 ALIGN_3 -.L10X: +L(10X): #endif movaps -32 * SIZE(Y1), %xmm0 @@ -1630,16 +1630,16 @@ movq MM, I sarq $3, I - jle .L105 + jle L(105) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) decq I - jle .L104 + jle L(104) ALIGN_3 -.L103: +L(103): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -1796,10 +1796,10 @@ subq $1, I BRANCH - jg .L103 + jg L(103) ALIGN_3 -.L104: +L(104): pshufd $0xb1, %xmm4, %xmm5 mulps %xmm8, %xmm4 addps %xmm4, %xmm0 @@ -1933,9 +1933,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L105: +L(105): testq $4, MM - je .L107 + je L(107) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) @@ -2009,9 +2009,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L107: +L(107): testq $2, MM - je .L108 + je L(108) MOVUPS_A1(-32 * SIZE, A1, %xmm4) movsd -32 * SIZE(A1, LDA), %xmm6 @@ -2052,9 +2052,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L108: +L(108): testq $1, MM - je .L109 + je L(109) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA), %xmm6 @@ -2088,23 +2088,23 @@ movlps %xmm0, -32 * SIZE(Y1) ALIGN_3 -.L109: +L(109): cmpq $4, N - jge .L101 + jge L(101) ALIGN_3 -.L110: +L(110): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L120 + jl L(120) #if GEMV_UNROLL == 2 ALIGN_3 -.L111: +L(111): #endif subq $2, N @@ -2172,7 +2172,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L11X + je L(11X) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A2), %xmm6 @@ -2198,7 +2198,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, Y1 ALIGN_3 -.L11X: +L(11X): #endif movaps -32 * SIZE(Y1), %xmm0 @@ -2209,7 +2209,7 @@ movq MM, I sarq $3, I - jle .L115 + jle L(115) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) @@ -2217,10 +2217,10 @@ MOVUPS_A1(-20 * SIZE, A1, %xmm10) decq I - jle .L114 + jle L(114) ALIGN_3 -.L113: +L(113): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A2) #endif @@ -2309,10 +2309,10 @@ subq $1, I BRANCH - jg .L113 + jg L(113) ALIGN_3 -.L114: +L(114): pshufd $0xb1, %xmm4, %xmm5 mulps %xmm12, %xmm4 addps %xmm4, %xmm0 @@ -2384,9 +2384,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L115: +L(115): testq $4, MM - je .L117 + je L(117) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) @@ -2431,9 +2431,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L117: +L(117): testq $2, MM - je .L118 + je L(118) MOVUPS_A1(-32 * SIZE, A1, %xmm4) movsd -32 * SIZE(A2), %xmm6 @@ -2459,12 +2459,12 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L118: +L(118): testq $1, MM #if GEMV_UNROLL == 2 - je .L119 + je L(119) #else - je .L120 + je L(120) #endif movsd -32 * SIZE(A1), %xmm4 @@ -2488,17 +2488,17 @@ #if GEMV_UNROLL == 2 ALIGN_3 -.L119: +L(119): cmpq $2, N - jge .L111 + jge L(111) #endif ALIGN_3 -.L120: +L(120): #endif cmpq $1, N - jl .L990 + jl L(990) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -2547,7 +2547,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L12X + je L(12X) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(Y1), %xmm0 @@ -2564,7 +2564,7 @@ addq $2 * SIZE, A1 addq $2 * SIZE, Y1 ALIGN_3 -.L12X: +L(12X): #endif movaps -32 * SIZE(Y1), %xmm0 @@ -2575,7 +2575,7 @@ movq MM, I sarq $3, I - jle .L125 + jle L(125) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) @@ -2583,10 +2583,10 @@ MOVUPS_A1(-20 * SIZE, A1, %xmm10) decq I - jle .L124 + jle L(124) ALIGN_3 -.L123: +L(123): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -2639,10 +2639,10 @@ subq $1, I BRANCH - jg .L123 + jg L(123) ALIGN_3 -.L124: +L(124): pshufd $0xb1, %xmm4, %xmm5 mulps %xmm12, %xmm4 addps %xmm4, %xmm0 @@ -2681,9 +2681,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L125: +L(125): testq $4, MM - je .L127 + je L(127) MOVUPS_A1(-32 * SIZE, A1, %xmm4) MOVUPS_A1(-28 * SIZE, A1, %xmm6) @@ -2710,9 +2710,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L127: +L(127): testq $2, MM - je .L128 + je L(128) MOVUPS_A1(-32 * SIZE, A1, %xmm4) @@ -2730,9 +2730,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L128: +L(128): testq $1, MM - je .L990 + je L(990) movsd -32 * SIZE(A1), %xmm4 @@ -2745,18 +2745,18 @@ movlps %xmm0, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_3 -.L200: +L(200): testq $2 * SIZE, LDA - jne .L300 + jne L(300) cmpq $2, N - jl .L210 + jl L(210) ALIGN_3 -.L201: +L(201): subq $2, N leaq 32 * SIZE(BUFFER), Y1 @@ -2822,7 +2822,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L20X + je L(20X) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A2), %xmm6 @@ -2848,7 +2848,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, Y1 ALIGN_3 -.L20X: +L(20X): #endif movaps -33 * SIZE(A1), %xmm4 @@ -2861,17 +2861,17 @@ movq MM, I sarq $3, I - jle .L205 + jle L(205) movaps -29 * SIZE(A1), %xmm8 movaps -25 * SIZE(A1), %xmm9 movaps -21 * SIZE(A1), %xmm10 decq I - jle .L204 + jle L(204) ALIGN_3 -.L203: +L(203): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A2) #endif @@ -2976,10 +2976,10 @@ subq $1, I BRANCH - jg .L203 + jg L(203) ALIGN_3 -.L204: +L(204): movss %xmm8, %xmm4 shufps $0x39, %xmm4, %xmm4 pshufd $0xb1, %xmm4, %xmm5 @@ -3068,9 +3068,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L205: +L(205): testq $4, MM - je .L207 + je L(207) movaps -29 * SIZE(A1), %xmm8 movaps -25 * SIZE(A1), %xmm9 @@ -3124,9 +3124,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L207: +L(207): testq $2, MM - je .L208 + je L(208) movaps -29 * SIZE(A1), %xmm8 movaps -29 * SIZE(A2), %xmm9 @@ -3156,9 +3156,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L208: +L(208): testq $1, MM - je .L209 + je L(209) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A2), %xmm6 @@ -3179,14 +3179,14 @@ movlps %xmm0, -32 * SIZE(Y1) ALIGN_3 -.L209: +L(209): cmpq $2, N - jge .L201 + jge L(201) ALIGN_3 -.L210: +L(210): cmpq $1, N - jl .L990 + jl L(990) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -3235,7 +3235,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L21X + je L(21X) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(Y1), %xmm0 @@ -3252,7 +3252,7 @@ addq $2 * SIZE, A1 addq $2 * SIZE, Y1 ALIGN_3 -.L21X: +L(21X): #endif movaps -33 * SIZE(A1), %xmm4 @@ -3264,17 +3264,17 @@ movq MM, I sarq $3, I - jle .L215 + jle L(215) movaps -29 * SIZE(A1), %xmm6 movaps -25 * SIZE(A1), %xmm8 movaps -21 * SIZE(A1), %xmm10 decq I - jle .L214 + jle L(214) ALIGN_3 -.L213: +L(213): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -3337,10 +3337,10 @@ subq $1, I BRANCH - jg .L213 + jg L(213) ALIGN_3 -.L214: +L(214): movss %xmm6, %xmm4 shufps $0x39, %xmm4, %xmm4 pshufd $0xb1, %xmm4, %xmm5 @@ -3390,9 +3390,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L215: +L(215): testq $4, MM - je .L217 + je L(217) movaps -29 * SIZE(A1), %xmm6 movaps -25 * SIZE(A1), %xmm8 @@ -3425,9 +3425,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L217: +L(217): testq $2, MM - je .L218 + je L(218) movaps -29 * SIZE(A1), %xmm6 @@ -3447,9 +3447,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L218: +L(218): testq $1, MM - je .L990 + je L(990) movsd -32 * SIZE(A1), %xmm4 @@ -3461,15 +3461,15 @@ SUBPS %xmm5, %xmm0 movlps %xmm0, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_3 -.L300: +L(300): cmpq $2, N - jl .L310 + jl L(310) ALIGN_3 -.L301: +L(301): subq $2, N leaq 32 * SIZE(BUFFER), Y1 @@ -3535,7 +3535,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L30X + je L(30X) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A2), %xmm6 @@ -3561,7 +3561,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, Y1 ALIGN_3 -.L30X: +L(30X): #endif movaps -33 * SIZE(A1), %xmm4 @@ -3574,17 +3574,17 @@ movq MM, I sarq $3, I - jle .L305 + jle L(305) movaps -29 * SIZE(A1), %xmm8 movaps -25 * SIZE(A1), %xmm9 movaps -21 * SIZE(A1), %xmm10 decq I - jle .L304 + jle L(304) ALIGN_3 -.L303: +L(303): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A2) #endif @@ -3689,10 +3689,10 @@ subq $1, I BRANCH - jg .L303 + jg L(303) ALIGN_3 -.L304: +L(304): movss %xmm8, %xmm4 shufps $0x39, %xmm4, %xmm4 pshufd $0xb1, %xmm4, %xmm5 @@ -3781,9 +3781,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L305: +L(305): testq $4, MM - je .L307 + je L(307) movaps -29 * SIZE(A1), %xmm8 movaps -25 * SIZE(A1), %xmm9 @@ -3837,9 +3837,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L307: +L(307): testq $2, MM - je .L308 + je L(308) movaps -29 * SIZE(A1), %xmm8 movaps -31 * SIZE(A2), %xmm9 @@ -3869,9 +3869,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L308: +L(308): testq $1, MM - je .L309 + je L(309) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A2), %xmm6 @@ -3892,14 +3892,14 @@ movlps %xmm0, -32 * SIZE(Y1) ALIGN_3 -.L309: +L(309): cmpq $2, N - jge .L301 + jge L(301) ALIGN_3 -.L310: +L(310): cmpq $1, N - jl .L990 + jl L(990) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -3948,7 +3948,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L31X + je L(31X) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(Y1), %xmm0 @@ -3965,7 +3965,7 @@ addq $2 * SIZE, A1 addq $2 * SIZE, Y1 ALIGN_3 -.L31X: +L(31X): #endif movaps -33 * SIZE(A1), %xmm4 @@ -3977,17 +3977,17 @@ movq MM, I sarq $3, I - jle .L315 + jle L(315) movaps -29 * SIZE(A1), %xmm6 movaps -25 * SIZE(A1), %xmm8 movaps -21 * SIZE(A1), %xmm10 decq I - jle .L314 + jle L(314) ALIGN_3 -.L313: +L(313): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -4050,10 +4050,10 @@ subq $1, I BRANCH - jg .L313 + jg L(313) ALIGN_3 -.L314: +L(314): movss %xmm6, %xmm4 shufps $0x39, %xmm4, %xmm4 pshufd $0xb1, %xmm4, %xmm5 @@ -4103,9 +4103,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L315: +L(315): testq $4, MM - je .L317 + je L(317) movaps -29 * SIZE(A1), %xmm6 movaps -25 * SIZE(A1), %xmm8 @@ -4138,9 +4138,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L317: +L(317): testq $2, MM - je .L318 + je L(318) movaps -29 * SIZE(A1), %xmm6 @@ -4160,9 +4160,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L318: +L(318): testq $1, MM - je .L990 + je L(990) movsd -32 * SIZE(A1), %xmm4 @@ -4177,12 +4177,12 @@ #endif ALIGN_3 -.L990: +L(990): movq Y, Y1 #ifdef ALIGNED_ACCESS cmpq M, MM - je .L991 + je L(991) movsd (Y), %xmm0 addq INCY, Y @@ -4196,14 +4196,14 @@ addq INCY, Y1 ALIGN_3 -.L991: +L(991): #endif movq MM, %rax sarq $3, %rax - jle .L994 + jle L(994) ALIGN_3 -.L992: +L(992): movsd (Y), %xmm0 addq INCY, Y movhps (Y), %xmm0 @@ -4251,15 +4251,15 @@ addq $16 * SIZE, BUFFER decq %rax - jg .L992 + jg L(992) ALIGN_3 -.L994: +L(994): testq $7, MM - jle .L999 + jle L(999) testq $4, MM - jle .L995 + jle L(995) movsd (Y), %xmm0 addq INCY, Y @@ -4287,9 +4287,9 @@ addq $8 * SIZE, BUFFER ALIGN_3 -.L995: +L(995): testq $2, MM - jle .L996 + jle L(996) movsd (Y), %xmm0 addq INCY, Y @@ -4306,9 +4306,9 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L996: +L(996): testq $1, MM - jle .L999 + jle L(999) movsd (Y), %xmm0 @@ -4317,12 +4317,12 @@ movlps %xmm0, (Y1) ALIGN_3 -.L999: +L(999): movq M, I salq $ZBASE_SHIFT,I addq I,AA - jmp .L0t -.L999x: + jmp L(0t) +L(999x): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/cgemv_t.S b/kernel/x86_64/cgemv_t.S index 430586bba2..a32be60dea 100644 --- a/kernel/x86_64/cgemv_t.S +++ b/kernel/x86_64/cgemv_t.S @@ -164,7 +164,7 @@ movss %xmm0,ALPHAR movss %xmm1,ALPHAI -.L0t: +L(0t): xorq I,I addq $1,I salq $20,I @@ -172,13 +172,13 @@ movq I,M movss ALPHAR,%xmm0 movss ALPHAI,%xmm1 - jge .L00t + jge L(00t) movq MMM,M addq I,M - jle .L999x + jle L(999x) -.L00t: +L(00t): movq AA, A movq NN, N movq LDAX, LDA @@ -197,9 +197,9 @@ movlps %xmm0, ALPHA testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) subq $-32 * SIZE, A @@ -211,7 +211,7 @@ andq $4 * SIZE - 1, %rax cmpq $2 * SIZE, %rax - jl .L0X + jl L(0X) movsd (X), %xmm0 addq INCX, X @@ -221,15 +221,15 @@ addq $4 * SIZE, X1 decq MM -.L0X: +L(0X): #endif movq MM, I sarq $3, I - jle .L05 + jle L(05) ALIGN_4 -.L02: +L(02): movsd (X), %xmm0 addq INCX, X movhps (X), %xmm0 @@ -257,41 +257,41 @@ addq $16 * SIZE, X1 decq I - jg .L02 + jg L(02) ALIGN_4 -.L05: +L(05): movq MM, I andq $7, I - jle .L10 + jle L(10) ALIGN_2 -.L06: +L(06): movsd (X), %xmm0 addq INCX, X movlps %xmm0, 0 * SIZE(X1) addq $2 * SIZE, X1 decq I - jg .L06 + jg L(06) ALIGN_4 -.L10: +L(10): movq Y, Y1 #ifdef ALIGNED_ACCESS testq $SIZE, A - jne .L200 + jne L(200) testq $2 * SIZE, LDA - jne .L100 + jne L(100) #endif #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $4, N leaq 32 * SIZE(BUFFER), X1 @@ -311,7 +311,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L1X + je L(1X) #ifdef movsd xorps %xmm8, %xmm8 @@ -363,7 +363,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, X1 ALIGN_3 -.L1X: +L(1X): #endif movaps -32 * SIZE(X1), %xmm12 @@ -375,7 +375,7 @@ movq MM, I sarq $3, I - jle .L15 + jle L(15) MOVUPS_A1(-32 * SIZE, A1, %xmm8) MOVUPS_A2(-32 * SIZE, A1, LDA, 1, %xmm9) @@ -383,10 +383,10 @@ MOVUPS_A2(-32 * SIZE, A2, LDA, 1, %xmm11) decq I - jle .L14 + jle L(14) ALIGN_3 -.L13: +L(13): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -529,10 +529,10 @@ subq $1, I BRANCH - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): pshufd $0xb1, %xmm8, %xmm14 mulps %xmm12, %xmm8 addps %xmm8, %xmm0 @@ -650,9 +650,9 @@ subq $-16 * SIZE, X1 ALIGN_3 -.L15: +L(15): testq $4, MM - je .L17 + je L(17) MOVUPS_A1(-32 * SIZE, A1, %xmm8) MOVUPS_A2(-32 * SIZE, A1, LDA, 1, %xmm9) @@ -718,9 +718,9 @@ addq $8 * SIZE, X1 ALIGN_3 -.L17: +L(17): testq $2, MM - je .L18 + je L(18) MOVUPS_A1(-32 * SIZE, A1, %xmm8) MOVUPS_A2(-32 * SIZE, A1, LDA, 1, %xmm9) @@ -757,9 +757,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L18: +L(18): testq $1, MM - je .L19 + je L(19) #ifdef movsd xorps %xmm8, %xmm8 @@ -803,7 +803,7 @@ SUBPS %xmm15, %xmm7 ALIGN_3 -.L19: +L(19): pcmpeqb %xmm11, %xmm11 psllq $63, %xmm11 @@ -923,18 +923,18 @@ addq INCY, Y1 cmpq $4, N - jge .L11 + jge L(11) ALIGN_3 -.L20: +L(20): #endif cmpq $2, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 2 ALIGN_3 -.L21: +L(21): #endif subq $2, N @@ -951,7 +951,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L2X + je L(2X) #ifdef movsd xorps %xmm8, %xmm8 @@ -983,7 +983,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, X1 ALIGN_3 -.L2X: +L(2X): #endif movaps -32 * SIZE(X1), %xmm12 @@ -995,7 +995,7 @@ movq MM, I sarq $3, I - jle .L25 + jle L(25) MOVUPS_A1(-32 * SIZE, A1, %xmm8) MOVUPS_A1(-32 * SIZE, A2, %xmm9) @@ -1003,10 +1003,10 @@ MOVUPS_A1(-28 * SIZE, A2, %xmm11) decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -1089,10 +1089,10 @@ subq $1, I BRANCH - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): pshufd $0xb1, %xmm8, %xmm4 mulps %xmm12, %xmm8 addps %xmm8, %xmm0 @@ -1158,9 +1158,9 @@ subq $-16 * SIZE, X1 ALIGN_3 -.L25: +L(25): testq $4, MM - je .L27 + je L(27) MOVUPS_A1(-32 * SIZE, A1, %xmm8) MOVUPS_A1(-32 * SIZE, A2, %xmm9) @@ -1199,9 +1199,9 @@ addq $8 * SIZE, A2 ALIGN_3 -.L27: +L(27): testq $2, MM - je .L28 + je L(28) MOVUPS_A1(-32 * SIZE, A1, %xmm8) MOVUPS_A1(-32 * SIZE, A2, %xmm9) @@ -1224,9 +1224,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L28: +L(28): testq $1, MM - je .L29 + je L(29) #ifdef movsd xorps %xmm8, %xmm8 @@ -1250,7 +1250,7 @@ SUBPS %xmm5, %xmm3 ALIGN_3 -.L29: +L(29): pcmpeqb %xmm5, %xmm5 psllq $63, %xmm5 @@ -1326,13 +1326,13 @@ #if GEMV_UNROLL == 2 cmpq $2, N - jge .L21 + jge L(21) #endif ALIGN_3 -.L30: +L(30): cmpq $1, N - jl .L999 + jl L(999) leaq 32 * SIZE(BUFFER), X1 @@ -1343,7 +1343,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L3X + je L(3X) #ifdef movsd xorps %xmm8, %xmm8 @@ -1363,7 +1363,7 @@ addq $2 * SIZE, A1 addq $2 * SIZE, X1 ALIGN_3 -.L3X: +L(3X): #endif movaps -32 * SIZE(X1), %xmm12 @@ -1371,16 +1371,16 @@ movq MM, I sarq $3, I - jle .L35 + jle L(35) MOVUPS_A1(-32 * SIZE, A1, %xmm8) MOVUPS_A1(-28 * SIZE, A1, %xmm10) decq I - jle .L34 + jle L(34) ALIGN_3 -.L33: +L(33): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -1430,10 +1430,10 @@ subq $1, I BRANCH - jg .L33 + jg L(33) ALIGN_3 -.L34: +L(34): pshufd $0xb1, %xmm8, %xmm4 mulps %xmm12, %xmm8 addps %xmm8, %xmm0 @@ -1472,9 +1472,9 @@ subq $-16 * SIZE, X1 ALIGN_3 -.L35: +L(35): testq $4, MM - je .L37 + je L(37) MOVUPS_A1(-32 * SIZE, A1, %xmm8) MOVUPS_A1(-28 * SIZE, A1, %xmm10) @@ -1498,9 +1498,9 @@ addq $8 * SIZE, A1 ALIGN_3 -.L37: +L(37): testq $2, MM - je .L38 + je L(38) MOVUPS_A1(-32 * SIZE, A1, %xmm8) @@ -1515,9 +1515,9 @@ addq $4 * SIZE, A1 ALIGN_3 -.L38: +L(38): testq $1, MM - je .L39 + je L(39) #ifdef movsd xorps %xmm8, %xmm8 @@ -1531,7 +1531,7 @@ SUBPS %xmm4, %xmm1 ALIGN_3 -.L39: +L(39): pcmpeqb %xmm5, %xmm5 psllq $63, %xmm5 @@ -1590,17 +1590,17 @@ movlps %xmm0, (Y1) addq INCY, Y1 #ifdef ALIGNED_ACCESS - jmp .L999 + jmp L(999) ALIGN_3 -.L100: +L(100): #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L110 + jl L(110) ALIGN_3 -.L101: +L(101): subq $4, N leaq 32 * SIZE(BUFFER), X1 @@ -1620,7 +1620,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L10X + je L(10X) #ifdef movsd xorps %xmm8, %xmm8 @@ -1672,7 +1672,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, X1 ALIGN_3 -.L10X: +L(10X): #endif movaps -32 * SIZE(X1), %xmm12 @@ -1684,7 +1684,7 @@ movq MM, I sarq $3, I - jle .L105 + jle L(105) movaps -32 * SIZE(A1), %xmm8 movsd -32 * SIZE(A1, LDA), %xmm9 @@ -1695,10 +1695,10 @@ movhps -30 * SIZE(A2, LDA), %xmm11 decq I - jle .L104 + jle L(104) ALIGN_3 -.L103: +L(103): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -1849,10 +1849,10 @@ subq $1, I BRANCH - jg .L103 + jg L(103) ALIGN_3 -.L104: +L(104): pshufd $0xb1, %xmm8, %xmm14 mulps %xmm12, %xmm8 addps %xmm8, %xmm0 @@ -1976,9 +1976,9 @@ subq $-16 * SIZE, X1 ALIGN_3 -.L105: +L(105): testq $4, MM - je .L107 + je L(107) movaps -32 * SIZE(A1), %xmm8 movsd -32 * SIZE(A1, LDA), %xmm9 @@ -2049,9 +2049,9 @@ addq $8 * SIZE, X1 ALIGN_3 -.L107: +L(107): testq $2, MM - je .L108 + je L(108) movaps -32 * SIZE(A1), %xmm8 movsd -32 * SIZE(A1, LDA), %xmm9 @@ -2091,9 +2091,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L108: +L(108): testq $1, MM - je .L109 + je L(109) #ifdef movsd xorps %xmm8, %xmm8 @@ -2137,7 +2137,7 @@ SUBPS %xmm15, %xmm7 ALIGN_3 -.L109: +L(109): pcmpeqb %xmm11, %xmm11 psllq $63, %xmm11 @@ -2257,18 +2257,18 @@ addq INCY, Y1 cmpq $4, N - jge .L101 + jge L(101) ALIGN_3 -.L110: +L(110): #endif cmpq $2, N - jl .L120 + jl L(120) #if GEMV_UNROLL == 2 ALIGN_3 -.L111: +L(111): #endif subq $2, N @@ -2285,7 +2285,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L11X + je L(11X) #ifdef movsd xorps %xmm8, %xmm8 @@ -2317,7 +2317,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, X1 ALIGN_3 -.L11X: +L(11X): #endif movaps -32 * SIZE(X1), %xmm12 @@ -2329,7 +2329,7 @@ movq MM, I sarq $3, I - jle .L115 + jle L(115) movaps -32 * SIZE(A1), %xmm8 movsd -32 * SIZE(A2), %xmm9 @@ -2340,10 +2340,10 @@ movhps -26 * SIZE(A2), %xmm11 decq I - jle .L114 + jle L(114) ALIGN_3 -.L113: +L(113): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -2430,10 +2430,10 @@ subq $1, I BRANCH - jg .L113 + jg L(113) ALIGN_3 -.L114: +L(114): pshufd $0xb1, %xmm8, %xmm4 mulps %xmm12, %xmm8 addps %xmm8, %xmm0 @@ -2501,9 +2501,9 @@ subq $-16 * SIZE, X1 ALIGN_3 -.L115: +L(115): testq $4, MM - je .L117 + je L(117) movaps -32 * SIZE(A1), %xmm8 movsd -32 * SIZE(A2), %xmm9 @@ -2545,9 +2545,9 @@ addq $8 * SIZE, A2 ALIGN_3 -.L117: +L(117): testq $2, MM - je .L118 + je L(118) movaps -32 * SIZE(A1), %xmm8 movsd -32 * SIZE(A2), %xmm9 @@ -2571,9 +2571,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L118: +L(118): testq $1, MM - je .L119 + je L(119) #ifdef movsd xorps %xmm8, %xmm8 @@ -2597,7 +2597,7 @@ SUBPS %xmm5, %xmm3 ALIGN_3 -.L119: +L(119): pcmpeqb %xmm5, %xmm5 psllq $63, %xmm5 @@ -2673,13 +2673,13 @@ #if GEMV_UNROLL == 2 cmpq $2, N - jge .L111 + jge L(111) #endif ALIGN_3 -.L120: +L(120): cmpq $1, N - jl .L999 + jl L(999) leaq 32 * SIZE(BUFFER), X1 @@ -2690,7 +2690,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L12X + je L(12X) #ifdef movsd xorps %xmm8, %xmm8 @@ -2710,7 +2710,7 @@ addq $2 * SIZE, A1 addq $2 * SIZE, X1 ALIGN_3 -.L12X: +L(12X): #endif movaps -32 * SIZE(X1), %xmm12 @@ -2718,16 +2718,16 @@ movq MM, I sarq $3, I - jle .L125 + jle L(125) MOVUPS_A1(-32 * SIZE, A1, %xmm8) MOVUPS_A1(-28 * SIZE, A1, %xmm10) decq I - jle .L124 + jle L(124) ALIGN_3 -.L123: +L(123): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -2777,10 +2777,10 @@ subq $1, I BRANCH - jg .L123 + jg L(123) ALIGN_3 -.L124: +L(124): pshufd $0xb1, %xmm8, %xmm4 mulps %xmm12, %xmm8 addps %xmm8, %xmm0 @@ -2819,9 +2819,9 @@ subq $-16 * SIZE, X1 ALIGN_3 -.L125: +L(125): testq $4, MM - je .L127 + je L(127) MOVUPS_A1(-32 * SIZE, A1, %xmm8) MOVUPS_A1(-28 * SIZE, A1, %xmm10) @@ -2845,9 +2845,9 @@ addq $8 * SIZE, A1 ALIGN_3 -.L127: +L(127): testq $2, MM - je .L128 + je L(128) MOVUPS_A1(-32 * SIZE, A1, %xmm8) @@ -2862,9 +2862,9 @@ addq $4 * SIZE, A1 ALIGN_3 -.L128: +L(128): testq $1, MM - je .L129 + je L(129) #ifdef movsd xorps %xmm8, %xmm8 @@ -2878,7 +2878,7 @@ SUBPS %xmm4, %xmm1 ALIGN_3 -.L129: +L(129): pcmpeqb %xmm5, %xmm5 psllq $63, %xmm5 @@ -2936,19 +2936,19 @@ movlps %xmm0, (Y1) addq INCY, Y1 - jmp .L999 + jmp L(999) ALIGN_3 -.L200: +L(200): testq $2 * SIZE, LDA - jne .L300 + jne L(300) cmpq $2, N - jl .L210 + jl L(210) ALIGN_3 -.L201: +L(201): subq $2, N leaq 32 * SIZE(BUFFER), X1 @@ -2964,7 +2964,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L20X + je L(20X) #ifdef movsd xorps %xmm8, %xmm8 @@ -2996,7 +2996,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, X1 ALIGN_3 -.L20X: +L(20X): #endif movaps -33 * SIZE(A1), %xmm4 @@ -3011,16 +3011,16 @@ movq MM, I sarq $3, I - jle .L205 + jle L(205) movaps -29 * SIZE(A1), %xmm6 movaps -29 * SIZE(A2), %xmm7 decq I - jle .L204 + jle L(204) ALIGN_3 -.L203: +L(203): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -3115,10 +3115,10 @@ subq $1, I BRANCH - jg .L203 + jg L(203) ALIGN_3 -.L204: +L(204): movss %xmm6, %xmm4 shufps $0x39, %xmm4, %xmm4 pshufd $0xb1, %xmm4, %xmm14 @@ -3198,9 +3198,9 @@ subq $-16 * SIZE, X1 ALIGN_3 -.L205: +L(205): testq $4, MM - je .L207 + je L(207) movaps -29 * SIZE(A1), %xmm6 @@ -3252,9 +3252,9 @@ addq $8 * SIZE, A2 ALIGN_3 -.L207: +L(207): testq $2, MM - je .L208 + je L(208) movaps -29 * SIZE(A1), %xmm6 movaps -29 * SIZE(A2), %xmm7 @@ -3283,9 +3283,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L208: +L(208): testq $1, MM - je .L209 + je L(209) #ifdef movsd xorps %xmm8, %xmm8 @@ -3309,7 +3309,7 @@ SUBPS %xmm5, %xmm3 ALIGN_3 -.L209: +L(209): pcmpeqb %xmm5, %xmm5 psllq $63, %xmm5 @@ -3384,12 +3384,12 @@ addq INCY, Y1 cmpq $2, N - jge .L201 + jge L(201) ALIGN_3 -.L210: +L(210): cmpq $1, N - jl .L999 + jl L(999) leaq 32 * SIZE(BUFFER), X1 @@ -3400,7 +3400,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L21X + je L(21X) #ifdef movsd xorps %xmm8, %xmm8 @@ -3420,7 +3420,7 @@ addq $2 * SIZE, A1 addq $2 * SIZE, X1 ALIGN_3 -.L21X: +L(21X): #endif movaps -33 * SIZE(A1), %xmm4 @@ -3430,17 +3430,17 @@ movq MM, I sarq $3, I - jle .L215 + jle L(215) movaps -29 * SIZE(A1), %xmm5 movaps -25 * SIZE(A1), %xmm6 movaps -21 * SIZE(A1), %xmm7 decq I - jle .L214 + jle L(214) ALIGN_3 -.L213: +L(213): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -3494,10 +3494,10 @@ subq $1, I BRANCH - jg .L213 + jg L(213) ALIGN_3 -.L214: +L(214): movss %xmm5, %xmm4 shufps $0x39, %xmm4, %xmm4 pshufd $0xb1, %xmm4, %xmm14 @@ -3539,9 +3539,9 @@ subq $-16 * SIZE, X1 ALIGN_3 -.L215: +L(215): testq $4, MM - je .L217 + je L(217) movaps -29 * SIZE(A1), %xmm5 movaps -25 * SIZE(A1), %xmm6 @@ -3569,9 +3569,9 @@ addq $8 * SIZE, A1 ALIGN_3 -.L217: +L(217): testq $2, MM - je .L218 + je L(218) movaps -29 * SIZE(A1), %xmm5 @@ -3589,9 +3589,9 @@ addq $4 * SIZE, A1 ALIGN_3 -.L218: +L(218): testq $1, MM - je .L219 + je L(219) #ifdef movsd xorps %xmm8, %xmm8 @@ -3605,7 +3605,7 @@ SUBPS %xmm4, %xmm1 ALIGN_3 -.L219: +L(219): pcmpeqb %xmm5, %xmm5 psllq $63, %xmm5 @@ -3663,14 +3663,14 @@ movlps %xmm0, (Y1) addq INCY, Y1 - jmp .L999 + jmp L(999) -.L300: +L(300): cmpq $2, N - jl .L310 + jl L(310) ALIGN_3 -.L301: +L(301): subq $2, N leaq 32 * SIZE(BUFFER), X1 @@ -3686,7 +3686,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L30X + je L(30X) #ifdef movsd xorps %xmm8, %xmm8 @@ -3718,7 +3718,7 @@ addq $2 * SIZE, A2 addq $2 * SIZE, X1 ALIGN_3 -.L30X: +L(30X): #endif movaps -33 * SIZE(A1), %xmm4 @@ -3733,16 +3733,16 @@ movq MM, I sarq $3, I - jle .L305 + jle L(305) movaps -29 * SIZE(A1), %xmm6 movaps -31 * SIZE(A2), %xmm7 decq I - jle .L304 + jle L(304) ALIGN_3 -.L303: +L(303): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -3837,10 +3837,10 @@ subq $1, I BRANCH - jg .L303 + jg L(303) ALIGN_3 -.L304: +L(304): movss %xmm6, %xmm4 shufps $0x39, %xmm4, %xmm4 pshufd $0xb1, %xmm4, %xmm14 @@ -3920,9 +3920,9 @@ subq $-16 * SIZE, X1 ALIGN_3 -.L305: +L(305): testq $4, MM - je .L307 + je L(307) movaps -29 * SIZE(A1), %xmm6 @@ -3974,9 +3974,9 @@ addq $8 * SIZE, A2 ALIGN_3 -.L307: +L(307): testq $2, MM - je .L308 + je L(308) movaps -29 * SIZE(A1), %xmm6 movaps -31 * SIZE(A2), %xmm7 @@ -4005,9 +4005,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L308: +L(308): testq $1, MM - je .L309 + je L(309) #ifdef movsd xorps %xmm8, %xmm8 @@ -4031,7 +4031,7 @@ SUBPS %xmm5, %xmm3 ALIGN_3 -.L309: +L(309): pcmpeqb %xmm5, %xmm5 psllq $63, %xmm5 @@ -4106,12 +4106,12 @@ addq INCY, Y1 cmpq $2, N - jge .L301 + jge L(301) ALIGN_3 -.L310: +L(310): cmpq $1, N - jl .L999 + jl L(999) leaq 32 * SIZE(BUFFER), X1 @@ -4122,7 +4122,7 @@ #ifdef ALIGNED_ACCESS cmpq M, MM - je .L31X + je L(31X) #ifdef movsd xorps %xmm8, %xmm8 @@ -4142,7 +4142,7 @@ addq $2 * SIZE, A1 addq $2 * SIZE, X1 ALIGN_3 -.L31X: +L(31X): #endif movaps -33 * SIZE(A1), %xmm4 @@ -4152,17 +4152,17 @@ movq MM, I sarq $3, I - jle .L315 + jle L(315) movaps -29 * SIZE(A1), %xmm5 movaps -25 * SIZE(A1), %xmm6 movaps -21 * SIZE(A1), %xmm7 decq I - jle .L314 + jle L(314) ALIGN_3 -.L313: +L(313): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -4216,10 +4216,10 @@ subq $1, I BRANCH - jg .L313 + jg L(313) ALIGN_3 -.L314: +L(314): movss %xmm5, %xmm4 shufps $0x39, %xmm4, %xmm4 pshufd $0xb1, %xmm4, %xmm14 @@ -4261,9 +4261,9 @@ subq $-16 * SIZE, X1 ALIGN_3 -.L315: +L(315): testq $4, MM - je .L317 + je L(317) movaps -29 * SIZE(A1), %xmm5 movaps -25 * SIZE(A1), %xmm6 @@ -4291,9 +4291,9 @@ addq $8 * SIZE, A1 ALIGN_3 -.L317: +L(317): testq $2, MM - je .L318 + je L(318) movaps -29 * SIZE(A1), %xmm5 @@ -4311,9 +4311,9 @@ addq $4 * SIZE, A1 ALIGN_3 -.L318: +L(318): testq $1, MM - je .L319 + je L(319) #ifdef movsd xorps %xmm8, %xmm8 @@ -4327,7 +4327,7 @@ SUBPS %xmm4, %xmm1 ALIGN_3 -.L319: +L(319): pcmpeqb %xmm5, %xmm5 psllq $63, %xmm5 @@ -4388,12 +4388,12 @@ #endif ALIGN_3 -.L999: +L(999): movq M, I salq $ZBASE_SHIFT,I addq I,AA - jmp .L0t -.L999x: + jmp L(0t) +L(999x): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/copy.S b/kernel/x86_64/copy.S index 5729b29568..e28ba1ff23 100644 --- a/kernel/x86_64/copy.S +++ b/kernel/x86_64/copy.S @@ -63,22 +63,22 @@ EMMS testq N, N # if m == 0 goto End - jle .L999 + jle L(999) salq $BASE_SHIFT, INCX salq $BASE_SHIFT, INCY cmpq $SIZE, INCX # if incx != 1 - jne .L100 + jne L(100) cmpq $SIZE, INCY # if incy != 1 - jne .L100 + jne L(100) movq N, %rax # i = m sarq $3, %rax - jle .L20 + jle L(20) ALIGN_2 -.L11: +L(11): #ifdef XDOUBLE #ifdef PREFETCH @@ -194,16 +194,16 @@ addq $8 * SIZE, X addq $8 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_2 -.L20: +L(20): movq N, %rax andq $7, %rax - jle .L99 + jle L(99) ALIGN_2 -.L21: +L(21): #ifdef XDOUBLE movq 0(X), %mm0 movq 8(X), %mm1 @@ -217,21 +217,21 @@ addq $SIZE, X addq $SIZE, Y decq %rax - jg .L21 + jg L(21) -.L99: +L(99): xorq %rax,%rax EMMS ret ALIGN_3 -.L100: +L(100): movq N, %rax sarq $3, %rax - jle .L120 + jle L(120) ALIGN_2 -.L111: +L(111): #ifdef XDOUBLE movq 0(X), %mm0 movq 8(X), %mm1 @@ -333,15 +333,15 @@ #endif decq %rax - jg .L111 + jg L(111) -.L120: +L(120): movq N, %rax andq $7, %rax - jle .L999 + jle L(999) ALIGN_2 -.L121: +L(121): #ifdef XDOUBLE movq 0(X), %mm0 movq 8(X), %mm1 @@ -355,9 +355,9 @@ addq INCY, Y decq %rax - jg .L121 + jg L(121) -.L999: +L(999): xorq %rax,%rax EMMS ret diff --git a/kernel/x86_64/copy_sse.S b/kernel/x86_64/copy_sse.S index e949172931..dd96916da2 100644 --- a/kernel/x86_64/copy_sse.S +++ b/kernel/x86_64/copy_sse.S @@ -70,18 +70,18 @@ leaq (, INCY, SIZE), INCY cmpq $SIZE, INCX - jne .L50 + jne L(50) cmpq $SIZE, INCY - jne .L50 + jne L(50) cmpq $3, M - jle .L55 + jle L(55) subq $-32 * SIZE, X subq $-32 * SIZE, Y testq $SIZE, Y - je .L05 + je L(05) movss -32 * SIZE(X), %xmm0 movss %xmm0, -32 * SIZE(Y) @@ -90,25 +90,25 @@ decq M ALIGN_4 -.L05: +L(05): testq $2 * SIZE, Y - je .L10 + je L(10) movsd -32 * SIZE(X), %xmm0 movlps %xmm0, -32 * SIZE(Y) addq $2 * SIZE, X addq $2 * SIZE, Y subq $2, M - jle .L19 + jle L(19) ALIGN_4 -.L10: +L(10): testq $3 * SIZE, X - jne .L20 + jne L(20) movq M, %rax sarq $5, %rax - jle .L13 + jle L(13) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -120,10 +120,10 @@ movaps -4 * SIZE(X), %xmm7 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -163,10 +163,10 @@ subq $-32 * SIZE, Y subq $-32 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movaps %xmm0, -32 * SIZE(Y) movaps %xmm1, -28 * SIZE(Y) movaps %xmm2, -24 * SIZE(Y) @@ -180,9 +180,9 @@ subq $-32 * SIZE, X ALIGN_3 -.L13: +L(13): testq $16, M - jle .L14 + jle L(14) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -198,9 +198,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $8, M - jle .L15 + jle L(15) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -212,9 +212,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $4, M - jle .L16 + jle L(16) movaps -32 * SIZE(X), %xmm0 movaps %xmm0, -32 * SIZE(Y) @@ -223,9 +223,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $2, M - jle .L17 + jle L(17) movsd -32 * SIZE(X), %xmm0 movlps %xmm0, -32 * SIZE(Y) @@ -234,15 +234,15 @@ addq $2 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $1, M - jle .L19 + jle L(19) movss -32 * SIZE(X), %xmm0 movss %xmm0, -32 * SIZE(Y) ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -251,15 +251,15 @@ ALIGN_3 -.L20: +L(20): testq $SIZE, X - jne .L30 + jne L(30) movhps -32 * SIZE(X), %xmm0 movq M, %rax sarq $5, %rax - jle .L23 + jle L(23) movaps -30 * SIZE(X), %xmm1 movaps -26 * SIZE(X), %xmm2 @@ -270,10 +270,10 @@ movaps -6 * SIZE(X), %xmm7 decq %rax - jle .L22 + jle L(22) ALIGN_4 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) @@ -326,10 +326,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): shufps $0x4e, %xmm1, %xmm0 movaps %xmm0, -32 * SIZE(Y) movaps -2 * SIZE(X), %xmm0 @@ -359,9 +359,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L23: +L(23): testq $16, M - jle .L24 + jle L(24) ALIGN_3 movaps -30 * SIZE(X), %xmm1 @@ -384,9 +384,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $8, M - jle .L25 + jle L(25) ALIGN_3 movaps -30 * SIZE(X), %xmm1 @@ -403,9 +403,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $4, M - jle .L26 + jle L(26) ALIGN_3 movaps -30 * SIZE(X), %xmm1 @@ -416,9 +416,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $2, M - jle .L27 + jle L(27) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -429,9 +429,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $1, M - jle .L29 + jle L(29) ALIGN_3 movss -32 * SIZE(X), %xmm0 @@ -439,7 +439,7 @@ addq $SIZE, Y ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -447,15 +447,15 @@ ret ALIGN_3 -.L30: +L(30): testq $2 * SIZE, X - jne .L40 + jne L(40) movaps -33 * SIZE(X), %xmm0 movq M, %rax sarq $5, %rax - jle .L33 + jle L(33) movaps -29 * SIZE(X), %xmm1 movaps -25 * SIZE(X), %xmm2 @@ -466,10 +466,10 @@ movaps -5 * SIZE(X), %xmm7 decq %rax - jle .L32 + jle L(32) ALIGN_4 -.L31: +L(31): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -529,10 +529,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L32: +L(32): movss %xmm1, %xmm0 shufps $0x39, %xmm0, %xmm0 movaps %xmm0, -32 * SIZE(Y) @@ -570,9 +570,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L33: +L(33): testq $16, M - jle .L34 + jle L(34) ALIGN_3 movaps -29 * SIZE(X), %xmm1 @@ -602,9 +602,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L34: +L(34): testq $8, M - jle .L35 + jle L(35) ALIGN_3 movaps -29 * SIZE(X), %xmm1 @@ -623,9 +623,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L35: +L(35): testq $4, M - jle .L36 + jle L(36) ALIGN_3 movaps -29 * SIZE(X), %xmm1 @@ -639,9 +639,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L36: +L(36): testq $2, M - jle .L37 + jle L(37) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -651,9 +651,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L37: +L(37): testq $1, M - jle .L39 + jle L(39) ALIGN_3 movss -32 * SIZE(X), %xmm0 @@ -661,7 +661,7 @@ addq $SIZE, Y ALIGN_3 -.L39: +L(39): xorq %rax,%rax RESTOREREGISTERS @@ -669,12 +669,12 @@ ret ALIGN_3 -.L40: +L(40): movaps -35 * SIZE(X), %xmm0 movq M, %rax sarq $5, %rax - jle .L43 + jle L(43) movaps -31 * SIZE(X), %xmm1 movaps -27 * SIZE(X), %xmm2 @@ -685,10 +685,10 @@ movaps -7 * SIZE(X), %xmm7 decq %rax - jle .L42 + jle L(42) ALIGN_4 -.L41: +L(41): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -748,10 +748,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L42: +L(42): movss %xmm1, %xmm0 shufps $0x93, %xmm1, %xmm0 movaps %xmm0, -32 * SIZE(Y) @@ -789,9 +789,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L43: +L(43): testq $16, M - jle .L44 + jle L(44) ALIGN_3 movaps -31 * SIZE(X), %xmm1 @@ -821,9 +821,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L44: +L(44): testq $8, M - jle .L45 + jle L(45) ALIGN_3 movaps -31 * SIZE(X), %xmm1 @@ -843,9 +843,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L45: +L(45): testq $4, M - jle .L46 + jle L(46) ALIGN_3 movaps -31 * SIZE(X), %xmm1 @@ -859,9 +859,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L46: +L(46): testq $2, M - jle .L47 + jle L(47) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -871,9 +871,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L47: +L(47): testq $1, M - jle .L49 + jle L(49) ALIGN_3 movss -32 * SIZE(X), %xmm0 @@ -881,7 +881,7 @@ addq $SIZE, Y ALIGN_3 -.L49: +L(49): xorq %rax,%rax RESTOREREGISTERS @@ -889,13 +889,13 @@ ret ALIGN_4 -.L50: +L(50): movq M, %rax sarq $3, %rax - jle .L55 + jle L(55) ALIGN_3 -.L51: +L(51): movss (X), %xmm0 addq INCX, X movss (X), %xmm1 @@ -931,25 +931,25 @@ addq INCY, Y decq %rax - jg .L51 + jg L(51) ALIGN_3 -.L55: +L(55): movq M, %rax andq $7, %rax - jle .L57 + jle L(57) ALIGN_3 -.L56: +L(56): movss (X), %xmm0 addq INCX, X movss %xmm0, (Y) addq INCY, Y decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L57: +L(57): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/copy_sse2.S b/kernel/x86_64/copy_sse2.S index a5ab2ea91d..081c8850e9 100644 --- a/kernel/x86_64/copy_sse2.S +++ b/kernel/x86_64/copy_sse2.S @@ -70,26 +70,26 @@ leaq (, INCY, SIZE), INCY cmpq $SIZE, INCX - jne .L40 + jne L(40) cmpq $SIZE, INCY - jne .L40 + jne L(40) #ifdef ALIGNED_ACCESS testq $SIZE, Y #else testq $SIZE, X #endif - je .L10 + je L(10) movsd (X), %xmm0 movsd %xmm0, (Y) addq $1 * SIZE, X addq $1 * SIZE, Y decq M - jle .L19 + jle L(19) ALIGN_4 -.L10: +L(10): subq $-16 * SIZE, X subq $-16 * SIZE, Y @@ -98,11 +98,11 @@ #else testq $SIZE, Y #endif - jne .L20 + jne L(20) movq M, %rax sarq $4, %rax - jle .L13 + jle L(13) movups -16 * SIZE(X), %xmm0 movups -14 * SIZE(X), %xmm1 @@ -114,10 +114,10 @@ movups -2 * SIZE(X), %xmm7 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -157,10 +157,10 @@ subq $-16 * SIZE, Y subq $-16 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movups %xmm0, -16 * SIZE(Y) movups %xmm1, -14 * SIZE(Y) movups %xmm2, -12 * SIZE(Y) @@ -174,9 +174,9 @@ subq $-16 * SIZE, X ALIGN_3 -.L13: +L(13): testq $8, M - jle .L14 + jle L(14) ALIGN_3 movups -16 * SIZE(X), %xmm0 @@ -193,9 +193,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $4, M - jle .L15 + jle L(15) ALIGN_3 movups -16 * SIZE(X), %xmm0 @@ -208,9 +208,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $2, M - jle .L16 + jle L(16) ALIGN_3 movups -16 * SIZE(X), %xmm0 @@ -220,16 +220,16 @@ addq $2 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $1, M - jle .L19 + jle L(19) ALIGN_3 movsd -16 * SIZE(X), %xmm0 movsd %xmm0, -16 * SIZE(Y) ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -237,14 +237,14 @@ ret ALIGN_3 -.L20: +L(20): #ifdef ALIGNED_ACCESS movhps -16 * SIZE(X), %xmm0 movq M, %rax sarq $4, %rax - jle .L23 + jle L(23) movups -15 * SIZE(X), %xmm1 movups -13 * SIZE(X), %xmm2 @@ -255,10 +255,10 @@ movups -3 * SIZE(X), %xmm7 decq %rax - jle .L22 + jle L(22) ALIGN_4 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -310,10 +310,10 @@ subq $-16 * SIZE, X subq $-16 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): SHUFPD_1 %xmm1, %xmm0 movups %xmm0, -16 * SIZE(Y) LOAD(-1 * SIZE, X, %xmm0) @@ -340,9 +340,9 @@ subq $-16 * SIZE, Y ALIGN_3 -.L23: +L(23): testq $8, M - jle .L24 + jle L(24) ALIGN_3 movups -15 * SIZE(X), %xmm1 @@ -368,9 +368,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $4, M - jle .L25 + jle L(25) ALIGN_3 movups -15 * SIZE(X), %xmm1 @@ -387,9 +387,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $2, M - jle .L26 + jle L(26) ALIGN_3 movups -15 * SIZE(X), %xmm1 @@ -401,16 +401,16 @@ addq $2 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $1, M - jle .L29 + jle L(29) ALIGN_3 movsd -16 * SIZE(X), %xmm0 movsd %xmm0, -16 * SIZE(Y) ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -422,7 +422,7 @@ movq M, %rax sarq $4, %rax - jle .L23 + jle L(23) movups -16 * SIZE(X), %xmm0 movups -14 * SIZE(X), %xmm1 @@ -434,10 +434,10 @@ movups -2 * SIZE(X), %xmm7 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -485,10 +485,10 @@ subq $-16 * SIZE, Y subq $-16 * SIZE, X decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): movlps %xmm0, -16 * SIZE(Y) movhps %xmm0, -15 * SIZE(Y) movlps %xmm1, -14 * SIZE(Y) @@ -510,9 +510,9 @@ subq $-16 * SIZE, X ALIGN_3 -.L23: +L(23): testq $8, M - jle .L24 + jle L(24) ALIGN_3 movups -16 * SIZE(X), %xmm0 @@ -532,9 +532,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $4, M - jle .L25 + jle L(25) ALIGN_3 movups -16 * SIZE(X), %xmm0 @@ -548,9 +548,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $2, M - jle .L26 + jle L(26) ALIGN_3 movups -16 * SIZE(X), %xmm0 @@ -561,16 +561,16 @@ addq $2 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $1, M - jle .L29 + jle L(29) ALIGN_3 movsd -16 * SIZE(X), %xmm0 movsd %xmm0, -16 * SIZE(Y) ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -580,13 +580,13 @@ #endif -.L40: +L(40): movq M, %rax sarq $3, %rax - jle .L45 + jle L(45) ALIGN_3 -.L41: +L(41): movsd (X), %xmm0 addq INCX, X movhps (X), %xmm0 @@ -622,25 +622,25 @@ addq INCY, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L45: +L(45): movq M, %rax andq $7, %rax - jle .L47 + jle L(47) ALIGN_3 -.L46: +L(46): movsd (X), %xmm0 addq INCX, X movlps %xmm0, (Y) addq INCY, Y decq %rax - jg .L46 + jg L(46) ALIGN_3 -.L47: +L(47): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/daxpy_bulldozer.S b/kernel/x86_64/daxpy_bulldozer.S index 799dad02df..88dcc69830 100644 --- a/kernel/x86_64/daxpy_bulldozer.S +++ b/kernel/x86_64/daxpy_bulldozer.S @@ -87,15 +87,15 @@ leaq (, INCY, SIZE), INCY testq M, M - jle .L47 + jle L(47) cmpq $SIZE, INCX - jne .L40 + jne L(40) cmpq $SIZE, INCY - jne .L40 + jne L(40) testq $SIZE, Y - je .L10 + je L(10) movsd (X), %xmm0 mulsd ALPHA, %xmm0 @@ -104,16 +104,16 @@ addq $1 * SIZE, X addq $1 * SIZE, Y decq M - jle .L19 + jle L(19) ALIGN_4 -.L10: +L(10): subq $-16 * SIZE, X subq $-16 * SIZE, Y movq M, %rax sarq $4, %rax - jle .L13 + jle L(13) vmovups -16 * SIZE(X), %xmm0 vmovups -14 * SIZE(X), %xmm1 @@ -121,10 +121,10 @@ vmovups -10 * SIZE(X), %xmm3 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): prefetchnta A_PRE(Y) @@ -167,10 +167,10 @@ subq $-16 * SIZE, Y subq $-16 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): vmovups -8 * SIZE(X), %xmm4 vfmaddpd -16 * SIZE(Y), ALPHA, %xmm0 , %xmm0 @@ -201,12 +201,12 @@ subq $-16 * SIZE, X ALIGN_3 -.L13: +L(13): movq M, %rax andq $8, %rax - jle .L14 + jle L(14) ALIGN_3 vmovups -16 * SIZE(X), %xmm0 @@ -228,10 +228,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L14: +L(14): movq M, %rax andq $4, %rax - jle .L15 + jle L(15) ALIGN_3 vmovups -16 * SIZE(X), %xmm0 @@ -247,10 +247,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L15: +L(15): movq M, %rax andq $2, %rax - jle .L16 + jle L(16) ALIGN_3 vmovups -16 * SIZE(X), %xmm0 @@ -261,10 +261,10 @@ addq $2 * SIZE, Y ALIGN_3 -.L16: +L(16): movq M, %rax andq $1, %rax - jle .L19 + jle L(19) ALIGN_3 vmovsd -16 * SIZE(X), %xmm0 @@ -273,7 +273,7 @@ vmovsd %xmm0, -16 * SIZE(Y) ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -282,17 +282,17 @@ ALIGN_3 -.L40: +L(40): movq Y, YY movq M, %rax //If incx==0 || incy==0, avoid unloop. cmpq $0, INCX - je .L46 + je L(46) cmpq $0, INCY - je .L46 + je L(46) sarq $3, %rax - jle .L45 + jle L(45) prefetchnta 512(X) prefetchnta 512+64(X) @@ -305,7 +305,7 @@ prefetchnta 512+192(Y) ALIGN_3 -.L41: +L(41): vmovsd 0 * SIZE(X), %xmm0 addq INCX, X @@ -376,16 +376,16 @@ addq INCY, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L45: +L(45): movq M, %rax andq $7, %rax - jle .L47 + jle L(47) ALIGN_3 -.L46: +L(46): vmovsd (X), %xmm0 addq INCX, X @@ -395,10 +395,10 @@ addq INCY, Y decq %rax - jg .L46 + jg L(46) ALIGN_3 -.L47: +L(47): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/dcopy_bulldozer.S b/kernel/x86_64/dcopy_bulldozer.S index 87f1a4e31d..451f0f5800 100644 --- a/kernel/x86_64/dcopy_bulldozer.S +++ b/kernel/x86_64/dcopy_bulldozer.S @@ -69,29 +69,29 @@ leaq (, INCY, SIZE), INCY cmpq $SIZE, INCX - jne .L40 + jne L(40) cmpq $SIZE, INCY - jne .L40 + jne L(40) testq $SIZE, X - je .L10 + je L(10) vmovsd (X), %xmm0 vmovsd %xmm0, (Y) addq $1 * SIZE, X addq $1 * SIZE, Y decq M - jle .L19 + jle L(19) ALIGN_4 -.L10: +L(10): subq $-16 * SIZE, X subq $-16 * SIZE, Y movq M, %rax sarq $4, %rax - jle .L13 + jle L(13) vmovups -16 * SIZE(X), %xmm0 vmovups -14 * SIZE(X), %xmm1 @@ -103,10 +103,10 @@ vmovups -2 * SIZE(X), %xmm7 decq %rax - jle .L12 + jle L(12) ALIGN_4 -.L11: +L(11): prefetchnta A_PRE(X) nop @@ -139,10 +139,10 @@ subq $-16 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): vmovups %xmm0, -16 * SIZE(Y) vmovups %xmm1, -14 * SIZE(Y) vmovups %xmm2, -12 * SIZE(Y) @@ -156,9 +156,9 @@ subq $-16 * SIZE, X ALIGN_3 -.L13: +L(13): testq $8, M - jle .L14 + jle L(14) ALIGN_3 vmovups -16 * SIZE(X), %xmm0 @@ -175,9 +175,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $4, M - jle .L15 + jle L(15) ALIGN_3 vmovups -16 * SIZE(X), %xmm0 @@ -190,9 +190,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $2, M - jle .L16 + jle L(16) ALIGN_3 vmovups -16 * SIZE(X), %xmm0 @@ -202,16 +202,16 @@ addq $2 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $1, M - jle .L19 + jle L(19) ALIGN_3 vmovsd -16 * SIZE(X), %xmm0 vmovsd %xmm0, -16 * SIZE(Y) ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -221,13 +221,13 @@ -.L40: +L(40): movq M, %rax sarq $3, %rax - jle .L45 + jle L(45) ALIGN_3 -.L41: +L(41): vmovsd (X), %xmm0 addq INCX, X vmovsd (X), %xmm4 @@ -263,25 +263,25 @@ addq INCY, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L45: +L(45): movq M, %rax andq $7, %rax - jle .L47 + jle L(47) ALIGN_3 -.L46: +L(46): vmovsd (X), %xmm0 addq INCX, X vmovsd %xmm0, (Y) addq INCY, Y decq %rax - jg .L46 + jg L(46) ALIGN_3 -.L47: +L(47): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/ddot_bulldozer.S b/kernel/x86_64/ddot_bulldozer.S index 61c7571166..7a2d70885b 100644 --- a/kernel/x86_64/ddot_bulldozer.S +++ b/kernel/x86_64/ddot_bulldozer.S @@ -71,18 +71,18 @@ vxorps %xmm3, %xmm3 , %xmm3 cmpq $0, N - jle .L999 + jle L(999) cmpq $SIZE, INCX - jne .L50 + jne L(50) cmpq $SIZE, INCY - jne .L50 + jne L(50) subq $-16 * SIZE, X subq $-16 * SIZE, Y testq $SIZE, Y - je .L10 + je L(10) vmovsd -16 * SIZE(X), %xmm0 vmulsd -16 * SIZE(Y), %xmm0 , %xmm0 @@ -91,11 +91,11 @@ decq N ALIGN_2 -.L10: +L(10): movq N, %rax sarq $4, %rax - jle .L14 + jle L(14) vmovups -16 * SIZE(X), %xmm4 vmovups -14 * SIZE(X), %xmm5 @@ -108,11 +108,11 @@ vmovups -2 * SIZE(X), %xmm11 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): prefetchnta A_PRE(Y) vfmaddpd %xmm0 , -16 * SIZE(Y), %xmm4 , %xmm0 @@ -142,10 +142,10 @@ subq $-16 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): vfmaddpd %xmm0 , -16 * SIZE(Y), %xmm4 , %xmm0 vfmaddpd %xmm1 , -14 * SIZE(Y), %xmm5 , %xmm1 @@ -161,12 +161,12 @@ subq $-16 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $15, N - jle .L999 + jle L(999) testq $8, N - jle .L15 + jle L(15) vmovups -16 * SIZE(X), %xmm4 vmovups -14 * SIZE(X), %xmm5 @@ -182,9 +182,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $4, N - jle .L16 + jle L(16) vmovups -16 * SIZE(X), %xmm4 vmovups -14 * SIZE(X), %xmm5 @@ -196,9 +196,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $2, N - jle .L17 + jle L(17) vmovups -16 * SIZE(X), %xmm4 vfmaddpd %xmm0 , -16 * SIZE(Y), %xmm4 , %xmm0 @@ -208,24 +208,24 @@ addq $2 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $1, N - jle .L999 + jle L(999) vmovsd -16 * SIZE(X), %xmm4 vmovsd -16 * SIZE(Y), %xmm5 vfmaddpd %xmm0, %xmm4 , %xmm5 , %xmm0 - jmp .L999 + jmp L(999) ALIGN_3 -.L50: +L(50): movq N, %rax sarq $3, %rax - jle .L55 + jle L(55) ALIGN_3 -.L53: +L(53): vmovsd 0 * SIZE(X), %xmm4 @@ -276,16 +276,16 @@ vfmaddpd %xmm3 , %xmm7 , %xmm11, %xmm3 decq %rax - jg .L53 + jg L(53) ALIGN_3 -.L55: +L(55): movq N, %rax andq $7, %rax - jle .L999 + jle L(999) ALIGN_3 -.L56: +L(56): vmovsd 0 * SIZE(X), %xmm4 addq INCX, X vmovsd 0 * SIZE(Y), %xmm8 @@ -294,10 +294,10 @@ vfmaddpd %xmm0 , %xmm4 , %xmm8 , %xmm0 decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L999: +L(999): vaddpd %xmm1, %xmm0 , %xmm0 vaddpd %xmm3, %xmm2 , %xmm2 vaddpd %xmm2, %xmm0 , %xmm0 diff --git a/kernel/x86_64/dgemm_kernel_16x2_haswell.S b/kernel/x86_64/dgemm_kernel_16x2_haswell.S index 899c5f2419..67c19e0556 100644 --- a/kernel/x86_64/dgemm_kernel_16x2_haswell.S +++ b/kernel/x86_64/dgemm_kernel_16x2_haswell.S @@ -1685,13 +1685,13 @@ STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -1711,10 +1711,10 @@ movq Ndiv6, J cmpq $0, J - je .L2_0 + je L(2_0) ALIGN_4 -.L6_01: +L(6_01): // copy to sub buffer movq K, %rax salq $1,%rax // K * 2 ; read 2 values @@ -1723,10 +1723,10 @@ leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $3 , %rax // K / 8 - jz .L6_01a_2 + jz L(6_01a_2) ALIGN_4 -.L6_01a_1: +L(6_01a_1): prefetcht0 512(BO1) prefetcht0 512(BO2) @@ -1774,19 +1774,19 @@ addq $ 12*SIZE,BO decq %rax - jnz .L6_01a_1 + jnz L(6_01a_1) -.L6_01a_2: +L(6_01a_2): movq K, %rax andq $7, %rax // K % 8 - jz .L6_02c + jz L(6_02c) ALIGN_4 -.L6_02b: +L(6_02b): vmovups 0 * SIZE(BO1), %xmm0 vmovsd 0 * SIZE(BO2), %xmm2 @@ -1796,9 +1796,9 @@ addq $ 2*SIZE,BO2 addq $ 3*SIZE,BO decq %rax - jnz .L6_02b + jnz L(6_02b) -.L6_02c: +L(6_02c): movq K, %rax salq $1,%rax // K * 2 @@ -1807,10 +1807,10 @@ leaq BUFFER2, BO // second buffer to BO movq K, %rax sarq $3 , %rax // K / 8 - jz .L6_02c_2 + jz L(6_02c_2) ALIGN_4 -.L6_02c_1: +L(6_02c_1): prefetcht0 512(BO2) prefetchw 512(BO) @@ -1857,17 +1857,17 @@ addq $12*SIZE,BO decq %rax - jnz .L6_02c_1 + jnz L(6_02c_1) -.L6_02c_2: +L(6_02c_2): movq K, %rax andq $7, %rax // K % 8 - jz .L6_03c + jz L(6_03c) ALIGN_4 -.L6_03b: +L(6_03b): vmovsd 1*SIZE(BO1), %xmm0 vmovups 0*SIZE(BO2), %xmm1 @@ -1877,14 +1877,14 @@ addq $2*SIZE,BO2 addq $3*SIZE,BO decq %rax - jnz .L6_03b + jnz L(6_03b) -.L6_03c: +L(6_03c): movq BO2, B // next offset of B -.L6_10: +L(6_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -1895,11 +1895,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L6_20 + je L(6_20) ALIGN_4 -.L6_11: +L(6_11): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -1915,11 +1915,11 @@ movq K, %rax sarq $1, %rax // K / 8 - je .L6_16 + je L(6_16) ALIGN_5 -.L6_12: +L(6_12): /* prefetcht0 B_PR1(BO) prefetcht0 B_PR1+64(BO) @@ -1937,50 +1937,50 @@ KERNEL16x3_SUBN */ dec %rax - jne .L6_12 + jne L(6_12) -.L6_16: +L(6_16): movq K, %rax andq $1, %rax # if (k & 1) - je .L6_19 + je L(6_19) ALIGN_4 -.L6_17: +L(6_17): KERNEL16x3_SUBN dec %rax - jne .L6_17 + jne L(6_17) ALIGN_4 -.L6_19: +L(6_19): SAVE16x3 addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L6_11 + jg L(6_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L6_20: +L(6_20): // Test rest of M testq $15, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N testq $8, M - jz .L6_21pre + jz L(6_21pre) ALIGN_4 /**************************************************************************/ -.L6_20_1: +L(6_20_1): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -1989,11 +1989,11 @@ movq K, %rax sarq $3, %rax - je .L6_20_6 + je L(6_20_6) ALIGN_4 -.L6_20_2: +L(6_20_2): KERNEL8x3_SUBN KERNEL8x3_SUBN @@ -2005,28 +2005,28 @@ KERNEL8x3_SUBN KERNEL8x3_SUBN dec %rax - jne .L6_20_2 + jne L(6_20_2) ALIGN_4 -.L6_20_6: +L(6_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_20_9 + je L(6_20_9) ALIGN_4 -.L6_20_7: +L(6_20_7): KERNEL8x3_SUBN dec %rax - jne .L6_20_7 + jne L(6_20_7) ALIGN_4 -.L6_20_9: +L(6_20_9): SAVE8x3 @@ -2037,13 +2037,13 @@ /**************************************************************************/ -.L6_21pre: +L(6_21pre): testq $4, M - jz .L6_30 + jz L(6_30) ALIGN_4 -.L6_21: +L(6_21): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -2052,11 +2052,11 @@ movq K, %rax sarq $3, %rax - je .L6_26 + je L(6_26) ALIGN_4 -.L6_22: +L(6_22): KERNEL4x3_SUBN KERNEL4x3_SUBN @@ -2068,27 +2068,27 @@ KERNEL4x3_SUBN KERNEL4x3_SUBN dec %rax - jne .L6_22 + jne L(6_22) ALIGN_4 -.L6_26: +L(6_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_29 + je L(6_29) ALIGN_4 -.L6_27: +L(6_27): KERNEL4x3_SUBN dec %rax - jne .L6_27 + jne L(6_27) ALIGN_4 -.L6_29: +L(6_29): SAVE4x3 @@ -2096,13 +2096,13 @@ ALIGN_4 -.L6_30: +L(6_30): testq $2, M - jz .L6_40 + jz L(6_40) ALIGN_4 -.L6_31: +L(6_31): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -2111,10 +2111,10 @@ movq K, %rax sarq $3, %rax - je .L6_36 + je L(6_36) ALIGN_4 -.L6_32: +L(6_32): KERNEL2x3_SUBN KERNEL2x3_SUBN @@ -2126,40 +2126,40 @@ KERNEL2x3_SUBN KERNEL2x3_SUBN dec %rax - jne .L6_32 + jne L(6_32) ALIGN_4 -.L6_36: +L(6_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_39 + je L(6_39) ALIGN_4 -.L6_37: +L(6_37): KERNEL2x3_SUBN dec %rax - jne .L6_37 + jne L(6_37) ALIGN_4 -.L6_39: +L(6_39): SAVE2x3 addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L6_40: +L(6_40): testq $1, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N ALIGN_4 -.L6_41: +L(6_41): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -2168,11 +2168,11 @@ movq K, %rax sarq $3,%rax - je .L6_46 + je L(6_46) ALIGN_4 -.L6_42: +L(6_42): KERNEL1x3_SUBN KERNEL1x3_SUBN @@ -2185,27 +2185,27 @@ KERNEL1x3_SUBN dec %rax - jne .L6_42 + jne L(6_42) ALIGN_4 -.L6_46: +L(6_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_49 + je L(6_49) ALIGN_4 -.L6_47: +L(6_47): KERNEL1x3_SUBN dec %rax - jne .L6_47 + jne L(6_47) ALIGN_4 -.L6_49: +L(6_49): SAVE1x3 @@ -2217,7 +2217,7 @@ /***************************************************************************************************************/ -.L7_10: +L(7_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -2228,11 +2228,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L7_20 + je L(7_20) ALIGN_4 -.L7_11: +L(7_11): leaq BUFFER2, BO // second buffer to BO addq $12 * SIZE, BO @@ -2248,10 +2248,10 @@ movq K, %rax sarq $3, %rax // K / 8 - je .L7_16 + je L(7_16) ALIGN_5 -.L7_12: +L(7_12): /* prefetcht0 B_PR1(BO) prefetcht0 B_PR1+64(BO) @@ -2267,50 +2267,50 @@ KERNEL16x3_SUBN KERNEL16x3_SUBN dec %rax - jne .L7_12 + jne L(7_12) ALIGN_4 -.L7_16: +L(7_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_19 + je L(7_19) ALIGN_5 -.L7_17: +L(7_17): KERNEL16x3_SUBN dec %rax - jne .L7_17 + jne L(7_17) -.L7_19: +L(7_19): SAVE16x3 addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L7_11 + jg L(7_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L7_20: +L(7_20): // Test rest of M testq $15, M - jz .L7_60 // to next 3 lines of N + jz L(7_60) // to next 3 lines of N testq $8, M - jz .L7_21pre + jz L(7_21pre) ALIGN_4 /**************************************************************************/ -.L7_20_1: +L(7_20_1): leaq BUFFER2, BO // first buffer to BO addq $12 * SIZE, BO @@ -2319,11 +2319,11 @@ movq K, %rax sarq $3, %rax - je .L7_20_6 + je L(7_20_6) ALIGN_4 -.L7_20_2: +L(7_20_2): KERNEL8x3_SUBN KERNEL8x3_SUBN @@ -2336,26 +2336,26 @@ KERNEL8x3_SUBN dec %rax - jne .L7_20_2 + jne L(7_20_2) ALIGN_4 -.L7_20_6: +L(7_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_20_9 + je L(7_20_9) ALIGN_4 -.L7_20_7: +L(7_20_7): KERNEL8x3_SUBN dec %rax - jne .L7_20_7 + jne L(7_20_7) ALIGN_4 -.L7_20_9: +L(7_20_9): SAVE8x3 @@ -2366,13 +2366,13 @@ /**************************************************************************/ -.L7_21pre: +L(7_21pre): testq $4, M - jz .L7_30 + jz L(7_30) ALIGN_4 -.L7_21: +L(7_21): leaq BUFFER2, BO // second buffer to BO addq $12 * SIZE, BO @@ -2381,11 +2381,11 @@ movq K, %rax sarq $3, %rax - je .L7_26 + je L(7_26) ALIGN_4 -.L7_22: +L(7_22): KERNEL4x3_SUBN KERNEL4x3_SUBN @@ -2398,27 +2398,27 @@ KERNEL4x3_SUBN dec %rax - jne .L7_22 + jne L(7_22) ALIGN_4 -.L7_26: +L(7_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_29 + je L(7_29) ALIGN_4 -.L7_27: +L(7_27): KERNEL4x3_SUBN dec %rax - jne .L7_27 + jne L(7_27) ALIGN_4 -.L7_29: +L(7_29): SAVE4x3 @@ -2426,13 +2426,13 @@ ALIGN_4 -.L7_30: +L(7_30): testq $2, M - jz .L7_40 + jz L(7_40) ALIGN_4 -.L7_31: +L(7_31): leaq BUFFER2, BO // second buffer to BO addq $12 * SIZE, BO @@ -2441,11 +2441,11 @@ movq K, %rax sarq $3, %rax - je .L7_36 + je L(7_36) ALIGN_4 -.L7_32: +L(7_32): KERNEL2x3_SUBN KERNEL2x3_SUBN @@ -2458,40 +2458,40 @@ KERNEL2x3_SUBN dec %rax - jne .L7_32 + jne L(7_32) ALIGN_4 -.L7_36: +L(7_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_39 + je L(7_39) ALIGN_4 -.L7_37: +L(7_37): KERNEL2x3_SUBN dec %rax - jne .L7_37 + jne L(7_37) ALIGN_4 -.L7_39: +L(7_39): SAVE2x3 addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L7_40: +L(7_40): testq $1, M - jz .L7_60 // to next 3 lines of N + jz L(7_60) // to next 3 lines of N ALIGN_4 -.L7_41: +L(7_41): leaq BUFFER2, BO // second buffer to BO addq $12 * SIZE, BO @@ -2500,11 +2500,11 @@ movq K, %rax sarq $3, %rax - je .L7_46 + je L(7_46) ALIGN_4 -.L7_42: +L(7_42): KERNEL1x3_SUBN KERNEL1x3_SUBN KERNEL1x3_SUBN @@ -2516,27 +2516,27 @@ KERNEL1x3_SUBN dec %rax - jne .L7_42 + jne L(7_42) ALIGN_4 -.L7_46: +L(7_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_49 + je L(7_49) ALIGN_4 -.L7_47: +L(7_47): KERNEL1x3_SUBN dec %rax - jne .L7_47 + jne L(7_47) ALIGN_4 -.L7_49: +L(7_49): SAVE1x3 @@ -2545,15 +2545,15 @@ -.L7_60: +L(7_60): decq J // j -- - jg .L6_01 + jg L(6_01) -.L2_0: +L(2_0): cmpq $0, Nmod6 // N % 6 == 0 - je .L999 + je L(999) /************************************************************************************************ * Loop for Nmod6 / 2 > 0 @@ -2561,19 +2561,19 @@ movq Nmod6, J sarq $1, J // j = j / 2 - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L2_01b + jz L(2_01b) ALIGN_4 -.L2_01a: +L(2_01a): prefetcht0 512(BO1) prefetchw 512(BO) @@ -2590,30 +2590,30 @@ addq $8*SIZE,BO1 addq $8*SIZE,BO decq %rax - jnz .L2_01a + jnz L(2_01a) -.L2_01b: +L(2_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L2_02d + jz L(2_02d) ALIGN_4 -.L2_02c: +L(2_02c): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02c + jnz L(2_02c) -.L2_02d: +L(2_02d): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -2623,11 +2623,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2636,7 +2636,7 @@ movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2647,7 +2647,7 @@ negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI,8) KERNEL16x2_1 @@ -2661,7 +2661,7 @@ KERNEL16x2_3 KERNEL16x2_4 - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI,8) KERNEL16x2_1 @@ -2675,16 +2675,16 @@ KERNEL16x2_3 KERNEL16x2_4 - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2696,39 +2696,39 @@ negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE16x2 addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 3 lines of N + jz L(2_60) // to next 3 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2737,7 +2737,7 @@ movq K, %rax andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2748,7 +2748,7 @@ negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1 @@ -2762,7 +2762,7 @@ KERNEL8x2_3 KERNEL8x2_4 - je .L2_20_6 + je L(2_20_6) prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1 @@ -2776,16 +2776,16 @@ KERNEL8x2_3 KERNEL8x2_4 - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2797,15 +2797,15 @@ negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): SAVE8x2 @@ -2816,13 +2816,13 @@ /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2831,7 +2831,7 @@ movq K, %rax andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -2842,7 +2842,7 @@ negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1 @@ -2856,7 +2856,7 @@ KERNEL4x2_3 KERNEL4x2_4 - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1 @@ -2870,16 +2870,16 @@ KERNEL4x2_3 KERNEL4x2_4 - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2891,15 +2891,15 @@ negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): SAVE4x2 @@ -2907,13 +2907,13 @@ ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2922,7 +2922,7 @@ movq K, %rax andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2933,7 +2933,7 @@ negq %rax ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_1 KERNEL2x2_2 @@ -2945,7 +2945,7 @@ KERNEL2x2_3 KERNEL2x2_4 - je .L2_36 + je L(2_36) KERNEL2x2_1 KERNEL2x2_2 @@ -2957,16 +2957,16 @@ KERNEL2x2_3 KERNEL2x2_4 - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2978,28 +2978,28 @@ negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): SAVE2x2 addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -3008,7 +3008,7 @@ movq K, %rax andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3018,7 +3018,7 @@ negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_1 KERNEL1x2_2 @@ -3030,7 +3030,7 @@ KERNEL1x2_3 KERNEL1x2_4 - je .L2_46 + je L(2_46) KERNEL1x2_1 KERNEL1x2_2 @@ -3042,16 +3042,16 @@ KERNEL1x2_3 KERNEL1x2_4 - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3062,29 +3062,29 @@ negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): SAVE1x2 addq $1 * SIZE, CO1 # coffset += 1 ALIGN_4 -.L2_60: +L(2_60): decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -3092,30 +3092,30 @@ movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -3125,11 +3125,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3138,7 +3138,7 @@ movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -3148,7 +3148,7 @@ negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI,8) KERNEL16x1_1 @@ -3161,7 +3161,7 @@ KERNEL16x1_3 KERNEL16x1_4 - je .L1_16 + je L(1_16) prefetcht0 B_PR1(BO,BI,8) KERNEL16x1_1 @@ -3174,16 +3174,16 @@ KERNEL16x1_3 KERNEL16x1_4 - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -3194,39 +3194,39 @@ negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE16x1 addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3235,7 +3235,7 @@ movq K, %rax andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -3245,7 +3245,7 @@ negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1 @@ -3258,7 +3258,7 @@ KERNEL8x1_3 KERNEL8x1_4 - je .L1_20_6 + je L(1_20_6) prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1 @@ -3271,16 +3271,16 @@ KERNEL8x1_3 KERNEL8x1_4 - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -3291,15 +3291,15 @@ negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): SAVE8x1 @@ -3310,13 +3310,13 @@ /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3325,7 +3325,7 @@ movq K, %rax andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -3335,7 +3335,7 @@ negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1 @@ -3348,7 +3348,7 @@ KERNEL4x1_3 KERNEL4x1_4 - je .L1_26 + je L(1_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1 @@ -3361,16 +3361,16 @@ KERNEL4x1_3 KERNEL4x1_4 - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -3381,15 +3381,15 @@ negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): SAVE4x1 @@ -3397,13 +3397,13 @@ ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3412,7 +3412,7 @@ movq K, %rax andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -3422,7 +3422,7 @@ negq %rax ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_1 KERNEL2x1_2 @@ -3434,7 +3434,7 @@ KERNEL2x1_3 KERNEL2x1_4 - je .L1_36 + je L(1_36) KERNEL2x1_1 KERNEL2x1_2 @@ -3446,16 +3446,16 @@ KERNEL2x1_3 KERNEL2x1_4 - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -3466,28 +3466,28 @@ negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): SAVE2x1 addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3496,7 +3496,7 @@ movq K, %rax andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -3505,7 +3505,7 @@ negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_1 KERNEL1x1_2 @@ -3517,7 +3517,7 @@ KERNEL1x1_3 KERNEL1x1_4 - je .L1_46 + je L(1_46) KERNEL1x1_1 KERNEL1x1_2 @@ -3529,16 +3529,16 @@ KERNEL1x1_3 KERNEL1x1_4 - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -3548,15 +3548,15 @@ negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): SAVE1x1 @@ -3564,7 +3564,7 @@ ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp @@ -3654,13 +3654,13 @@ STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -3689,19 +3689,19 @@ movq Ndiv6, J cmpq $0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L2_01b + jz L(2_01b) ALIGN_4 -.L2_01a: +L(2_01a): prefetcht0 512(BO1) prefetchw 512(BO) @@ -3718,30 +3718,30 @@ addq $8*SIZE,BO1 addq $8*SIZE,BO decq %rax - jnz .L2_01a + jnz L(2_01a) -.L2_01b: +L(2_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L2_02d + jz L(2_02d) ALIGN_4 -.L2_02c: +L(2_02c): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02c + jnz L(2_02c) -.L2_02d: +L(2_02d): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -3755,11 +3755,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3796,7 +3796,7 @@ #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3807,7 +3807,7 @@ negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI,8) KERNEL16x2_1 @@ -3821,7 +3821,7 @@ KERNEL16x2_3 KERNEL16x2_4 - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI,8) KERNEL16x2_1 @@ -3835,12 +3835,12 @@ KERNEL16x2_3 KERNEL16x2_4 - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3848,7 +3848,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3860,15 +3860,15 @@ negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE16x2 @@ -3890,25 +3890,25 @@ addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 3 lines of N + jz L(2_60) // to next 3 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3946,7 +3946,7 @@ andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3957,7 +3957,7 @@ negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1 @@ -3971,7 +3971,7 @@ KERNEL8x2_3 KERNEL8x2_4 - je .L2_20_6 + je L(2_20_6) prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1 @@ -3985,12 +3985,12 @@ KERNEL8x2_3 KERNEL8x2_4 - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -3998,7 +3998,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4010,15 +4010,15 @@ negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): SAVE8x2 @@ -4045,13 +4045,13 @@ /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4089,7 +4089,7 @@ andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -4100,7 +4100,7 @@ negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1 @@ -4114,7 +4114,7 @@ KERNEL4x2_3 KERNEL4x2_4 - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1 @@ -4128,12 +4128,12 @@ KERNEL4x2_3 KERNEL4x2_4 - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4141,7 +4141,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4153,15 +4153,15 @@ negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): SAVE4x2 @@ -4185,13 +4185,13 @@ ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4229,7 +4229,7 @@ andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4240,7 +4240,7 @@ negq %rax ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_1 KERNEL2x2_2 @@ -4252,7 +4252,7 @@ KERNEL2x2_3 KERNEL2x2_4 - je .L2_36 + je L(2_36) KERNEL2x2_1 KERNEL2x2_2 @@ -4264,12 +4264,12 @@ KERNEL2x2_3 KERNEL2x2_4 - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -4277,7 +4277,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4289,15 +4289,15 @@ negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): SAVE2x2 @@ -4320,13 +4320,13 @@ addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4362,7 +4362,7 @@ #endif andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4372,7 +4372,7 @@ negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_1 KERNEL1x2_2 @@ -4384,7 +4384,7 @@ KERNEL1x2_3 KERNEL1x2_4 - je .L2_46 + je L(2_46) KERNEL1x2_1 KERNEL1x2_2 @@ -4396,12 +4396,12 @@ KERNEL1x2_3 KERNEL1x2_4 - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -4409,7 +4409,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4420,15 +4420,15 @@ negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): SAVE1x2 @@ -4454,17 +4454,17 @@ -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -4472,30 +4472,30 @@ movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -4509,11 +4509,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4549,7 +4549,7 @@ #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -4559,7 +4559,7 @@ negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI,8) KERNEL16x1_1 @@ -4572,7 +4572,7 @@ KERNEL16x1_3 KERNEL16x1_4 - je .L1_16 + je L(1_16) prefetcht0 B_PR1(BO,BI,8) KERNEL16x1_1 @@ -4585,12 +4585,12 @@ KERNEL16x1_3 KERNEL16x1_4 - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -4598,7 +4598,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -4609,15 +4609,15 @@ negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE16x1 @@ -4638,25 +4638,25 @@ addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4693,7 +4693,7 @@ andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -4703,7 +4703,7 @@ negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1 @@ -4716,7 +4716,7 @@ KERNEL8x1_3 KERNEL8x1_4 - je .L1_20_6 + je L(1_20_6) prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1 @@ -4729,12 +4729,12 @@ KERNEL8x1_3 KERNEL8x1_4 - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -4742,7 +4742,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -4753,15 +4753,15 @@ negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): SAVE8x1 @@ -4787,13 +4787,13 @@ /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4830,7 +4830,7 @@ andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -4840,7 +4840,7 @@ negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1 @@ -4853,7 +4853,7 @@ KERNEL4x1_3 KERNEL4x1_4 - je .L1_26 + je L(1_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1 @@ -4866,12 +4866,12 @@ KERNEL4x1_3 KERNEL4x1_4 - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4879,7 +4879,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -4890,15 +4890,15 @@ negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): SAVE4x1 @@ -4921,13 +4921,13 @@ ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4964,7 +4964,7 @@ andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -4974,7 +4974,7 @@ negq %rax ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_1 KERNEL2x1_2 @@ -4986,7 +4986,7 @@ KERNEL2x1_3 KERNEL2x1_4 - je .L1_36 + je L(1_36) KERNEL2x1_1 KERNEL2x1_2 @@ -4998,12 +4998,12 @@ KERNEL2x1_3 KERNEL2x1_4 - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -5011,7 +5011,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -5022,15 +5022,15 @@ negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): SAVE2x1 @@ -5052,13 +5052,13 @@ addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5093,7 +5093,7 @@ #endif andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -5102,7 +5102,7 @@ negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_1 KERNEL1x1_2 @@ -5114,7 +5114,7 @@ KERNEL1x1_3 KERNEL1x1_4 - je .L1_46 + je L(1_46) KERNEL1x1_1 KERNEL1x1_2 @@ -5126,12 +5126,12 @@ KERNEL1x1_3 KERNEL1x1_4 - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -5139,7 +5139,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -5149,15 +5149,15 @@ negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): SAVE1x1 @@ -5179,7 +5179,7 @@ ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/dgemm_kernel_16x2_skylakex.S b/kernel/x86_64/dgemm_kernel_16x2_skylakex.S index 91ac512805..02e04240d3 100644 --- a/kernel/x86_64/dgemm_kernel_16x2_skylakex.S +++ b/kernel/x86_64/dgemm_kernel_16x2_skylakex.S @@ -1608,13 +1608,13 @@ STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -1634,10 +1634,10 @@ movq Ndiv6, J cmpq $0, J - je .L2_0 + je L(2_0) ALIGN_4 -.L6_01: +L(6_01): // copy to sub buffer movq K, %rax salq $1,%rax // K * 2 ; read 2 values @@ -1646,10 +1646,10 @@ leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $3 , %rax // K / 8 - jz .L6_01a_2 + jz L(6_01a_2) ALIGN_4 -.L6_01a_1: +L(6_01a_1): prefetcht0 512(BO1) prefetcht0 512(BO2) @@ -1697,19 +1697,19 @@ addq $ 12*SIZE,BO decq %rax - jnz .L6_01a_1 + jnz L(6_01a_1) -.L6_01a_2: +L(6_01a_2): movq K, %rax andq $7, %rax // K % 8 - jz .L6_02c + jz L(6_02c) ALIGN_4 -.L6_02b: +L(6_02b): vmovups 0 * SIZE(BO1), %xmm0 vmovsd 0 * SIZE(BO2), %xmm2 @@ -1719,9 +1719,9 @@ addq $ 2*SIZE,BO2 addq $ 3*SIZE,BO decq %rax - jnz .L6_02b + jnz L(6_02b) -.L6_02c: +L(6_02c): movq K, %rax salq $1,%rax // K * 2 @@ -1730,10 +1730,10 @@ leaq BUFFER2, BO // second buffer to BO movq K, %rax sarq $3 , %rax // K / 8 - jz .L6_02c_2 + jz L(6_02c_2) ALIGN_4 -.L6_02c_1: +L(6_02c_1): prefetcht0 512(BO2) prefetchw 512(BO) @@ -1780,17 +1780,17 @@ addq $12*SIZE,BO decq %rax - jnz .L6_02c_1 + jnz L(6_02c_1) -.L6_02c_2: +L(6_02c_2): movq K, %rax andq $7, %rax // K % 8 - jz .L6_03c + jz L(6_03c) ALIGN_4 -.L6_03b: +L(6_03b): vmovsd 1*SIZE(BO1), %xmm0 vmovups 0*SIZE(BO2), %xmm1 @@ -1800,14 +1800,14 @@ addq $2*SIZE,BO2 addq $3*SIZE,BO decq %rax - jnz .L6_03b + jnz L(6_03b) -.L6_03c: +L(6_03c): movq BO2, B // next offset of B -.L6_10: +L(6_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -1818,11 +1818,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L6_20 + je L(6_20) ALIGN_4 -.L6_11: +L(6_11): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -1838,11 +1838,11 @@ movq K, %rax sarq $1, %rax // K / 8 - je .L6_16 + je L(6_16) ALIGN_5 -.L6_12: +L(6_12): /* prefetcht0 B_PR1(BO) prefetcht0 B_PR1+64(BO) @@ -1860,50 +1860,50 @@ KERNEL16x3_SUBN */ dec %rax - jne .L6_12 + jne L(6_12) -.L6_16: +L(6_16): movq K, %rax andq $1, %rax # if (k & 1) - je .L6_19 + je L(6_19) ALIGN_4 -.L6_17: +L(6_17): KERNEL16x3_SUBN dec %rax - jne .L6_17 + jne L(6_17) ALIGN_4 -.L6_19: +L(6_19): SAVE16x3 addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L6_11 + jg L(6_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L6_20: +L(6_20): // Test rest of M testq $15, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N testq $8, M - jz .L6_21pre + jz L(6_21pre) ALIGN_4 /**************************************************************************/ -.L6_20_1: +L(6_20_1): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -1912,11 +1912,11 @@ movq K, %rax sarq $3, %rax - je .L6_20_6 + je L(6_20_6) ALIGN_4 -.L6_20_2: +L(6_20_2): KERNEL8x3_SUBN KERNEL8x3_SUBN @@ -1928,28 +1928,28 @@ KERNEL8x3_SUBN KERNEL8x3_SUBN dec %rax - jne .L6_20_2 + jne L(6_20_2) ALIGN_4 -.L6_20_6: +L(6_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_20_9 + je L(6_20_9) ALIGN_4 -.L6_20_7: +L(6_20_7): KERNEL8x3_SUBN dec %rax - jne .L6_20_7 + jne L(6_20_7) ALIGN_4 -.L6_20_9: +L(6_20_9): SAVE8x3 @@ -1960,13 +1960,13 @@ /**************************************************************************/ -.L6_21pre: +L(6_21pre): testq $4, M - jz .L6_30 + jz L(6_30) ALIGN_4 -.L6_21: +L(6_21): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -1975,11 +1975,11 @@ movq K, %rax sarq $3, %rax - je .L6_26 + je L(6_26) ALIGN_4 -.L6_22: +L(6_22): KERNEL4x3_SUBN KERNEL4x3_SUBN @@ -1991,27 +1991,27 @@ KERNEL4x3_SUBN KERNEL4x3_SUBN dec %rax - jne .L6_22 + jne L(6_22) ALIGN_4 -.L6_26: +L(6_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_29 + je L(6_29) ALIGN_4 -.L6_27: +L(6_27): KERNEL4x3_SUBN dec %rax - jne .L6_27 + jne L(6_27) ALIGN_4 -.L6_29: +L(6_29): SAVE4x3 @@ -2019,13 +2019,13 @@ ALIGN_4 -.L6_30: +L(6_30): testq $2, M - jz .L6_40 + jz L(6_40) ALIGN_4 -.L6_31: +L(6_31): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -2034,10 +2034,10 @@ movq K, %rax sarq $3, %rax - je .L6_36 + je L(6_36) ALIGN_4 -.L6_32: +L(6_32): KERNEL2x3_SUBN KERNEL2x3_SUBN @@ -2049,40 +2049,40 @@ KERNEL2x3_SUBN KERNEL2x3_SUBN dec %rax - jne .L6_32 + jne L(6_32) ALIGN_4 -.L6_36: +L(6_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_39 + je L(6_39) ALIGN_4 -.L6_37: +L(6_37): KERNEL2x3_SUBN dec %rax - jne .L6_37 + jne L(6_37) ALIGN_4 -.L6_39: +L(6_39): SAVE2x3 addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L6_40: +L(6_40): testq $1, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N ALIGN_4 -.L6_41: +L(6_41): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -2091,11 +2091,11 @@ movq K, %rax sarq $3,%rax - je .L6_46 + je L(6_46) ALIGN_4 -.L6_42: +L(6_42): KERNEL1x3_SUBN KERNEL1x3_SUBN @@ -2108,27 +2108,27 @@ KERNEL1x3_SUBN dec %rax - jne .L6_42 + jne L(6_42) ALIGN_4 -.L6_46: +L(6_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_49 + je L(6_49) ALIGN_4 -.L6_47: +L(6_47): KERNEL1x3_SUBN dec %rax - jne .L6_47 + jne L(6_47) ALIGN_4 -.L6_49: +L(6_49): SAVE1x3 @@ -2140,7 +2140,7 @@ /***************************************************************************************************************/ -.L7_10: +L(7_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -2151,11 +2151,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L7_20 + je L(7_20) ALIGN_4 -.L7_11: +L(7_11): leaq BUFFER2, BO // second buffer to BO addq $12 * SIZE, BO @@ -2171,10 +2171,10 @@ movq K, %rax sarq $3, %rax // K / 8 - je .L7_16 + je L(7_16) ALIGN_5 -.L7_12: +L(7_12): /* prefetcht0 B_PR1(BO) prefetcht0 B_PR1+64(BO) @@ -2190,50 +2190,50 @@ KERNEL16x3_SUBN KERNEL16x3_SUBN dec %rax - jne .L7_12 + jne L(7_12) ALIGN_4 -.L7_16: +L(7_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_19 + je L(7_19) ALIGN_5 -.L7_17: +L(7_17): KERNEL16x3_SUBN dec %rax - jne .L7_17 + jne L(7_17) -.L7_19: +L(7_19): SAVE16x3 addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L7_11 + jg L(7_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L7_20: +L(7_20): // Test rest of M testq $15, M - jz .L7_60 // to next 3 lines of N + jz L(7_60) // to next 3 lines of N testq $8, M - jz .L7_21pre + jz L(7_21pre) ALIGN_4 /**************************************************************************/ -.L7_20_1: +L(7_20_1): leaq BUFFER2, BO // first buffer to BO addq $12 * SIZE, BO @@ -2242,11 +2242,11 @@ movq K, %rax sarq $3, %rax - je .L7_20_6 + je L(7_20_6) ALIGN_4 -.L7_20_2: +L(7_20_2): KERNEL8x3_SUBN KERNEL8x3_SUBN @@ -2259,26 +2259,26 @@ KERNEL8x3_SUBN dec %rax - jne .L7_20_2 + jne L(7_20_2) ALIGN_4 -.L7_20_6: +L(7_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_20_9 + je L(7_20_9) ALIGN_4 -.L7_20_7: +L(7_20_7): KERNEL8x3_SUBN dec %rax - jne .L7_20_7 + jne L(7_20_7) ALIGN_4 -.L7_20_9: +L(7_20_9): SAVE8x3 @@ -2289,13 +2289,13 @@ /**************************************************************************/ -.L7_21pre: +L(7_21pre): testq $4, M - jz .L7_30 + jz L(7_30) ALIGN_4 -.L7_21: +L(7_21): leaq BUFFER2, BO // second buffer to BO addq $12 * SIZE, BO @@ -2304,11 +2304,11 @@ movq K, %rax sarq $3, %rax - je .L7_26 + je L(7_26) ALIGN_4 -.L7_22: +L(7_22): KERNEL4x3_SUBN KERNEL4x3_SUBN @@ -2321,27 +2321,27 @@ KERNEL4x3_SUBN dec %rax - jne .L7_22 + jne L(7_22) ALIGN_4 -.L7_26: +L(7_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_29 + je L(7_29) ALIGN_4 -.L7_27: +L(7_27): KERNEL4x3_SUBN dec %rax - jne .L7_27 + jne L(7_27) ALIGN_4 -.L7_29: +L(7_29): SAVE4x3 @@ -2349,13 +2349,13 @@ ALIGN_4 -.L7_30: +L(7_30): testq $2, M - jz .L7_40 + jz L(7_40) ALIGN_4 -.L7_31: +L(7_31): leaq BUFFER2, BO // second buffer to BO addq $12 * SIZE, BO @@ -2364,11 +2364,11 @@ movq K, %rax sarq $3, %rax - je .L7_36 + je L(7_36) ALIGN_4 -.L7_32: +L(7_32): KERNEL2x3_SUBN KERNEL2x3_SUBN @@ -2381,40 +2381,40 @@ KERNEL2x3_SUBN dec %rax - jne .L7_32 + jne L(7_32) ALIGN_4 -.L7_36: +L(7_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_39 + je L(7_39) ALIGN_4 -.L7_37: +L(7_37): KERNEL2x3_SUBN dec %rax - jne .L7_37 + jne L(7_37) ALIGN_4 -.L7_39: +L(7_39): SAVE2x3 addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L7_40: +L(7_40): testq $1, M - jz .L7_60 // to next 3 lines of N + jz L(7_60) // to next 3 lines of N ALIGN_4 -.L7_41: +L(7_41): leaq BUFFER2, BO // second buffer to BO addq $12 * SIZE, BO @@ -2423,11 +2423,11 @@ movq K, %rax sarq $3, %rax - je .L7_46 + je L(7_46) ALIGN_4 -.L7_42: +L(7_42): KERNEL1x3_SUBN KERNEL1x3_SUBN KERNEL1x3_SUBN @@ -2439,27 +2439,27 @@ KERNEL1x3_SUBN dec %rax - jne .L7_42 + jne L(7_42) ALIGN_4 -.L7_46: +L(7_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_49 + je L(7_49) ALIGN_4 -.L7_47: +L(7_47): KERNEL1x3_SUBN dec %rax - jne .L7_47 + jne L(7_47) ALIGN_4 -.L7_49: +L(7_49): SAVE1x3 @@ -2468,15 +2468,15 @@ -.L7_60: +L(7_60): decq J // j -- - jg .L6_01 + jg L(6_01) -.L2_0: +L(2_0): cmpq $0, Nmod6 // N % 6 == 0 - je .L999 + je L(999) /************************************************************************************************ * Loop for Nmod6 / 2 > 0 @@ -2484,19 +2484,19 @@ movq Nmod6, J sarq $1, J // j = j / 2 - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L2_01b + jz L(2_01b) ALIGN_4 -.L2_01a: +L(2_01a): prefetcht0 512(BO1) prefetchw 512(BO) @@ -2513,30 +2513,30 @@ addq $8*SIZE,BO1 addq $8*SIZE,BO decq %rax - jnz .L2_01a + jnz L(2_01a) -.L2_01b: +L(2_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L2_02d + jz L(2_02d) ALIGN_4 -.L2_02c: +L(2_02c): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02c + jnz L(2_02c) -.L2_02d: +L(2_02d): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -2546,11 +2546,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2559,7 +2559,7 @@ movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2570,7 +2570,7 @@ negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI,8) KERNEL16x2_1 @@ -2584,7 +2584,7 @@ KERNEL16x2_3 KERNEL16x2_4 - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI,8) KERNEL16x2_1 @@ -2598,16 +2598,16 @@ KERNEL16x2_3 KERNEL16x2_4 - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2619,39 +2619,39 @@ negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE16x2 addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 3 lines of N + jz L(2_60) // to next 3 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2660,7 +2660,7 @@ movq K, %rax andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2671,7 +2671,7 @@ negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1 @@ -2685,7 +2685,7 @@ KERNEL8x2_3 KERNEL8x2_4 - je .L2_20_6 + je L(2_20_6) prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1 @@ -2699,16 +2699,16 @@ KERNEL8x2_3 KERNEL8x2_4 - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2720,15 +2720,15 @@ negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): SAVE8x2 @@ -2739,13 +2739,13 @@ /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2754,7 +2754,7 @@ movq K, %rax andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -2765,7 +2765,7 @@ negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1 @@ -2779,7 +2779,7 @@ KERNEL4x2_3 KERNEL4x2_4 - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1 @@ -2793,16 +2793,16 @@ KERNEL4x2_3 KERNEL4x2_4 - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2814,15 +2814,15 @@ negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): SAVE4x2 @@ -2830,13 +2830,13 @@ ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2845,7 +2845,7 @@ movq K, %rax andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2856,7 +2856,7 @@ negq %rax ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_1 KERNEL2x2_2 @@ -2868,7 +2868,7 @@ KERNEL2x2_3 KERNEL2x2_4 - je .L2_36 + je L(2_36) KERNEL2x2_1 KERNEL2x2_2 @@ -2880,16 +2880,16 @@ KERNEL2x2_3 KERNEL2x2_4 - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2901,28 +2901,28 @@ negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): SAVE2x2 addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2931,7 +2931,7 @@ movq K, %rax andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2941,7 +2941,7 @@ negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_1 KERNEL1x2_2 @@ -2953,7 +2953,7 @@ KERNEL1x2_3 KERNEL1x2_4 - je .L2_46 + je L(2_46) KERNEL1x2_1 KERNEL1x2_2 @@ -2965,16 +2965,16 @@ KERNEL1x2_3 KERNEL1x2_4 - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2985,29 +2985,29 @@ negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): SAVE1x2 addq $1 * SIZE, CO1 # coffset += 1 ALIGN_4 -.L2_60: +L(2_60): decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -3015,30 +3015,30 @@ movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -3048,11 +3048,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3061,7 +3061,7 @@ movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -3071,7 +3071,7 @@ negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI,8) KERNEL16x1_1 @@ -3084,7 +3084,7 @@ KERNEL16x1_3 KERNEL16x1_4 - je .L1_16 + je L(1_16) prefetcht0 B_PR1(BO,BI,8) KERNEL16x1_1 @@ -3097,16 +3097,16 @@ KERNEL16x1_3 KERNEL16x1_4 - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -3117,39 +3117,39 @@ negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE16x1 addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3158,7 +3158,7 @@ movq K, %rax andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -3168,7 +3168,7 @@ negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1 @@ -3181,7 +3181,7 @@ KERNEL8x1_3 KERNEL8x1_4 - je .L1_20_6 + je L(1_20_6) prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1 @@ -3194,16 +3194,16 @@ KERNEL8x1_3 KERNEL8x1_4 - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -3214,15 +3214,15 @@ negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): SAVE8x1 @@ -3233,13 +3233,13 @@ /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3248,7 +3248,7 @@ movq K, %rax andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -3258,7 +3258,7 @@ negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1 @@ -3271,7 +3271,7 @@ KERNEL4x1_3 KERNEL4x1_4 - je .L1_26 + je L(1_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1 @@ -3284,16 +3284,16 @@ KERNEL4x1_3 KERNEL4x1_4 - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -3304,15 +3304,15 @@ negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): SAVE4x1 @@ -3320,13 +3320,13 @@ ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3335,7 +3335,7 @@ movq K, %rax andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -3345,7 +3345,7 @@ negq %rax ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_1 KERNEL2x1_2 @@ -3357,7 +3357,7 @@ KERNEL2x1_3 KERNEL2x1_4 - je .L1_36 + je L(1_36) KERNEL2x1_1 KERNEL2x1_2 @@ -3369,16 +3369,16 @@ KERNEL2x1_3 KERNEL2x1_4 - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -3389,28 +3389,28 @@ negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): SAVE2x1 addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3419,7 +3419,7 @@ movq K, %rax andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -3428,7 +3428,7 @@ negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_1 KERNEL1x1_2 @@ -3440,7 +3440,7 @@ KERNEL1x1_3 KERNEL1x1_4 - je .L1_46 + je L(1_46) KERNEL1x1_1 KERNEL1x1_2 @@ -3452,16 +3452,16 @@ KERNEL1x1_3 KERNEL1x1_4 - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -3471,15 +3471,15 @@ negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): SAVE1x1 @@ -3487,7 +3487,7 @@ ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp @@ -3577,13 +3577,13 @@ STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -3612,19 +3612,19 @@ movq Ndiv6, J cmpq $0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L2_01b + jz L(2_01b) ALIGN_4 -.L2_01a: +L(2_01a): prefetcht0 512(BO1) prefetchw 512(BO) @@ -3641,30 +3641,30 @@ addq $8*SIZE,BO1 addq $8*SIZE,BO decq %rax - jnz .L2_01a + jnz L(2_01a) -.L2_01b: +L(2_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L2_02d + jz L(2_02d) ALIGN_4 -.L2_02c: +L(2_02c): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02c + jnz L(2_02c) -.L2_02d: +L(2_02d): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -3678,11 +3678,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3719,7 +3719,7 @@ #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3730,7 +3730,7 @@ negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI,8) KERNEL16x2_1 @@ -3744,7 +3744,7 @@ KERNEL16x2_3 KERNEL16x2_4 - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI,8) KERNEL16x2_1 @@ -3758,12 +3758,12 @@ KERNEL16x2_3 KERNEL16x2_4 - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3771,7 +3771,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3783,15 +3783,15 @@ negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE16x2 @@ -3813,25 +3813,25 @@ addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 3 lines of N + jz L(2_60) // to next 3 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3869,7 +3869,7 @@ andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3880,7 +3880,7 @@ negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1 @@ -3894,7 +3894,7 @@ KERNEL8x2_3 KERNEL8x2_4 - je .L2_20_6 + je L(2_20_6) prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1 @@ -3908,12 +3908,12 @@ KERNEL8x2_3 KERNEL8x2_4 - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -3921,7 +3921,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3933,15 +3933,15 @@ negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): SAVE8x2 @@ -3968,13 +3968,13 @@ /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4012,7 +4012,7 @@ andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -4023,7 +4023,7 @@ negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1 @@ -4037,7 +4037,7 @@ KERNEL4x2_3 KERNEL4x2_4 - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1 @@ -4051,12 +4051,12 @@ KERNEL4x2_3 KERNEL4x2_4 - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4064,7 +4064,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4076,15 +4076,15 @@ negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): SAVE4x2 @@ -4108,13 +4108,13 @@ ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4152,7 +4152,7 @@ andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4163,7 +4163,7 @@ negq %rax ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_1 KERNEL2x2_2 @@ -4175,7 +4175,7 @@ KERNEL2x2_3 KERNEL2x2_4 - je .L2_36 + je L(2_36) KERNEL2x2_1 KERNEL2x2_2 @@ -4187,12 +4187,12 @@ KERNEL2x2_3 KERNEL2x2_4 - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -4200,7 +4200,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4212,15 +4212,15 @@ negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): SAVE2x2 @@ -4243,13 +4243,13 @@ addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4285,7 +4285,7 @@ #endif andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4295,7 +4295,7 @@ negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_1 KERNEL1x2_2 @@ -4307,7 +4307,7 @@ KERNEL1x2_3 KERNEL1x2_4 - je .L2_46 + je L(2_46) KERNEL1x2_1 KERNEL1x2_2 @@ -4319,12 +4319,12 @@ KERNEL1x2_3 KERNEL1x2_4 - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -4332,7 +4332,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4343,15 +4343,15 @@ negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): SAVE1x2 @@ -4377,17 +4377,17 @@ -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -4395,30 +4395,30 @@ movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -4432,11 +4432,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4472,7 +4472,7 @@ #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -4482,7 +4482,7 @@ negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI,8) KERNEL16x1_1 @@ -4495,7 +4495,7 @@ KERNEL16x1_3 KERNEL16x1_4 - je .L1_16 + je L(1_16) prefetcht0 B_PR1(BO,BI,8) KERNEL16x1_1 @@ -4508,12 +4508,12 @@ KERNEL16x1_3 KERNEL16x1_4 - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -4521,7 +4521,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -4532,15 +4532,15 @@ negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE16x1 @@ -4561,25 +4561,25 @@ addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4616,7 +4616,7 @@ andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -4626,7 +4626,7 @@ negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1 @@ -4639,7 +4639,7 @@ KERNEL8x1_3 KERNEL8x1_4 - je .L1_20_6 + je L(1_20_6) prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1 @@ -4652,12 +4652,12 @@ KERNEL8x1_3 KERNEL8x1_4 - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -4665,7 +4665,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -4676,15 +4676,15 @@ negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): SAVE8x1 @@ -4710,13 +4710,13 @@ /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4753,7 +4753,7 @@ andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -4763,7 +4763,7 @@ negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1 @@ -4776,7 +4776,7 @@ KERNEL4x1_3 KERNEL4x1_4 - je .L1_26 + je L(1_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1 @@ -4789,12 +4789,12 @@ KERNEL4x1_3 KERNEL4x1_4 - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4802,7 +4802,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -4813,15 +4813,15 @@ negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): SAVE4x1 @@ -4844,13 +4844,13 @@ ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4887,7 +4887,7 @@ andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -4897,7 +4897,7 @@ negq %rax ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_1 KERNEL2x1_2 @@ -4909,7 +4909,7 @@ KERNEL2x1_3 KERNEL2x1_4 - je .L1_36 + je L(1_36) KERNEL2x1_1 KERNEL2x1_2 @@ -4921,12 +4921,12 @@ KERNEL2x1_3 KERNEL2x1_4 - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -4934,7 +4934,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -4945,15 +4945,15 @@ negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): SAVE2x1 @@ -4975,13 +4975,13 @@ addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5016,7 +5016,7 @@ #endif andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -5025,7 +5025,7 @@ negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_1 KERNEL1x1_2 @@ -5037,7 +5037,7 @@ KERNEL1x1_3 KERNEL1x1_4 - je .L1_46 + je L(1_46) KERNEL1x1_1 KERNEL1x1_2 @@ -5049,12 +5049,12 @@ KERNEL1x1_3 KERNEL1x1_4 - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -5062,7 +5062,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -5072,15 +5072,15 @@ negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): SAVE1x1 @@ -5102,7 +5102,7 @@ ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/dgemm_kernel_4x4_haswell.S b/kernel/x86_64/dgemm_kernel_4x4_haswell.S index 29501df8e3..5516742364 100644 --- a/kernel/x86_64/dgemm_kernel_4x4_haswell.S +++ b/kernel/x86_64/dgemm_kernel_4x4_haswell.S @@ -1295,13 +1295,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $ 0, OLD_M - je .L999 + je L(999) cmpq $ 0, OLD_N - je .L999 + je L(999) cmpq $ 0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -1321,10 +1321,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv12, J cmpq $ 0, J - je .L4_0 + je L(4_0) ALIGN_4 -.L12_01: +L(12_01): // copy to sub buffer movq K, %rax salq $2,%rax // K * 4 ; read 2 values @@ -1336,10 +1336,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $1 , %rax // K / 2 - jz .L12_01a_2 + jz L(12_01a_2) ALIGN_4 -.L12_01a_1: +L(12_01a_1): prefetcht0 512(BO1) prefetcht0 512(BO2) @@ -1368,19 +1368,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 24 *SIZE ,BO decq %rax - jnz .L12_01a_1 + jnz L(12_01a_1) -.L12_01a_2: +L(12_01a_2): movq K, %rax andq $1, %rax // K % 2 - jz .L12_03c + jz L(12_03c) ALIGN_4 -.L12_02b: +L(12_02b): vmovups 0 * SIZE(BO1), %ymm1 vmovups 0 * SIZE(BO2), %ymm2 @@ -1393,13 +1393,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4*SIZE,BO3 addq $ 12*SIZE,BO decq %rax - jnz .L12_02b + jnz L(12_02b) -.L12_03c: +L(12_03c): movq BO3, B // next offset of B -.L12_10: +L(12_10): movq C, CO1 leaq (C, LDC, 8), C leaq (C, LDC, 4), C // c += 12 * ldc @@ -1410,11 +1410,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L12_20 + je L(12_20) ALIGN_4 -.L12_11: +L(12_11): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -1423,7 +1423,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, %rax // K / 8 cmpq $2, %rax - jl .L12_13 + jl L(12_13) KERNEL4x12_I @@ -1437,10 +1437,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M2 subq $2, %rax - je .L12_12a + je L(12_12a) ALIGN_5 -.L12_12: +L(12_12): KERNEL4x12_M1 KERNEL4x12_M2 @@ -1453,9 +1453,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M2 dec %rax - jne .L12_12 + jne L(12_12) -.L12_12a: +L(12_12a): KERNEL4x12_M1 KERNEL4x12_M2 @@ -1467,13 +1467,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M1 KERNEL4x12_E - jmp .L12_16 + jmp L(12_16) -.L12_13: +L(12_13): test $1, %rax - jz .L12_14 + jz L(12_14) KERNEL4x12_I KERNEL4x12_M2 @@ -1485,56 +1485,56 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M1 KERNEL4x12_E - jmp .L12_16 + jmp L(12_16) -.L12_14: +L(12_14): INIT4x12 -.L12_16: +L(12_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L12_19 + je L(12_19) ALIGN_4 -.L12_17: +L(12_17): KERNEL4x12_SUB dec %rax - jne .L12_17 + jne L(12_17) ALIGN_4 -.L12_19: +L(12_19): SAVE4x12 decq I # i -- - jne .L12_11 + jne L(12_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L12_20: +L(12_20): // Test rest of M testq $3, M - jz .L12_100 // to next 16 lines of N + jz L(12_100) // to next 16 lines of N -.L12_30: +L(12_30): testq $2, M - jz .L12_40 + jz L(12_40) ALIGN_4 -.L12_31: +L(12_31): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -1543,10 +1543,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax - je .L12_36 + je L(12_36) ALIGN_4 -.L12_32: +L(12_32): KERNEL2x12_SUB KERNEL2x12_SUB @@ -1559,39 +1559,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x12_SUB dec %rax - jne .L12_32 + jne L(12_32) ALIGN_4 -.L12_36: +L(12_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L12_39 + je L(12_39) ALIGN_4 -.L12_37: +L(12_37): KERNEL2x12_SUB dec %rax - jne .L12_37 + jne L(12_37) ALIGN_4 -.L12_39: +L(12_39): SAVE2x12 ALIGN_4 -.L12_40: +L(12_40): testq $1, M - jz .L12_100 // to next 3 lines of N + jz L(12_100) // to next 3 lines of N ALIGN_4 -.L12_41: +L(12_41): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -1600,11 +1600,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3,%rax - je .L12_46 + je L(12_46) ALIGN_4 -.L12_42: +L(12_42): KERNEL1x12_SUB KERNEL1x12_SUB @@ -1618,48 +1618,48 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. dec %rax - jne .L12_42 + jne L(12_42) ALIGN_4 -.L12_46: +L(12_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L12_49 + je L(12_49) ALIGN_4 -.L12_47: +L(12_47): KERNEL1x12_SUB dec %rax - jne .L12_47 + jne L(12_47) ALIGN_4 -.L12_49: +L(12_49): SAVE1x12 ALIGN_4 -.L12_100: +L(12_100): decq J // j -- - jg .L12_01 + jg L(12_01) -.L4_0: +L(4_0): cmpq $ 0, Nmod12 // N % 12 == 0 - je .L999 + je L(999) movq Nmod12, J sarq $2, J // j = j / 4 - je .L2_0 + je L(2_0) -.L4_10: +L(4_10): movq C, CO1 leaq (C, LDC, 4), C // c += 4 * ldc @@ -1669,11 +1669,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L4_20 + je L(4_20) ALIGN_4 -.L4_11: +L(4_11): movq B, BO addq $12 * SIZE, BO @@ -1681,7 +1681,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, %rax // K / 8 cmpq $2, %rax - jl .L4_13 + jl L(4_13) KERNEL4x4_I @@ -1695,11 +1695,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 subq $2, %rax - je .L4_12a + je L(4_12a) ALIGN_5 -.L4_12: +L(4_12): KERNEL4x4_M1 KERNEL4x4_M2 @@ -1712,9 +1712,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 dec %rax - jne .L4_12 + jne L(4_12) -.L4_12a: +L(4_12a): KERNEL4x4_M1 KERNEL4x4_M2 @@ -1726,13 +1726,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - jmp .L4_16 + jmp L(4_16) -.L4_13: +L(4_13): test $1, %rax - jz .L4_14 + jz L(4_14) KERNEL4x4_I KERNEL4x4_M2 @@ -1744,56 +1744,56 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - jmp .L4_16 + jmp L(4_16) -.L4_14: +L(4_14): INIT4x4 -.L4_16: +L(4_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L4_19 + je L(4_19) ALIGN_4 -.L4_17: +L(4_17): KERNEL4x4_SUB dec %rax - jne .L4_17 + jne L(4_17) ALIGN_4 -.L4_19: +L(4_19): SAVE4x4 decq I # i -- - jg .L4_11 + jg L(4_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L4_20: +L(4_20): // Test rest of M testq $3, M - jz .L4_100 // to next 16 lines of N + jz L(4_100) // to next 16 lines of N -.L4_30: +L(4_30): testq $2, M - jz .L4_40 + jz L(4_40) ALIGN_4 -.L4_31: +L(4_31): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -1802,10 +1802,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax - je .L4_36 + je L(4_36) ALIGN_4 -.L4_32: +L(4_32): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1818,36 +1818,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB dec %rax - jne .L4_32 + jne L(4_32) ALIGN_4 -.L4_36: +L(4_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L4_39 + je L(4_39) ALIGN_4 -.L4_37: +L(4_37): KERNEL2x4_SUB dec %rax - jne .L4_37 + jne L(4_37) -.L4_39: +L(4_39): SAVE2x4 -.L4_40: +L(4_40): testq $1, M - jz .L4_100 // to next 3 lines of N + jz L(4_100) // to next 3 lines of N ALIGN_4 -.L4_41: +L(4_41): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -1856,11 +1856,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3,%rax - je .L4_46 + je L(4_46) ALIGN_4 -.L4_42: +L(4_42): KERNEL1x4_SUB KERNEL1x4_SUB @@ -1873,52 +1873,52 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB dec %rax - jne .L4_42 + jne L(4_42) ALIGN_4 -.L4_46: +L(4_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L4_49 + je L(4_49) ALIGN_4 -.L4_47: +L(4_47): KERNEL1x4_SUB dec %rax - jne .L4_47 + jne L(4_47) ALIGN_4 -.L4_49: +L(4_49): SAVE1x4 ALIGN_4 -.L4_100: +L(4_100): movq K, %rax salq $2, %rax // * 4 leaq (B , %rax, SIZE), B decq J // j -- - jg .L4_10 + jg L(4_10) /***************************************************************************************************************/ -.L2_0: +L(2_0): movq Nmod12, J testq $2, J - je .L1_0 + je L(1_0) -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -1928,11 +1928,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): movq B, BO addq $12 * SIZE, BO @@ -1941,11 +1941,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax // K / 8 - je .L2_16 + je L(2_16) ALIGN_5 -.L2_12: +L(2_12): KERNEL4x2_SUB KERNEL4x2_SUB @@ -1958,51 +1958,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB dec %rax - jne .L2_12 + jne L(2_12) -.L2_16: +L(2_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) ALIGN_4 -.L2_17: +L(2_17): KERNEL4x2_SUB dec %rax - jne .L2_17 + jne L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE4x2 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $3, M - jz .L2_100 // to next 16 lines of N + jz L(2_100) // to next 16 lines of N -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -2011,10 +2011,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax - je .L2_36 + je L(2_36) ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_SUB KERNEL2x2_SUB @@ -2027,33 +2027,33 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB dec %rax - jne .L2_32 + jne L(2_32) -.L2_36: +L(2_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB dec %rax - jne .L2_37 + jne L(2_37) -.L2_39: +L(2_39): SAVE2x2 -.L2_40: +L(2_40): testq $1, M - jz .L2_100 // to next 3 lines of N + jz L(2_100) // to next 3 lines of N -.L2_41: +L(2_41): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -2062,11 +2062,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3,%rax - je .L2_46 + je L(2_46) ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_SUB KERNEL1x2_SUB @@ -2079,28 +2079,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB dec %rax - jne .L2_42 + jne L(2_42) -.L2_46: +L(2_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB dec %rax - jne .L2_47 + jne L(2_47) -.L2_49: +L(2_49): SAVE1x2 -.L2_100: +L(2_100): movq K, %rax salq $1, %rax // * 2 @@ -2108,13 +2108,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /***************************************************************************************************************/ -.L1_0: +L(1_0): movq Nmod12, J testq $1, J - je .L999 + je L(999) -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -2124,11 +2124,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): movq B, BO addq $12 * SIZE, BO @@ -2137,59 +2137,59 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax // K / 8 - je .L1_16 + je L(1_16) ALIGN_5 -.L1_12: +L(1_12): KERNEL4x1 dec %rax - jne .L1_12 + jne L(1_12) -.L1_16: +L(1_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) ALIGN_4 -.L1_17: +L(1_17): KERNEL4x1_SUB dec %rax - jne .L1_17 + jne L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE4x1 decq I # i -- - jg .L1_11 + jg L(1_11) /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $3, M - jz .L1_100 + jz L(1_100) -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -2198,10 +2198,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax - je .L1_36 + je L(1_36) ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_SUB KERNEL2x1_SUB @@ -2215,33 +2215,33 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. dec %rax - jne .L1_32 + jne L(1_32) -.L1_36: +L(1_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB dec %rax - jne .L1_37 + jne L(1_37) -.L1_39: +L(1_39): SAVE2x1 -.L1_40: +L(1_40): testq $1, M - jz .L1_100 // to next 3 lines of N + jz L(1_100) // to next 3 lines of N -.L1_41: +L(1_41): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -2250,11 +2250,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3,%rax - je .L1_46 + je L(1_46) ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_SUB KERNEL1x1_SUB @@ -2267,34 +2267,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB dec %rax - jne .L1_42 + jne L(1_42) -.L1_46: +L(1_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB dec %rax - jne .L1_47 + jne L(1_47) -.L1_49: +L(1_49): SAVE1x1 -.L1_100: +L(1_100): -.L999: +L(999): vzeroupper movq SP, %rsp @@ -2386,13 +2386,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $ 0, OLD_M - je .L999 + je L(999) cmpq $ 0, OLD_N - je .L999 + je L(999) cmpq $ 0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -2421,10 +2421,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv12, J cmpq $ 0, J - je .L2_0 + je L(2_0) ALIGN_4 -.L4_10: +L(4_10): movq C, CO1 leaq (C, LDC, 4), C // c += 4 * ldc @@ -2439,11 +2439,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L4_20 + je L(4_20) ALIGN_4 -.L4_11: +L(4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2478,7 +2478,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, %rax // K / 8 cmpq $2, %rax - jl .L4_13 + jl L(4_13) KERNEL4x4_I @@ -2492,11 +2492,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 subq $2, %rax - je .L4_12a + je L(4_12a) ALIGN_5 -.L4_12: +L(4_12): KERNEL4x4_M1 KERNEL4x4_M2 @@ -2509,9 +2509,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 dec %rax - jne .L4_12 + jne L(4_12) -.L4_12a: +L(4_12a): KERNEL4x4_M1 KERNEL4x4_M2 @@ -2523,13 +2523,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - jmp .L4_16 + jmp L(4_16) -.L4_13: +L(4_13): test $1, %rax - jz .L4_14 + jz L(4_14) KERNEL4x4_I KERNEL4x4_M2 @@ -2541,32 +2541,32 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - jmp .L4_16 + jmp L(4_16) -.L4_14: +L(4_14): INIT4x4 -.L4_16: +L(4_16): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L4_19 + je L(4_19) ALIGN_4 -.L4_17: +L(4_17): KERNEL4x4_SUB dec %rax - jne .L4_17 + jne L(4_17) ALIGN_4 -.L4_19: +L(4_19): SAVE4x4 @@ -2584,26 +2584,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif decq I # i -- - jg .L4_11 + jg L(4_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L4_20: +L(4_20): // Test rest of M testq $3, M - jz .L4_100 // to next 16 lines of N + jz L(4_100) // to next 16 lines of N -.L4_30: +L(4_30): testq $2, M - jz .L4_40 + jz L(4_40) ALIGN_4 -.L4_31: +L(4_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2639,10 +2639,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT2x4 sarq $3, %rax - je .L4_36 + je L(4_36) ALIGN_4 -.L4_32: +L(4_32): KERNEL2x4_SUB KERNEL2x4_SUB @@ -2655,26 +2655,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB dec %rax - jne .L4_32 + jne L(4_32) ALIGN_4 -.L4_36: +L(4_36): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L4_39 + je L(4_39) ALIGN_4 -.L4_37: +L(4_37): KERNEL2x4_SUB dec %rax - jne .L4_37 + jne L(4_37) -.L4_39: +L(4_39): SAVE2x4 @@ -2692,13 +2692,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.L4_40: +L(4_40): testq $1, M - jz .L4_100 // to next 3 lines of N + jz L(4_100) // to next 3 lines of N ALIGN_4 -.L4_41: +L(4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2734,11 +2734,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT1x4 sarq $3,%rax - je .L4_46 + je L(4_46) ALIGN_4 -.L4_42: +L(4_42): KERNEL1x4_SUB KERNEL1x4_SUB @@ -2751,27 +2751,27 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB dec %rax - jne .L4_42 + jne L(4_42) ALIGN_4 -.L4_46: +L(4_46): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L4_49 + je L(4_49) ALIGN_4 -.L4_47: +L(4_47): KERNEL1x4_SUB dec %rax - jne .L4_47 + jne L(4_47) ALIGN_4 -.L4_49: +L(4_49): SAVE1x4 @@ -2788,7 +2788,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $1, KK // number of values in A #endif -.L4_100: +L(4_100): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK // number of values in B @@ -2799,20 +2799,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. salq $2, %rax // * 4 leaq (B , %rax, SIZE), B decq J // j -- - jg .L4_10 + jg L(4_10) /***************************************************************************************************************/ -.L2_0: +L(2_0): movq Nmod12, J testq $2, J - je .L1_0 + je L(1_0) -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -2828,11 +2828,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2869,11 +2869,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, %rax // K / 8 - je .L2_16 + je L(2_16) ALIGN_5 -.L2_12: +L(2_12): KERNEL4x2_SUB KERNEL4x2_SUB @@ -2886,27 +2886,27 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB dec %rax - jne .L2_12 + jne L(2_12) -.L2_16: +L(2_16): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) ALIGN_4 -.L2_17: +L(2_17): KERNEL4x2_SUB dec %rax - jne .L2_17 + jne L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE4x2 @@ -2925,26 +2925,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $3, M - jz .L2_100 // to next 16 lines of N + jz L(2_100) // to next 16 lines of N -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2980,10 +2980,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT2x2 sarq $3, %rax - je .L2_36 + je L(2_36) ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_SUB KERNEL2x2_SUB @@ -2996,25 +2996,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB dec %rax - jne .L2_32 + jne L(2_32) -.L2_36: +L(2_36): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB dec %rax - jne .L2_37 + jne L(2_37) -.L2_39: +L(2_39): SAVE2x2 @@ -3032,11 +3032,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.L2_40: +L(2_40): testq $1, M - jz .L2_100 // to next 3 lines of N + jz L(2_100) // to next 3 lines of N -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3072,11 +3072,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT1x2 sarq $3,%rax - je .L2_46 + je L(2_46) ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_SUB KERNEL1x2_SUB @@ -3089,24 +3089,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB dec %rax - jne .L2_42 + jne L(2_42) -.L2_46: +L(2_46): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB dec %rax - jne .L2_47 + jne L(2_47) -.L2_49: +L(2_49): SAVE1x2 @@ -3124,7 +3124,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.L2_100: +L(2_100): #if defined(TRMMKERNEL) && !defined(LEFT) @@ -3137,13 +3137,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /***************************************************************************************************************/ -.L1_0: +L(1_0): movq Nmod12, J testq $1, J - je .L999 + je L(999) -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -3157,11 +3157,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3197,36 +3197,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT4x1 sarq $3, %rax // K / 8 - je .L1_16 + je L(1_16) ALIGN_5 -.L1_12: +L(1_12): KERNEL4x1 dec %rax - jne .L1_12 + jne L(1_12) -.L1_16: +L(1_16): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) ALIGN_4 -.L1_17: +L(1_17): KERNEL4x1_SUB dec %rax - jne .L1_17 + jne L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE4x1 @@ -3245,25 +3245,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. decq I # i -- - jg .L1_11 + jg L(1_11) /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $3, M - jz .L1_100 + jz L(1_100) -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3299,10 +3299,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT2x1 sarq $3, %rax - je .L1_36 + je L(1_36) ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_SUB KERNEL2x1_SUB @@ -3316,24 +3316,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. dec %rax - jne .L1_32 + jne L(1_32) -.L1_36: +L(1_36): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB dec %rax - jne .L1_37 + jne L(1_37) -.L1_39: +L(1_39): SAVE2x1 @@ -3351,12 +3351,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.L1_40: +L(1_40): testq $1, M - jz .L1_100 // to next 3 lines of N + jz L(1_100) // to next 3 lines of N -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3392,11 +3392,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT1x1 sarq $3,%rax - je .L1_46 + je L(1_46) ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_SUB KERNEL1x1_SUB @@ -3409,25 +3409,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB dec %rax - jne .L1_42 + jne L(1_42) -.L1_46: +L(1_46): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB dec %rax - jne .L1_47 + jne L(1_47) -.L1_49: +L(1_49): SAVE1x1 @@ -3446,7 +3446,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L1_100: +L(1_100): #if defined(TRMMKERNEL) && !defined(LEFT) @@ -3455,7 +3455,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L999: +L(999): vzeroupper diff --git a/kernel/x86_64/dgemm_kernel_4x8_haswell.S b/kernel/x86_64/dgemm_kernel_4x8_haswell.S index adaa28bbc4..7d36e2b00d 100644 --- a/kernel/x86_64/dgemm_kernel_4x8_haswell.S +++ b/kernel/x86_64/dgemm_kernel_4x8_haswell.S @@ -1999,13 +1999,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $ 0, OLD_M - je .L999 + je L(999) cmpq $ 0, OLD_N - je .L999 + je L(999) cmpq $ 0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -2025,10 +2025,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv12, J cmpq $ 0, J - je .L8_0 + je L(8_0) ALIGN_4 -.L12_01: +L(12_01): // copy to sub buffer movq K, %rax salq $3,%rax // K * 8 ; read 8 values from BO1 @@ -2041,7 +2041,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L12_02b: +L(12_02b): vmovups 0 * SIZE(BO1), %ymm1 vmovups 4 * SIZE(BO1), %ymm2 @@ -2053,12 +2053,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 8*SIZE,BO2 addq $ 12*SIZE,BO decq %rax - jnz .L12_02b + jnz L(12_02b) -.L12_03c: +L(12_03c): -.L12_10: +L(12_10): movq C, CO1 leaq (C, LDC, 8), C leaq (C, LDC, 4), C // c += 12 * ldc @@ -2068,11 +2068,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L12_20 + je L(12_20) ALIGN_4 -.L12_11: +L(12_11): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -2081,7 +2081,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, %rax // K / 8 cmpq $2, %rax - jl .L12_13 + jl L(12_13) KERNEL4x12_I @@ -2095,10 +2095,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M2 subq $2, %rax - je .L12_12a + je L(12_12a) ALIGN_5 -.L12_12: +L(12_12): KERNEL4x12_M1 KERNEL4x12_M2 @@ -2111,9 +2111,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M2 dec %rax - jne .L12_12 + jne L(12_12) -.L12_12a: +L(12_12a): prefetcht0 ALPHA PREFETCHT0_C addq LDC,CO1 @@ -2134,13 +2134,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M1 KERNEL4x12_E - jmp .L12_16 + jmp L(12_16) -.L12_13: +L(12_13): test $1, %rax - jz .L12_14 + jz L(12_14) KERNEL4x12_I KERNEL4x12_M2 @@ -2152,32 +2152,32 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M1 KERNEL4x12_E - jmp .L12_16 + jmp L(12_16) -.L12_14: +L(12_14): INIT4x12 -.L12_16: +L(12_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L12_19 + je L(12_19) ALIGN_4 -.L12_17: +L(12_17): KERNEL4x12_SUB dec %rax - jne .L12_17 + jne L(12_17) ALIGN_4 -.L12_19: +L(12_19): SAVE4x12 @@ -2199,7 +2199,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, K decq I # i -- - jne .L12_11 + jne L(12_11) ALIGN_4 /************************************************************************** @@ -2216,20 +2216,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subq I, B -.L12_20: +L(12_20): // Test rest of M testq $3, M - jz .L12_100 // to next 16 lines of N + jz L(12_100) // to next 16 lines of N -.L12_30: +L(12_30): testq $2, M - jz .L12_40 + jz L(12_40) ALIGN_4 -.L12_31: +L(12_31): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -2238,10 +2238,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax - je .L12_36 + je L(12_36) ALIGN_4 -.L12_32: +L(12_32): KERNEL2x12_SUB KERNEL2x12_SUB @@ -2254,39 +2254,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x12_SUB dec %rax - jne .L12_32 + jne L(12_32) ALIGN_4 -.L12_36: +L(12_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L12_39 + je L(12_39) ALIGN_4 -.L12_37: +L(12_37): KERNEL2x12_SUB dec %rax - jne .L12_37 + jne L(12_37) ALIGN_4 -.L12_39: +L(12_39): SAVE2x12 ALIGN_4 -.L12_40: +L(12_40): testq $1, M - jz .L12_100 // to next 3 lines of N + jz L(12_100) // to next 3 lines of N ALIGN_4 -.L12_41: +L(12_41): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -2295,11 +2295,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3,%rax - je .L12_46 + je L(12_46) ALIGN_4 -.L12_42: +L(12_42): KERNEL1x12_SUB KERNEL1x12_SUB @@ -2313,39 +2313,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. dec %rax - jne .L12_42 + jne L(12_42) ALIGN_4 -.L12_46: +L(12_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L12_49 + je L(12_49) ALIGN_4 -.L12_47: +L(12_47): KERNEL1x12_SUB dec %rax - jne .L12_47 + jne L(12_47) ALIGN_4 -.L12_49: +L(12_49): SAVE1x12 ALIGN_4 -.L12_100: +L(12_100): /**************************************************************************************************/ -.L13_01: +L(13_01): // copy to sub buffer movq K, %rax salq $3,%rax // K * 8 ; read 8 values @@ -2360,7 +2360,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L13_02b: +L(13_02b): vmovups 4 * SIZE(BO2), %ymm1 vmovups 0 * SIZE(BO3), %ymm2 @@ -2372,11 +2372,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 8*SIZE,BO3 addq $ 12*SIZE,BO decq %rax - jnz .L13_02b + jnz L(13_02b) -.L13_10: +L(13_10): movq C, CO1 leaq (C, LDC, 8), C leaq (C, LDC, 4), C // c += 12 * ldc @@ -2387,11 +2387,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L13_20 + je L(13_20) ALIGN_4 -.L13_11: +L(13_11): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -2400,7 +2400,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, %rax // K / 8 cmpq $2, %rax - jl .L13_13 + jl L(13_13) KERNEL4x12_I @@ -2414,10 +2414,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M2 subq $2, %rax - je .L13_12a + je L(13_12a) ALIGN_5 -.L13_12: +L(13_12): KERNEL4x12_M1 KERNEL4x12_M2 @@ -2430,9 +2430,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M2 dec %rax - jne .L13_12 + jne L(13_12) -.L13_12a: +L(13_12a): prefetcht0 ALPHA PREFETCHT0_C addq LDC,CO1 @@ -2453,12 +2453,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M1 KERNEL4x12_E - jmp .L13_16 + jmp L(13_16) -.L13_13: +L(13_13): test $1, %rax - jz .L13_14 + jz L(13_14) KERNEL4x12_I KERNEL4x12_M2 @@ -2470,32 +2470,32 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x12_M1 KERNEL4x12_E - jmp .L13_16 + jmp L(13_16) -.L13_14: +L(13_14): INIT4x12 -.L13_16: +L(13_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L13_19 + je L(13_19) ALIGN_4 -.L13_17: +L(13_17): KERNEL4x12_SUB dec %rax - jne .L13_17 + jne L(13_17) ALIGN_4 -.L13_19: +L(13_19): SAVE4x12 @@ -2517,7 +2517,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, K decq I # i -- - jne .L13_11 + jne L(13_11) ALIGN_4 /************************************************************************** @@ -2533,20 +2533,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subq I, B -.L13_20: +L(13_20): // Test rest of M testq $3, M - jz .L13_100 // to next 16 lines of N + jz L(13_100) // to next 16 lines of N -.L13_30: +L(13_30): testq $2, M - jz .L13_40 + jz L(13_40) ALIGN_4 -.L13_31: +L(13_31): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -2555,10 +2555,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax - je .L13_36 + je L(13_36) ALIGN_4 -.L13_32: +L(13_32): KERNEL2x12_SUB KERNEL2x12_SUB @@ -2571,39 +2571,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x12_SUB dec %rax - jne .L13_32 + jne L(13_32) ALIGN_4 -.L13_36: +L(13_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L13_39 + je L(13_39) ALIGN_4 -.L13_37: +L(13_37): KERNEL2x12_SUB dec %rax - jne .L13_37 + jne L(13_37) ALIGN_4 -.L13_39: +L(13_39): SAVE2x12 ALIGN_4 -.L13_40: +L(13_40): testq $1, M - jz .L13_100 // to next 3 lines of N + jz L(13_100) // to next 3 lines of N ALIGN_4 -.L13_41: +L(13_41): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO @@ -2612,11 +2612,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3,%rax - je .L13_46 + je L(13_46) ALIGN_4 -.L13_42: +L(13_42): KERNEL1x12_SUB KERNEL1x12_SUB @@ -2630,52 +2630,52 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. dec %rax - jne .L13_42 + jne L(13_42) ALIGN_4 -.L13_46: +L(13_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L13_49 + je L(13_49) ALIGN_4 -.L13_47: +L(13_47): KERNEL1x12_SUB dec %rax - jne .L13_47 + jne L(13_47) ALIGN_4 -.L13_49: +L(13_49): SAVE1x12 ALIGN_4 -.L13_100: +L(13_100): decq J // j -- - jg .L12_01 + jg L(12_01) /**************************************************************************************************/ -.L8_0: +L(8_0): cmpq $ 0, Nmod12 // N % 12 == 0 - je .L999 + je L(999) movq Nmod12, J sarq $3, J // j = j / 8 - je .L4_0 + je L(4_0) -.L8_10: +L(8_10): movq C, CO1 leaq (C, LDC, 8), C // c += 4 * ldc @@ -2685,11 +2685,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L8_20 + je L(8_20) ALIGN_4 -.L8_11: +L(8_11): movq B, BO addq $12 * SIZE, BO @@ -2697,7 +2697,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, %rax // K / 8 cmpq $2, %rax - jl .L8_13 + jl L(8_13) KERNEL4x8_I @@ -2711,11 +2711,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x8_M2 subq $2, %rax - je .L8_12a + je L(8_12a) ALIGN_5 -.L8_12: +L(8_12): KERNEL4x8_M1 KERNEL4x8_M2 @@ -2728,9 +2728,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x8_M2 dec %rax - jne .L8_12 + jne L(8_12) -.L8_12a: +L(8_12a): KERNEL4x8_M1 KERNEL4x8_M2 @@ -2742,13 +2742,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x8_M1 KERNEL4x8_E - jmp .L8_16 + jmp L(8_16) -.L8_13: +L(8_13): test $1, %rax - jz .L8_14 + jz L(8_14) KERNEL4x8_I KERNEL4x8_M2 @@ -2760,56 +2760,56 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x8_M1 KERNEL4x8_E - jmp .L8_16 + jmp L(8_16) -.L8_14: +L(8_14): INIT4x8 -.L8_16: +L(8_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L8_19 + je L(8_19) ALIGN_4 -.L8_17: +L(8_17): KERNEL4x8_SUB dec %rax - jne .L8_17 + jne L(8_17) ALIGN_4 -.L8_19: +L(8_19): SAVE4x8 decq I # i -- - jg .L8_11 + jg L(8_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L8_20: +L(8_20): // Test rest of M testq $3, M - jz .L8_100 // to next 16 lines of N + jz L(8_100) // to next 16 lines of N -.L8_30: +L(8_30): testq $2, M - jz .L8_40 + jz L(8_40) ALIGN_4 -.L8_31: +L(8_31): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -2818,10 +2818,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax - je .L8_36 + je L(8_36) ALIGN_4 -.L8_32: +L(8_32): KERNEL2x8_SUB KERNEL2x8_SUB @@ -2834,36 +2834,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x8_SUB dec %rax - jne .L8_32 + jne L(8_32) ALIGN_4 -.L8_36: +L(8_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L8_39 + je L(8_39) ALIGN_4 -.L8_37: +L(8_37): KERNEL2x8_SUB dec %rax - jne .L8_37 + jne L(8_37) -.L8_39: +L(8_39): SAVE2x8 -.L8_40: +L(8_40): testq $1, M - jz .L8_100 // to next 3 lines of N + jz L(8_100) // to next 3 lines of N ALIGN_4 -.L8_41: +L(8_41): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -2872,11 +2872,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3,%rax - je .L8_46 + je L(8_46) ALIGN_4 -.L8_42: +L(8_42): KERNEL1x8_SUB KERNEL1x8_SUB @@ -2889,54 +2889,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x8_SUB dec %rax - jne .L8_42 + jne L(8_42) ALIGN_4 -.L8_46: +L(8_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L8_49 + je L(8_49) ALIGN_4 -.L8_47: +L(8_47): KERNEL1x8_SUB dec %rax - jne .L8_47 + jne L(8_47) ALIGN_4 -.L8_49: +L(8_49): SAVE1x8 ALIGN_4 -.L8_100: +L(8_100): movq K, %rax salq $3, %rax // * 8 leaq (B , %rax, SIZE), B decq J // j -- - jg .L8_10 + jg L(8_10) /**************************************************************************************************/ -.L4_0: +L(4_0): cmpq $ 0, Nmod12 // N % 12 == 0 - je .L999 + je L(999) movq Nmod12, J testq $4, J // j = j / 4 - je .L2_0 + je L(2_0) -.L4_10: +L(4_10): movq C, CO1 leaq (C, LDC, 4), C // c += 4 * ldc @@ -2946,11 +2946,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L4_20 + je L(4_20) ALIGN_4 -.L4_11: +L(4_11): movq B, BO addq $12 * SIZE, BO @@ -2958,7 +2958,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, %rax // K / 8 cmpq $2, %rax - jl .L4_13 + jl L(4_13) KERNEL4x4_I @@ -2972,11 +2972,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 subq $2, %rax - je .L4_12a + je L(4_12a) ALIGN_5 -.L4_12: +L(4_12): KERNEL4x4_M1 KERNEL4x4_M2 @@ -2989,9 +2989,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 dec %rax - jne .L4_12 + jne L(4_12) -.L4_12a: +L(4_12a): KERNEL4x4_M1 KERNEL4x4_M2 @@ -3003,13 +3003,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - jmp .L4_16 + jmp L(4_16) -.L4_13: +L(4_13): test $1, %rax - jz .L4_14 + jz L(4_14) KERNEL4x4_I KERNEL4x4_M2 @@ -3021,57 +3021,57 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - jmp .L4_16 + jmp L(4_16) -.L4_14: +L(4_14): INIT4x4 -.L4_16: +L(4_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L4_19 + je L(4_19) ALIGN_4 -.L4_17: +L(4_17): KERNEL4x4_SUB dec %rax - jne .L4_17 + jne L(4_17) ALIGN_4 -.L4_19: +L(4_19): SAVE4x4 decq I # i -- - jg .L4_11 + jg L(4_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L4_20: +L(4_20): // Test rest of M testq $3, M - jz .L4_100 // to next 16 lines of N + jz L(4_100) // to next 16 lines of N -.L4_30: +L(4_30): testq $2, M - jz .L4_40 + jz L(4_40) ALIGN_4 -.L4_31: +L(4_31): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -3080,10 +3080,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax - je .L4_36 + je L(4_36) ALIGN_4 -.L4_32: +L(4_32): KERNEL2x4_SUB KERNEL2x4_SUB @@ -3096,36 +3096,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB dec %rax - jne .L4_32 + jne L(4_32) ALIGN_4 -.L4_36: +L(4_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L4_39 + je L(4_39) ALIGN_4 -.L4_37: +L(4_37): KERNEL2x4_SUB dec %rax - jne .L4_37 + jne L(4_37) -.L4_39: +L(4_39): SAVE2x4 -.L4_40: +L(4_40): testq $1, M - jz .L4_100 // to next 3 lines of N + jz L(4_100) // to next 3 lines of N ALIGN_4 -.L4_41: +L(4_41): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -3134,11 +3134,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3,%rax - je .L4_46 + je L(4_46) ALIGN_4 -.L4_42: +L(4_42): KERNEL1x4_SUB KERNEL1x4_SUB @@ -3151,33 +3151,33 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB dec %rax - jne .L4_42 + jne L(4_42) ALIGN_4 -.L4_46: +L(4_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L4_49 + je L(4_49) ALIGN_4 -.L4_47: +L(4_47): KERNEL1x4_SUB dec %rax - jne .L4_47 + jne L(4_47) ALIGN_4 -.L4_49: +L(4_49): SAVE1x4 ALIGN_4 -.L4_100: +L(4_100): movq K, %rax salq $2, %rax // * 4 @@ -3188,13 +3188,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /***************************************************************************************************************/ -.L2_0: +L(2_0): movq Nmod12, J testq $2, J - je .L1_0 + je L(1_0) -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -3204,11 +3204,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): movq B, BO addq $12 * SIZE, BO @@ -3217,11 +3217,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax // K / 8 - je .L2_16 + je L(2_16) ALIGN_5 -.L2_12: +L(2_12): KERNEL4x2_SUB KERNEL4x2_SUB @@ -3234,51 +3234,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB dec %rax - jne .L2_12 + jne L(2_12) -.L2_16: +L(2_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) ALIGN_4 -.L2_17: +L(2_17): KERNEL4x2_SUB dec %rax - jne .L2_17 + jne L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE4x2 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $3, M - jz .L2_100 // to next 16 lines of N + jz L(2_100) // to next 16 lines of N -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -3287,10 +3287,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax - je .L2_36 + je L(2_36) ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_SUB KERNEL2x2_SUB @@ -3303,33 +3303,33 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB dec %rax - jne .L2_32 + jne L(2_32) -.L2_36: +L(2_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB dec %rax - jne .L2_37 + jne L(2_37) -.L2_39: +L(2_39): SAVE2x2 -.L2_40: +L(2_40): testq $1, M - jz .L2_100 // to next 3 lines of N + jz L(2_100) // to next 3 lines of N -.L2_41: +L(2_41): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -3338,11 +3338,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3,%rax - je .L2_46 + je L(2_46) ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_SUB KERNEL1x2_SUB @@ -3355,28 +3355,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB dec %rax - jne .L2_42 + jne L(2_42) -.L2_46: +L(2_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB dec %rax - jne .L2_47 + jne L(2_47) -.L2_49: +L(2_49): SAVE1x2 -.L2_100: +L(2_100): movq K, %rax salq $1, %rax // * 2 @@ -3384,13 +3384,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /***************************************************************************************************************/ -.L1_0: +L(1_0): movq Nmod12, J testq $1, J - je .L999 + je L(999) -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -3400,11 +3400,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): movq B, BO addq $12 * SIZE, BO @@ -3413,59 +3413,59 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax // K / 8 - je .L1_16 + je L(1_16) ALIGN_5 -.L1_12: +L(1_12): KERNEL4x1 dec %rax - jne .L1_12 + jne L(1_12) -.L1_16: +L(1_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) ALIGN_4 -.L1_17: +L(1_17): KERNEL4x1_SUB dec %rax - jne .L1_17 + jne L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE4x1 decq I # i -- - jg .L1_11 + jg L(1_11) /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $3, M - jz .L1_100 + jz L(1_100) -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -3474,10 +3474,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3, %rax - je .L1_36 + je L(1_36) ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_SUB KERNEL2x1_SUB @@ -3491,33 +3491,33 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. dec %rax - jne .L1_32 + jne L(1_32) -.L1_36: +L(1_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB dec %rax - jne .L1_37 + jne L(1_37) -.L1_39: +L(1_39): SAVE2x1 -.L1_40: +L(1_40): testq $1, M - jz .L1_100 // to next 3 lines of N + jz L(1_100) // to next 3 lines of N -.L1_41: +L(1_41): movq B, BO // first buffer to BO addq $12 * SIZE, BO @@ -3526,11 +3526,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax sarq $3,%rax - je .L1_46 + je L(1_46) ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_SUB KERNEL1x1_SUB @@ -3543,34 +3543,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB dec %rax - jne .L1_42 + jne L(1_42) -.L1_46: +L(1_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB dec %rax - jne .L1_47 + jne L(1_47) -.L1_49: +L(1_49): SAVE1x1 -.L1_100: +L(1_100): -.L999: +L(999): vzeroupper movq SP, %rsp @@ -3662,13 +3662,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $ 0, OLD_M - je .L999 + je L(999) cmpq $ 0, OLD_N - je .L999 + je L(999) cmpq $ 0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -3694,13 +3694,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif /*************************************************************************************************/ -.L8_0: +L(8_0): movq Ndiv12, J cmpq $ 0, J - je .L4_0 + je L(4_0) ALIGN_4 -.L8_10: +L(8_10): movq C, CO1 leaq (C, LDC, 8), C // c += 8 * ldc @@ -3715,11 +3715,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L8_20 + je L(8_20) ALIGN_4 -.L8_11: +L(8_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3754,7 +3754,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, %rax // K / 8 cmpq $2, %rax - jl .L8_13 + jl L(8_13) KERNEL4x8_I @@ -3768,11 +3768,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x8_M2 subq $2, %rax - je .L8_12a + je L(8_12a) ALIGN_5 -.L8_12: +L(8_12): KERNEL4x8_M1 KERNEL4x8_M2 @@ -3785,9 +3785,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x8_M2 dec %rax - jne .L8_12 + jne L(8_12) -.L8_12a: +L(8_12a): KERNEL4x8_M1 KERNEL4x8_M2 @@ -3799,13 +3799,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x8_M1 KERNEL4x8_E - jmp .L8_16 + jmp L(8_16) -.L8_13: +L(8_13): test $1, %rax - jz .L8_14 + jz L(8_14) KERNEL4x8_I KERNEL4x8_M2 @@ -3817,32 +3817,32 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x8_M1 KERNEL4x8_E - jmp .L8_16 + jmp L(8_16) -.L8_14: +L(8_14): INIT4x8 -.L8_16: +L(8_16): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L8_19 + je L(8_19) ALIGN_4 -.L8_17: +L(8_17): KERNEL4x8_SUB dec %rax - jne .L8_17 + jne L(8_17) ALIGN_4 -.L8_19: +L(8_19): SAVE4x8 @@ -3860,26 +3860,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif decq I # i -- - jg .L8_11 + jg L(8_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L8_20: +L(8_20): // Test rest of M testq $3, M - jz .L8_100 // to next 16 lines of N + jz L(8_100) // to next 16 lines of N -.L8_30: +L(8_30): testq $2, M - jz .L8_40 + jz L(8_40) ALIGN_4 -.L8_31: +L(8_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3915,10 +3915,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT2x8 sarq $3, %rax - je .L8_36 + je L(8_36) ALIGN_4 -.L8_32: +L(8_32): KERNEL2x8_SUB KERNEL2x8_SUB @@ -3931,26 +3931,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x8_SUB dec %rax - jne .L8_32 + jne L(8_32) ALIGN_4 -.L8_36: +L(8_36): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L8_39 + je L(8_39) ALIGN_4 -.L8_37: +L(8_37): KERNEL2x8_SUB dec %rax - jne .L8_37 + jne L(8_37) -.L8_39: +L(8_39): SAVE2x8 @@ -3968,13 +3968,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.L8_40: +L(8_40): testq $1, M - jz .L8_100 // to next 3 lines of N + jz L(8_100) // to next 3 lines of N ALIGN_4 -.L8_41: +L(8_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4010,11 +4010,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT1x8 sarq $3,%rax - je .L8_46 + je L(8_46) ALIGN_4 -.L8_42: +L(8_42): KERNEL1x8_SUB KERNEL1x8_SUB @@ -4027,27 +4027,27 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x8_SUB dec %rax - jne .L8_42 + jne L(8_42) ALIGN_4 -.L8_46: +L(8_46): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L8_49 + je L(8_49) ALIGN_4 -.L8_47: +L(8_47): KERNEL1x8_SUB dec %rax - jne .L8_47 + jne L(8_47) ALIGN_4 -.L8_49: +L(8_49): SAVE1x8 @@ -4064,7 +4064,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $1, KK // number of values in A #endif -.L8_100: +L(8_100): #if defined(TRMMKERNEL) && !defined(LEFT) addq $8, KK // number of values in B @@ -4072,20 +4072,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. decq J // j -- - jg .L8_10 + jg L(8_10) /*************************************************************************************************/ -.L4_0: +L(4_0): movq Nmod12, J testq $4, J - je .L2_0 + je L(2_0) ALIGN_4 -.L4_10: +L(4_10): movq C, CO1 leaq (C, LDC, 4), C // c += 4 * ldc @@ -4100,11 +4100,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L4_20 + je L(4_20) ALIGN_4 -.L4_11: +L(4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4139,7 +4139,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, %rax // K / 8 cmpq $2, %rax - jl .L4_13 + jl L(4_13) KERNEL4x4_I @@ -4153,11 +4153,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 subq $2, %rax - je .L4_12a + je L(4_12a) ALIGN_5 -.L4_12: +L(4_12): KERNEL4x4_M1 KERNEL4x4_M2 @@ -4170,9 +4170,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 dec %rax - jne .L4_12 + jne L(4_12) -.L4_12a: +L(4_12a): KERNEL4x4_M1 KERNEL4x4_M2 @@ -4184,13 +4184,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - jmp .L4_16 + jmp L(4_16) -.L4_13: +L(4_13): test $1, %rax - jz .L4_14 + jz L(4_14) KERNEL4x4_I KERNEL4x4_M2 @@ -4202,32 +4202,32 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - jmp .L4_16 + jmp L(4_16) -.L4_14: +L(4_14): INIT4x4 -.L4_16: +L(4_16): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L4_19 + je L(4_19) ALIGN_4 -.L4_17: +L(4_17): KERNEL4x4_SUB dec %rax - jne .L4_17 + jne L(4_17) ALIGN_4 -.L4_19: +L(4_19): SAVE4x4 @@ -4245,26 +4245,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif decq I # i -- - jg .L4_11 + jg L(4_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L4_20: +L(4_20): // Test rest of M testq $3, M - jz .L4_100 // to next 16 lines of N + jz L(4_100) // to next 16 lines of N -.L4_30: +L(4_30): testq $2, M - jz .L4_40 + jz L(4_40) ALIGN_4 -.L4_31: +L(4_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4300,10 +4300,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT2x4 sarq $3, %rax - je .L4_36 + je L(4_36) ALIGN_4 -.L4_32: +L(4_32): KERNEL2x4_SUB KERNEL2x4_SUB @@ -4316,26 +4316,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB dec %rax - jne .L4_32 + jne L(4_32) ALIGN_4 -.L4_36: +L(4_36): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L4_39 + je L(4_39) ALIGN_4 -.L4_37: +L(4_37): KERNEL2x4_SUB dec %rax - jne .L4_37 + jne L(4_37) -.L4_39: +L(4_39): SAVE2x4 @@ -4353,13 +4353,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.L4_40: +L(4_40): testq $1, M - jz .L4_100 // to next 3 lines of N + jz L(4_100) // to next 3 lines of N ALIGN_4 -.L4_41: +L(4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4395,11 +4395,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT1x4 sarq $3,%rax - je .L4_46 + je L(4_46) ALIGN_4 -.L4_42: +L(4_42): KERNEL1x4_SUB KERNEL1x4_SUB @@ -4412,27 +4412,27 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB dec %rax - jne .L4_42 + jne L(4_42) ALIGN_4 -.L4_46: +L(4_46): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L4_49 + je L(4_49) ALIGN_4 -.L4_47: +L(4_47): KERNEL1x4_SUB dec %rax - jne .L4_47 + jne L(4_47) ALIGN_4 -.L4_49: +L(4_49): SAVE1x4 @@ -4449,7 +4449,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $1, KK // number of values in A #endif -.L4_100: +L(4_100): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK // number of values in B @@ -4465,13 +4465,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /***************************************************************************************************************/ -.L2_0: +L(2_0): movq Nmod12, J testq $2, J - je .L1_0 + je L(1_0) -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -4487,11 +4487,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4528,11 +4528,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. sarq $3, %rax // K / 8 - je .L2_16 + je L(2_16) ALIGN_5 -.L2_12: +L(2_12): KERNEL4x2_SUB KERNEL4x2_SUB @@ -4545,27 +4545,27 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB dec %rax - jne .L2_12 + jne L(2_12) -.L2_16: +L(2_16): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) ALIGN_4 -.L2_17: +L(2_17): KERNEL4x2_SUB dec %rax - jne .L2_17 + jne L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE4x2 @@ -4584,26 +4584,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $3, M - jz .L2_100 // to next 16 lines of N + jz L(2_100) // to next 16 lines of N -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4639,10 +4639,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT2x2 sarq $3, %rax - je .L2_36 + je L(2_36) ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_SUB KERNEL2x2_SUB @@ -4655,25 +4655,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB dec %rax - jne .L2_32 + jne L(2_32) -.L2_36: +L(2_36): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB dec %rax - jne .L2_37 + jne L(2_37) -.L2_39: +L(2_39): SAVE2x2 @@ -4691,11 +4691,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.L2_40: +L(2_40): testq $1, M - jz .L2_100 // to next 3 lines of N + jz L(2_100) // to next 3 lines of N -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4731,11 +4731,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT1x2 sarq $3,%rax - je .L2_46 + je L(2_46) ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_SUB KERNEL1x2_SUB @@ -4748,24 +4748,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB dec %rax - jne .L2_42 + jne L(2_42) -.L2_46: +L(2_46): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB dec %rax - jne .L2_47 + jne L(2_47) -.L2_49: +L(2_49): SAVE1x2 @@ -4783,7 +4783,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.L2_100: +L(2_100): #if defined(TRMMKERNEL) && !defined(LEFT) @@ -4796,13 +4796,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /***************************************************************************************************************/ -.L1_0: +L(1_0): movq Nmod12, J testq $1, J - je .L999 + je L(999) -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -4816,11 +4816,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $2, I // i = m / 4 - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4856,36 +4856,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT4x1 sarq $3, %rax // K / 8 - je .L1_16 + je L(1_16) ALIGN_5 -.L1_12: +L(1_12): KERNEL4x1 dec %rax - jne .L1_12 + jne L(1_12) -.L1_16: +L(1_16): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) ALIGN_4 -.L1_17: +L(1_17): KERNEL4x1_SUB dec %rax - jne .L1_17 + jne L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE4x1 @@ -4904,25 +4904,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. decq I # i -- - jg .L1_11 + jg L(1_11) /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $3, M - jz .L1_100 + jz L(1_100) -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -4958,10 +4958,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT2x1 sarq $3, %rax - je .L1_36 + je L(1_36) ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_SUB KERNEL2x1_SUB @@ -4975,24 +4975,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. dec %rax - jne .L1_32 + jne L(1_32) -.L1_36: +L(1_36): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB dec %rax - jne .L1_37 + jne L(1_37) -.L1_39: +L(1_39): SAVE2x1 @@ -5010,12 +5010,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.L1_40: +L(1_40): testq $1, M - jz .L1_100 // to next 3 lines of N + jz L(1_100) // to next 3 lines of N -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -5051,11 +5051,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT1x1 sarq $3,%rax - je .L1_46 + je L(1_46) ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_SUB KERNEL1x1_SUB @@ -5068,25 +5068,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB dec %rax - jne .L1_42 + jne L(1_42) -.L1_46: +L(1_46): movq KKK, %rax andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB dec %rax - jne .L1_47 + jne L(1_47) -.L1_49: +L(1_49): SAVE1x1 @@ -5105,7 +5105,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L1_100: +L(1_100): #if defined(TRMMKERNEL) && !defined(LEFT) @@ -5114,7 +5114,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L999: +L(999): vzeroupper diff --git a/kernel/x86_64/dgemm_kernel_4x8_sandy.S b/kernel/x86_64/dgemm_kernel_4x8_sandy.S index 926395c493..50e2a6348e 100644 --- a/kernel/x86_64/dgemm_kernel_4x8_sandy.S +++ b/kernel/x86_64/dgemm_kernel_4x8_sandy.S @@ -249,9 +249,9 @@ movq %r11, kk MOVQ bn,j; SARQ $2,j; # Rn = 4 -JLE .L0_loopE; +JLE L(0_loopE); ALIGN_5; -.L0_bodyB:; +L(0_bodyB):; #if defined(TRMMKERNEL) && defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk; @@ -265,9 +265,9 @@ LEAQ (bb, k, 1), prebb; MOVQ ba,ptrba; MOVQ bm,i; SARQ $3,i; # Rm = 8 -JLE .L1_loopE; +JLE L(1_loopE); ALIGN_5; -.L1_bodyB:; +L(1_bodyB):; #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -312,9 +312,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2,k; -JLE .L2_loopE; +JLE L(2_loopE); ALIGN_5; -.L2_bodyB:; +L(2_bodyB):; # Computing kernel //#### Unroll times 1 #### @@ -430,11 +430,11 @@ MUL_DY yvec1, yvec4, yvec6; MUL_DY yvec1, yvec5, yvec7; ADD_DY yvec10, yvec6, yvec10; ADD_DY yvec8, yvec7, yvec8; -.L2_bodyE:; +L(2_bodyE):; DECQ k; -JG .L2_bodyB; +JG L(2_bodyB); ALIGN_5 -.L2_loopE:; +L(2_loopE):; PREFETCH2 0*SIZE(prebb); ADDQ $8*SIZE, prebb; #ifndef TRMMKERNEL @@ -443,9 +443,9 @@ TEST $2, bk; MOVQ kkk, %rax; TEST $2, %rax; #endif -JLE .L3_loopE; +JLE L(3_loopE); ALIGN_5 -.L3_bodyB: +L(3_bodyB): //#### Unroll times 1 #### PREFETCH0 64*SIZE(ptrba) LD_DY 4*SIZE(ptrba), yvec1; @@ -504,7 +504,7 @@ MUL_DY yvec1, yvec5, yvec7; ADD_DY yvec10, yvec6, yvec10; ADD_DY yvec8, yvec7, yvec8; -.L3_loopE: +L(3_loopE): PREFETCH2 0*SIZE(prebb); ADDQ $8*SIZE, prebb #ifndef TRMMKERNEL @@ -513,9 +513,9 @@ TEST $1, bk; MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L4_loopE; +JLE L(4_loopE); ALIGN_5 -.L4_bodyB:; +L(4_bodyB):; //#### Unroll times 1 #### PREFETCH0 64*SIZE(ptrba) LD_DY 4*SIZE(ptrba), yvec1; @@ -543,7 +543,7 @@ MUL_DY yvec1, yvec5, yvec7; ADD_DY yvec10, yvec6, yvec10; ADD_DY yvec8, yvec7, yvec8; -.L4_loopE:; +L(4_loopE):; //#### Load Alpha #### BROAD_DY MEMALPHA,yvec7; //#### Multiply Alpha #### @@ -572,7 +572,7 @@ REVS_DY $0x0a,yvec7,yvec8,yvec8; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L4_loopEx; # Unalign part write back +JNE L(4_loopEx); # Unalign part write back ALIGN_5 //#### Writing Back #### EXTRA_DY $1,yvec15,xvec7; @@ -629,12 +629,12 @@ ADDQ $8, kk #endif ADDQ $8*SIZE,C0; ADDQ $8*SIZE,C1; -.L1_bodyE:; +L(1_bodyE):; DECQ i; -JG .L1_bodyB; -JMP .L1_loopE; +JG L(1_bodyB); +JMP L(1_loopE); ALIGN_5; -.L4_loopEx:; +L(4_loopEx):; EXTRA_DY $1, yvec15, xvec7; #ifndef TRMMKERNEL LDL_DY 0*SIZE(C0), xvec6, xvec6; @@ -760,13 +760,13 @@ ADDQ $8, kk ADDQ $8*SIZE, C0; ADDQ $8*SIZE, C1; DECQ i; -JG .L1_bodyB; +JG L(1_bodyB); ALIGN_5 -.L1_loopE:; +L(1_loopE):; TEST $4, bm; # Rm = 4 -JLE .L5_loopE; +JLE L(5_loopE); ALIGN_5 -.L5_bodyB:; +L(5_bodyB):; #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -800,9 +800,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L6_loopE; +JLE L(6_loopE); ALIGN_5; -.L6_bodyB:; +L(6_bodyB):; # Computing kernel //#### Untoll time 1 #### @@ -871,18 +871,18 @@ VPERMILP_DY $0x05, yvec2, yvec3; MUL_DY yvec1, yvec5, yvec7; ADD_DY yvec9, yvec7, yvec9; DECQ k; -JG .L6_bodyB; +JG L(6_bodyB); ALIGN_5 -.L6_loopE:; +L(6_loopE):; #ifndef TRMMKERNEL TEST $2, bk; #else MOVQ kkk, %rax; TEST $2, %rax; #endif -JLE .L7_loopE; +JLE L(7_loopE); ALIGN_5 -.L7_bodyB:; +L(7_bodyB):; //#### Untoll time 1 #### LD_DY 4*SIZE(ptrba), yvec1; MUL_DY yvec0, yvec2, yvec6; @@ -917,16 +917,16 @@ VPERMILP_DY $0x05, yvec2, yvec3; MUL_DY yvec1, yvec5, yvec7; ADD_DY yvec9, yvec7, yvec9; -.L7_loopE:; +L(7_loopE):; #ifndef TRMMKERNEL TEST $1, bk #else MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L8_loopE; +JLE L(8_loopE); ALIGN_5 -.L8_bodyB:; +L(8_bodyB):; //#### Untoll time 1 #### MUL_DY yvec0, yvec2, yvec6; ADD_DY yvec15, yvec6, yvec15; @@ -942,7 +942,7 @@ ADDQ $4*SIZE, ptrbb; MUL_DY yvec0, yvec5, yvec7; ADD_DY yvec9, yvec7, yvec9; -.L8_loopE:; +L(8_loopE):; //#### Load Alpha #### BROAD_DY MEMALPHA, yvec7; //#### Multiply Alpha #### @@ -961,7 +961,7 @@ REVS_DY $0x0a,yvec7,yvec9,yvec9; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L8_loopEx; # Unalign part write back +JNE L(8_loopEx); # Unalign part write back ALIGN_5 //#### Writing Back #### EXTRA_DY $1,yvec15,xvec7; @@ -998,9 +998,9 @@ ADDQ $4, kk #endif ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -JMP .L5_loopE; +JMP L(5_loopE); ALIGN_5 -.L8_loopEx:; +L(8_loopEx):; EXTRA_DY $1,yvec15,xvec7; EXTRA_DY $1,yvec13,xvec5; EXTRA_DY $1,yvec11,xvec3; @@ -1062,11 +1062,11 @@ ADDQ $4, kk ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -.L5_loopE:; +L(5_loopE):; TEST $2, bm; -JLE .L9_loopE; +JLE L(9_loopE); ALIGN_5 -.L9_bodyB:; +L(9_bodyB):; #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -1101,9 +1101,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L10_loopE; +JLE L(10_loopE); ALIGN_5; -.L10_bodyB:; +L(10_bodyB):; # Computing kernel //#### Unroll time 1 #### @@ -1176,18 +1176,18 @@ SHUF_DX $0x4e, xvec2, xvec4; MUL_DX xvec1, xvec5, xvec5; ADD_DX xvec5, xvec9, xvec9; DECQ k; -JG .L10_bodyB; +JG L(10_bodyB); ALIGN_5 -.L10_loopE:; +L(10_loopE):; #ifndef TRMMKERNEL TEST $2, bk #else MOVQ kkk, %rax; TEST $2, %rax; #endif -JLE .L11_loopE; +JLE L(11_loopE); ALIGN_5 -.L11_bodyB:; +L(11_bodyB):; //#### Unroll time 1 #### LD_DX 4*SIZE(ptrbb), xvec6; SHUF_DX $0x4e, xvec3, xvec5; @@ -1225,16 +1225,16 @@ SHUF_DX $0x4e, xvec2, xvec4; MUL_DX xvec1, xvec5, xvec5; ADD_DX xvec5, xvec9, xvec9; -.L11_loopE:; +L(11_loopE):; #ifndef TRMMKERNEL TEST $1, bk #else MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L12_loopE; +JLE L(12_loopE); ALIGN_5 -.L12_bodyB:; +L(12_bodyB):; SHUF_DX $0x4e, xvec3, xvec5; MUL_DX xvec0, xvec2, xvec2; ADD_DX xvec2, xvec15, xvec15; @@ -1250,7 +1250,7 @@ ADD_DX xvec4, xvec13, xvec13; MUL_DX xvec0, xvec5, xvec5; ADD_DX xvec5, xvec9, xvec9; -.L12_loopE:; +L(12_loopE):; //#### Load Alpha #### BROAD_DX MEMALPHA, xvec7; //#### Multiply Alpha #### @@ -1269,7 +1269,7 @@ REVS_DX xvec6, xvec9, xvec9; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L12_loopEx; +JNE L(12_loopEx); ALIGN_5 //#### Writing Back #### #ifndef TRMMKERNEL @@ -1294,9 +1294,9 @@ ADDQ $2, kk #endif ADDQ $2*SIZE, C0 ADDQ $2*SIZE, C1 -JMP .L9_loopE; +JMP L(9_loopE); ALIGN_5 -.L12_loopEx: +L(12_loopEx): #ifndef TRMMKERNEL LDL_DX 0*SIZE(C0), xvec14, xvec14; LDH_DX 1*SIZE(C0), xvec14, xvec14; @@ -1331,11 +1331,11 @@ ADDQ $2, kk #endif ADDQ $2*SIZE, C0; ADDQ $2*SIZE, C1; -.L9_loopE:; +L(9_loopE):; TEST $1, bm -JLE .L13_loopE; +JLE L(13_loopE); ALIGN_5 -.L13_bodyB:; +L(13_bodyB):; #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -1363,9 +1363,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L14_loopE; +JLE L(14_loopE); ALIGN_5 -.L14_bodyB:; +L(14_bodyB):; BROAD_DY 0*SIZE(ptrba), yvec0; LD_DY 0*SIZE(ptrbb), yvec2; MUL_DY yvec0, yvec2, yvec6; @@ -1388,18 +1388,18 @@ ADD_DY yvec15, yvec7, yvec15; ADDQ $4*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; DECQ k; -JG .L14_bodyB; +JG L(14_bodyB); ALIGN_5 -.L14_loopE: +L(14_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else MOVQ kkk, %rax; TEST $2, %rax; #endif -JLE .L15_loopE; +JLE L(15_loopE); ALIGN_5 -.L15_bodyB: +L(15_bodyB): BROAD_DY 0*SIZE(ptrba), yvec0; LD_DY 0*SIZE(ptrbb), yvec2; MUL_DY yvec0, yvec2, yvec6; @@ -1411,16 +1411,16 @@ MUL_DY yvec1, yvec3, yvec7; ADD_DY yvec15, yvec7, yvec15; ADDQ $2*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L15_loopE:; +L(15_loopE):; #ifndef TRMMKERNEL TEST $1, bk; #else MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L16_loopE; +JLE L(16_loopE); ALIGN_5 -.L16_bodyB:; +L(16_bodyB):; BROAD_DY 0*SIZE(ptrba), yvec0; LD_DY 0*SIZE(ptrbb), yvec2; MUL_DY yvec0, yvec2, yvec6; @@ -1428,7 +1428,7 @@ ADD_DY yvec15, yvec6, yvec15; ADDQ $1*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L16_loopE: +L(16_loopE): //#### Load Alpha #### BROAD_DY MEMALPHA, yvec7; //#### Multiply Alpha #### @@ -1459,7 +1459,7 @@ ADDQ $1, kk #endif ADDQ $1*SIZE, C0 ADDQ $1*SIZE, C1 -.L13_loopE:; +L(13_loopE):; #if defined(TRMMKERNEL)&&!defined(LEFT) ADDQ $4, kk #endif @@ -1467,15 +1467,15 @@ MOVQ bk,k; SALQ $5,k; ADDQ k,bb; LEAQ (C,ldc,4),C; -.L0_bodyE:; +L(0_bodyE):; DECQ j; -JG .L0_bodyB; +JG L(0_bodyB); ALIGN_5; -.L0_loopE:; +L(0_loopE):; TEST $2, bn; -JLE .L20_loopE; +JLE L(20_loopE); ALIGN_5; -.L20_loopB:; +L(20_loopB):; #if defined(TRMMKERNEL) && defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk @@ -1485,9 +1485,9 @@ LEAQ (C, ldc, 1), C1; MOVQ ba, ptrba; MOVQ bm, i; SARQ $3, i; # Rm = 8 -JLE .L21_loopE; +JLE L(21_loopE); ALIGN_5; -.L21_bodyB:; +L(21_bodyB):; #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -1522,9 +1522,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L211_loopE; +JLE L(211_loopE); ALIGN_5; -.L211_bodyB: +L(211_bodyB): # Computing kernel //#### Unroll time 1 #### LD_DX 0*SIZE(ptrba), xvec0; @@ -1676,18 +1676,18 @@ ADD_DX xvec6, xvec9, xvec9; MUL_DX xvec3, xvec7, xvec7; ADD_DX xvec7, xvec8, xvec8; DECQ k; -JG .L211_bodyB; +JG L(211_bodyB); ALIGN_5 -.L211_loopE: +L(211_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else MOVQ kkk, %rax; TEST $2, %rax; #endif -JLE .L212_loopE; +JLE L(212_loopE); ALIGN_5; -.L212_bodyB: +L(212_bodyB): # Computing kernel //#### Unroll time 1 #### LD_DX 0*SIZE(ptrba), xvec0; @@ -1765,16 +1765,16 @@ ADD_DX xvec6, xvec9, xvec9; MUL_DX xvec3, xvec7, xvec7; ADD_DX xvec7, xvec8, xvec8; -.L212_loopE: +L(212_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L213_loopE; +JLE L(213_loopE); ALIGN_5 -.L213_bodyB: +L(213_bodyB): //#### Unroll time 1 #### LD_DX 0*SIZE(ptrba), xvec0; LD_DX 0*SIZE(ptrbb), xvec4; @@ -1814,7 +1814,7 @@ ADD_DX xvec6, xvec9, xvec9; MUL_DX xvec3, xvec7, xvec7; ADD_DX xvec7, xvec8, xvec8; -.L213_loopE: +L(213_loopE): //#### Multiply Alpha #### BROAD_DX MEMALPHA, xvec7; MUL_DX xvec7, xvec15, xvec15; @@ -1842,7 +1842,7 @@ REVS_DX xvec6, xvec8, xvec8; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L213_loopEx; +JNE L(213_loopEx); ALIGN_5 //#### Writing Back #### #ifndef TRMMKERNEL @@ -1876,10 +1876,10 @@ ADDQ $8, kk ADDQ $8*SIZE, C0; ADDQ $8*SIZE, C1; DECQ i; -JG .L21_bodyB; -JMP .L21_loopE; +JG L(21_bodyB); +JMP L(21_loopE); ALIGN_5 -.L213_loopEx:; +L(213_loopEx):; #ifndef TRMMKERNEL LDL_DX 0*SIZE(C0), xvec0, xvec0; LDH_DX 1*SIZE(C0), xvec0, xvec0; @@ -1937,12 +1937,12 @@ ADDQ $8, kk ADDQ $8*SIZE, C0; ADDQ $8*SIZE, C1; DECQ i; -JG .L21_bodyB; -.L21_loopE:; +JG L(21_bodyB); +L(21_loopE):; TEST $4, bm; # Rm = 4 -JLE .L22_loopE; +JLE L(22_loopE); ALIGN_5; -.L22_bodyB:; +L(22_bodyB):; #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -1973,9 +1973,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L221_loopE; +JLE L(221_loopE); ALIGN_5 -.L221_bodyB:; +L(221_bodyB):; # Computing kernel //#### Unroll time 1 #### LD_DX 0*SIZE(ptrba), xvec0; @@ -2055,18 +2055,18 @@ ADD_DX xvec4, xvec11, xvec11; MUL_DX xvec1, xvec5, xvec5; ADD_DX xvec5, xvec10, xvec10; DECQ k; -JG .L221_bodyB; +JG L(221_bodyB); ALIGN_5 -.L221_loopE:; +L(221_loopE):; #ifndef TRMMKERNEL TEST $2, bk; #else MOVQ kkk, %rax; TEST $2, %rax; #endif -JLE .L222_loopE; +JLE L(222_loopE); ALIGN_5 -.L222_bodyB: +L(222_bodyB): //#### Unroll time 1 #### LD_DX 0*SIZE(ptrba), xvec0; LD_DX 0*SIZE(ptrbb), xvec4; @@ -2106,16 +2106,16 @@ ADD_DX xvec4, xvec11, xvec11; MUL_DX xvec1, xvec5, xvec5; ADD_DX xvec5, xvec10, xvec10; -.L222_loopE: +L(222_loopE): #ifndef TRMMKERNEL TEST $1, bk #else MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L223_loopE; +JLE L(223_loopE); ALIGN_5 -.L223_bodyB: +L(223_bodyB): //#### Unroll time 1 #### LD_DX 0*SIZE(ptrba), xvec0; LD_DX 0*SIZE(ptrbb), xvec4; @@ -2137,7 +2137,7 @@ ADD_DX xvec4, xvec11, xvec11; MUL_DX xvec1, xvec5, xvec5; ADD_DX xvec5, xvec10, xvec10; -.L223_loopE: +L(223_loopE): //#### Multiply Alpha #### BROAD_DX MEMALPHA, xvec7; MUL_DX xvec7, xvec15, xvec15; @@ -2155,7 +2155,7 @@ REVS_DX xvec6, xvec10, xvec10; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L223_loopEx; +JNE L(223_loopEx); ALIGN_5 //#### Writing Back #### #ifndef TRMMKERNEL @@ -2180,9 +2180,9 @@ ADDQ $4, kk #endif ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -JMP .L22_loopE; +JMP L(22_loopE); ALIGN_5 -.L223_loopEx:; +L(223_loopEx):; #ifndef TRMMKERNEL LDL_DX 0*SIZE(C0), xvec0, xvec0; LDH_DX 1*SIZE(C0), xvec0, xvec0; @@ -2219,11 +2219,11 @@ ADDQ $4, kk #endif ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -.L22_loopE:; +L(22_loopE):; TEST $2, bm; // Rm = 2 -JLE .L23_loopE; +JLE L(23_loopE); ALIGN_5; -.L23_bodyB: +L(23_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -2251,9 +2251,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L231_loopE; +JLE L(231_loopE); ALIGN_5 -.L231_bodyB: +L(231_bodyB): # Computing kernel //#### Unroll time 1 #### LD_DX 0*SIZE(ptrba), xvec0; @@ -2293,18 +2293,18 @@ MUL_DX xvec0, xvec5, xvec5; ADD_DX xvec5, xvec11, xvec11; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L231_bodyB; +JG L(231_bodyB); ALIGN_5 -.L231_loopE: +L(231_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else MOVQ kkk, %rax; TEST $2, %rax; #endif -JLE .L232_loopE; +JLE L(232_loopE); ALIGN_5 -.L232_bodyB: +L(232_bodyB): //#### Unroll time 1 #### LD_DX 0*SIZE(ptrba), xvec0; LD_DX 0*SIZE(ptrbb), xvec4; @@ -2324,16 +2324,16 @@ ADDQ $4*SIZE, ptrba; MUL_DX xvec0, xvec5, xvec5; ADD_DX xvec5, xvec11, xvec11; ADDQ $4*SIZE, ptrbb; -.L232_loopE: +L(232_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L233_loopE; +JLE L(233_loopE); ALIGN_5 -.L233_bodyB: +L(233_bodyB): //#### Unroll time 1 #### LD_DX 0*SIZE(ptrba), xvec0; LD_DX 0*SIZE(ptrbb), xvec4; @@ -2344,7 +2344,7 @@ ADDQ $2*SIZE, ptrba; MUL_DX xvec0, xvec5, xvec5; ADD_DX xvec5, xvec11, xvec11; ADDQ $2*SIZE, ptrbb; -.L233_loopE: +L(233_loopE): //#### Multiply Alpha #### BROAD_DX MEMALPHA, xvec7; MUL_DX xvec7, xvec15, xvec15; @@ -2357,7 +2357,7 @@ REVS_DX xvec6, xvec11, xvec11; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L233_loopEx; +JNE L(233_loopEx); ALIGN_5 //#### Writing Back #### #ifndef TRMMKERNEL @@ -2378,9 +2378,9 @@ ADDQ $2, kk; #endif ADDQ $2*SIZE, C0; ADDQ $2*SIZE, C1; -JMP .L23_loopE; +JMP L(23_loopE); ALIGN_5 -.L233_loopEx:; +L(233_loopEx):; #ifndef TRMMKERNEL LDL_DX 0*SIZE(C0), xvec0, xvec0; LDH_DX 1*SIZE(C0), xvec0, xvec0; @@ -2407,11 +2407,11 @@ ADDQ $2, kk; #endif ADDQ $2*SIZE, C0; ADDQ $2*SIZE, C1; -.L23_loopE: +L(23_loopE): TEST $1, bm; // Rm = 1 -JLE .L24_loopE; +JLE L(24_loopE); ALIGN_5; -.L24_bodyB: +L(24_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -2438,9 +2438,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L241_loopE; +JLE L(241_loopE); ALIGN_5 -.L241_bodyB: +L(241_bodyB): BROAD_DX 0*SIZE(ptrba), xvec0; LD_DX 0*SIZE(ptrbb), xvec2; MUL_DX xvec0, xvec2, xvec2; @@ -2463,18 +2463,18 @@ ADD_DX xvec3, xvec15, xvec15; ADDQ $4*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L241_bodyB; +JG L(241_bodyB); ALIGN_5 -.L241_loopE: +L(241_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else MOVQ kkk, %rax; TEST $2, %rax; #endif -JLE .L242_loopE; +JLE L(242_loopE); ALIGN_5 -.L242_bodyB: +L(242_bodyB): BROAD_DX 0*SIZE(ptrba), xvec0; LD_DX 0*SIZE(ptrbb), xvec2; MUL_DX xvec0, xvec2, xvec2; @@ -2486,16 +2486,16 @@ MUL_DX xvec1, xvec3, xvec3; ADD_DX xvec3, xvec15, xvec15; ADDQ $2*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L242_loopE: +L(242_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L243_loopE; +JLE L(243_loopE); ALIGN_5 -.L243_bodyB: +L(243_bodyB): BROAD_DX 0*SIZE(ptrba), xvec0; LD_DX 0*SIZE(ptrbb), xvec2; MUL_DX xvec0, xvec2, xvec2; @@ -2503,7 +2503,7 @@ ADD_DX xvec2, xvec15, xvec15; ADDQ $1*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L243_loopE: +L(243_loopE): BROAD_DX MEMALPHA, xvec7; MUL_DX xvec7, xvec15, xvec15; #ifndef TRMMKERNEL @@ -2525,7 +2525,7 @@ ADDQ $1, kk; #endif ADDQ $1*SIZE, C0; ADDQ $1*SIZE, C1; -.L24_loopE: +L(24_loopE): #if defined(TRMMKERNEL) && !defined(LEFT) ADDQ $2, kk; #endif @@ -2533,11 +2533,11 @@ MOVQ bk, k; SALQ $4, k; ADDQ k, bb; LEAQ (C, ldc, 2), C; -.L20_loopE:; +L(20_loopE):; TEST $1, bn; // Rn = 1 -JLE .L30_loopE; +JLE L(30_loopE); ALIGN_5 -.L30_bodyB: +L(30_bodyB): #if defined(TRMMKERNEL)&&defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk; @@ -2546,9 +2546,9 @@ MOVQ C, C0; MOVQ ba, ptrba; MOVQ bm, i; SARQ $3, i; -JLE .L31_loopE; +JLE L(31_loopE); ALIGN_5 -.L31_bodyB: +L(31_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -2577,9 +2577,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L311_loopE; +JLE L(311_loopE); ALIGN_5 -.L311_bodyB: +L(311_bodyB): //#### Unroll time 1 #### LD_DY 0*SIZE(ptrba), yvec0; LD_DY 4*SIZE(ptrba), yvec1; @@ -2618,18 +2618,18 @@ MUL_DY yvec5, yvec4, yvec4; ADD_DY yvec4, yvec14, yvec14; ADDQ $4*SIZE, ptrbb; DECQ k; -JG .L311_bodyB; +JG L(311_bodyB); ALIGN_5 -.L311_loopE: +L(311_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else MOVQ kkk, %rax; TEST $2, %rax; #endif -JLE .L312_loopE; +JLE L(312_loopE); ALIGN_5 -.L312_bodyB: +L(312_bodyB): //#### Unroll time 1 #### LD_DY 0*SIZE(ptrba), yvec0; LD_DY 4*SIZE(ptrba), yvec1; @@ -2650,16 +2650,16 @@ MUL_DY yvec5, yvec4, yvec4 ADD_DY yvec4, yvec14, yvec14; ADDQ $2*SIZE, ptrbb; -.L312_loopE: +L(312_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L313_loopE; +JLE L(313_loopE); ALIGN_5 -.L313_bodyB: +L(313_bodyB): //#### Unroll time 1 #### LD_DY 0*SIZE(ptrba), yvec0; LD_DY 4*SIZE(ptrba), yvec1; @@ -2671,7 +2671,7 @@ MUL_DY yvec2, yvec1, yvec1; ADD_DY yvec1, yvec14, yvec14; ADDQ $1*SIZE, ptrbb; -.L313_loopE: +L(313_loopE): //#### Multiply Alpha #### BROAD_DY MEMALPHA, yvec7; MUL_DY yvec7, yvec15, yvec15; @@ -2680,7 +2680,7 @@ MUL_DY yvec7, yvec14, yvec14; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L313_loopEx; +JNE L(313_loopEx); ALIGN_5 //#### Writing Back #### EXTRA_DY $1, yvec15, xvec13; @@ -2707,10 +2707,10 @@ ADDQ $8, kk; #endif ADDQ $8*SIZE, C0; DECQ i; -JG .L31_bodyB; -JMP .L31_loopE; +JG L(31_bodyB); +JMP L(31_loopE); ALIGN_5 -.L313_loopEx: +L(313_loopEx): EXTRA_DY $1, yvec15, xvec13; EXTRA_DY $1, yvec14, xvec12; #ifndef TRMMKERNEL @@ -2747,12 +2747,12 @@ ADDQ $8, kk; #endif ADDQ $8*SIZE, C0; DECQ i; -JG .L31_bodyB; -.L31_loopE: +JG L(31_bodyB); +L(31_loopE): TEST $4, bm -JLE .L32_loopE; +JLE L(32_loopE); ALIGN_5 -.L32_bodyB: +L(32_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -2780,9 +2780,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk #endif SARQ $2, k; -JLE .L321_loopE; +JLE L(321_loopE); ALIGN_5 -.L321_bodyB: +L(321_bodyB): LD_DY 0*SIZE(ptrba), yvec0; BROAD_DY 0*SIZE(ptrbb), yvec1; MUL_DY yvec0, yvec1, yvec1; @@ -2805,18 +2805,18 @@ ADD_DY yvec7, yvec15, yvec15; ADDQ $16*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; DECQ k; -JG .L321_bodyB; +JG L(321_bodyB); ALIGN_5 -.L321_loopE: +L(321_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else MOVQ kkk, %rax; TEST $2, %rax; #endif -JLE .L322_loopE; +JLE L(322_loopE); ALIGN_5 -.L322_bodyB: +L(322_bodyB): LD_DY 0*SIZE(ptrba), yvec0; BROAD_DY 0*SIZE(ptrbb), yvec1; MUL_DY yvec0, yvec1, yvec1; @@ -2829,16 +2829,16 @@ ADD_DY yvec3, yvec15, yvec15; ADDQ $8*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L322_loopE: +L(322_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L323_loopE; +JLE L(323_loopE); ALIGN_5 -.L323_bodyB: +L(323_bodyB): LD_DY 0*SIZE(ptrba), yvec0; BROAD_DY 0*SIZE(ptrbb), yvec1; MUL_DY yvec0, yvec1, yvec1; @@ -2846,7 +2846,7 @@ ADD_DY yvec1, yvec15, yvec15; ADDQ $4*SIZE, ptrba; ADDQ $1*SIZE, ptrbb; -.L323_loopE: +L(323_loopE): //#### Multiply Alpha #### BROAD_DY MEMALPHA, yvec7; MUL_DY yvec7, yvec15, yvec15; @@ -2854,7 +2854,7 @@ MUL_DY yvec7, yvec15, yvec15; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L323_loopEx; +JNE L(323_loopEx); ALIGN_5 //#### Writing Back #### EXTRA_DY $1, yvec15, xvec14; @@ -2875,9 +2875,9 @@ ADDQ %rax, ptrbb; ADDQ $4, kk #endif ADDQ $4*SIZE, C0; -JMP .L32_loopE; +JMP L(32_loopE); ALIGN_5 -.L323_loopEx: +L(323_loopEx): //#### Writing Back #### EXTRA_DY $1, yvec15, xvec14; #ifndef TRMMKERNEL @@ -2903,11 +2903,11 @@ ADDQ %rax, ptrbb; ADDQ $4, kk #endif ADDQ $4*SIZE, C0; -.L32_loopE: +L(32_loopE): TEST $2, bm -JLE .L33_loopE; +JLE L(33_loopE); ALIGN_5 -.L33_bodyB: +L(33_bodyB): #if !defined(TRMMKERNEL) || (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -2935,9 +2935,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L331_loopE; +JLE L(331_loopE); ALIGN_5 -.L331_bodyB: +L(331_bodyB): LD_DX 0*SIZE(ptrba), xvec0; BROAD_DX 0*SIZE(ptrbb), xvec2; MUL_DX xvec0, xvec2, xvec2; @@ -2960,18 +2960,18 @@ ADD_DX xvec7, xvec15, xvec15; ADDQ $8*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; DECQ k; -JG .L331_bodyB; +JG L(331_bodyB); ALIGN_5 -.L331_loopE: +L(331_loopE): #ifndef TRMMKERNEL TEST $2,bk; #else MOVQ kkk, %rax; TEST $2, %rax #endif -JLE .L332_loopE; +JLE L(332_loopE); ALIGN_5 -.L332_bodyB: +L(332_bodyB): LD_DX 0*SIZE(ptrba), xvec0; BROAD_DX 0*SIZE(ptrbb), xvec2; MUL_DX xvec0, xvec2, xvec2; @@ -2983,23 +2983,23 @@ MUL_DX xvec1, xvec3, xvec3; ADD_DX xvec3, xvec15, xvec15; ADDQ $4*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L332_loopE: +L(332_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L333_loopE; +JLE L(333_loopE); ALIGN_5 -.L333_bodyB: +L(333_bodyB): LD_DX 0*SIZE(ptrba), xvec0; BROAD_DX 0*SIZE(ptrbb), xvec2; MUL_DX xvec0, xvec2, xvec2; ADD_DX xvec2, xvec15, xvec15; ADDQ $2*SIZE, ptrba; ADDQ $1*SIZE, ptrbb; -.L333_loopE: +L(333_loopE): //#### Multiply Alpha #### BROAD_DX MEMALPHA, xvec7; MUL_DX xvec7, xvec15, xvec15; @@ -3021,11 +3021,11 @@ ADDQ %rax, ptrbb; addq $2, kk #endif ADDQ $2*SIZE, C0; -.L33_loopE: +L(33_loopE): TEST $1, bm -JLE .L34_loopE; +JLE L(34_loopE); ALIGN_5 -.L34_bodyB: +L(34_bodyB): #if !defined(TRMMKERNEL) || (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -3052,9 +3052,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L341_loopE; +JLE L(341_loopE); ALIGN_5 -.L341_bodyB: +L(341_bodyB): vmovsd 0*SIZE(ptrba), xvec0; vmovsd 0*SIZE(ptrbb), xvec1; vmulsd xvec0, xvec1, xvec1; @@ -3077,18 +3077,18 @@ vaddsd xvec1, xvec15, xvec15; addq $4*SIZE, ptrba; addq $4*SIZE, ptrbb; decq k; -JG .L341_bodyB; +JG L(341_bodyB); ALIGN_5 -.L341_loopE: +L(341_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else MOVQ kkk, %rax; TEST $2, %rax; #endif -JLE .L342_loopE; +JLE L(342_loopE); ALIGN_5 -.L342_bodyB: +L(342_bodyB): vmovsd 0*SIZE(ptrba), xvec0; vmovsd 0*SIZE(ptrbb), xvec1; vmulsd xvec0, xvec1, xvec1; @@ -3101,16 +3101,16 @@ vaddsd xvec1, xvec15, xvec15; addq $2*SIZE, ptrba; addq $2*SIZE, ptrbb; -.L342_loopE: +L(342_loopE): #ifndef TRMMKERNEL TEST $1, bk #else MOVQ kkk, %rax; TEST $1, %rax; #endif -JLE .L343_loopE; +JLE L(343_loopE); ALIGN_5 -.L343_bodyB: +L(343_bodyB): vmovsd 0*SIZE(ptrba), xvec0; vmovsd 0*SIZE(ptrbb), xvec1; vmulsd xvec0, xvec1, xvec1; @@ -3118,7 +3118,7 @@ vaddsd xvec1, xvec15, xvec15; addq $1*SIZE, ptrba; addq $1*SIZE, ptrbb; -.L343_loopE: +L(343_loopE): //#### Writing Back #### vmovsd MEMALPHA, xvec7; vmulsd xvec7, xvec15, xvec15; @@ -3138,13 +3138,13 @@ ADDQ %rax, ptrbb; addq $1, kk #endif addq $1*SIZE, C0; -.L34_loopE: +L(34_loopE): MOVQ bk, k SALQ $3, k; ADDQ k, bb; LEAQ (C, ldc, 1), C; -.L30_loopE: +L(30_loopE): movq 0(%rsp), %rbx; movq 8(%rsp), %rbp; movq 16(%rsp), %r12; diff --git a/kernel/x86_64/dgemm_kernel_8x2_bulldozer.S b/kernel/x86_64/dgemm_kernel_8x2_bulldozer.S index c353a59136..d534f6fd2e 100644 --- a/kernel/x86_64/dgemm_kernel_8x2_bulldozer.S +++ b/kernel/x86_64/dgemm_kernel_8x2_bulldozer.S @@ -922,13 +922,13 @@ STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -950,10 +950,10 @@ movq Ndiv6, J cmpq $0, J - je .L2_0 + je L(2_0) ALIGN_4 -.L6_01: +L(6_01): // copy to sub buffer movq K, %rax salq $1,%rax // K * 2 @@ -962,10 +962,10 @@ leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L6_02a + jz L(6_02a) ALIGN_4 -.L6_02: +L(6_02): prefetcht0 512(BO1) prefetcht0 512(BO2) prefetchw 512(BO) @@ -989,16 +989,16 @@ addq $8*SIZE,BO2 addq $12*SIZE,BO decq %rax - jnz .L6_02 + jnz L(6_02) -.L6_02a: +L(6_02a): movq K, %rax andq $3, %rax // K % 4 - jz .L6_02c + jz L(6_02c) ALIGN_4 -.L6_02b: +L(6_02b): vmovups (BO1), %xmm0 vmovsd (BO2), %xmm1 @@ -1008,9 +1008,9 @@ addq $2*SIZE,BO2 addq $3*SIZE,BO decq %rax - jnz .L6_02b + jnz L(6_02b) -.L6_02c: +L(6_02c): movq K, %rax salq $1,%rax // K * 2 @@ -1019,11 +1019,11 @@ leaq BUFFER2, BO // second buffer to BO movq K, %rax sarq $2, %rax // k / 4 - jz .L6_03a + jz L(6_03a) ALIGN_4 -.L6_03: +L(6_03): prefetcht0 512(BO2) prefetchw 512(BO) @@ -1047,17 +1047,17 @@ addq $8*SIZE,BO2 addq $12*SIZE,BO decq %rax - jnz .L6_03 + jnz L(6_03) -.L6_03a: +L(6_03a): movq K, %rax andq $3, %rax // K % 4 - jz .L6_03c + jz L(6_03c) ALIGN_4 -.L6_03b: +L(6_03b): vmovsd 1*SIZE(BO1), %xmm0 vmovups (BO2), %xmm1 @@ -1067,14 +1067,14 @@ addq $2*SIZE,BO2 addq $3*SIZE,BO decq %rax - jnz .L6_03b + jnz L(6_03b) -.L6_03c: +L(6_03c): movq BO2, B // next offset of B -.L6_10: +L(6_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -1085,11 +1085,11 @@ movq M, I sarq $3, I // i = (m >> 3) - je .L6_20 + je L(6_20) ALIGN_4 -.L6_11: +L(6_11): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1098,7 +1098,7 @@ movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L6_16 + je L(6_16) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1109,7 +1109,7 @@ negq %rax ALIGN_4 -.L6_12: +L(6_12): prefetcht0 B_PR1(BO,BI,8) KERNEL8x3_1(xxx) @@ -1124,7 +1124,7 @@ KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L6_16 + je L(6_16) prefetcht0 B_PR1(BO,BI,8) KERNEL8x3_1(xxx) @@ -1139,16 +1139,16 @@ KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L6_16 + je L(6_16) - jmp .L6_12 + jmp L(6_12) ALIGN_4 -.L6_16: +L(6_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_19 + je L(6_19) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1160,16 +1160,16 @@ negq %rax ALIGN_4 -.L6_17: +L(6_17): KERNEL8x3_SUB(xxx) addq $3, BI addq $8, %rax - jl .L6_17 + jl L(6_17) ALIGN_4 -.L6_19: +L(6_19): vmovddup ALPHA, %xmm0 @@ -1207,24 +1207,24 @@ addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L6_11 + jg L(6_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L6_20: +L(6_20): // Test rest of M testq $7, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N testq $4, M - jz .L6_30 + jz L(6_30) ALIGN_4 -.L6_21: +L(6_21): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1233,7 +1233,7 @@ movq K, %rax andq $-8, %rax - je .L6_26 + je L(6_26) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1244,7 +1244,7 @@ negq %rax ALIGN_4 -.L6_22: +L(6_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x3_1(xxx) @@ -1259,7 +1259,7 @@ KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L6_26 + je L(6_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x3_1(xxx) @@ -1274,16 +1274,16 @@ KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L6_26 + je L(6_26) - jmp .L6_22 + jmp L(6_22) ALIGN_4 -.L6_26: +L(6_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_29 + je L(6_29) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1295,16 +1295,16 @@ negq %rax ALIGN_4 -.L6_27: +L(6_27): KERNEL4x3_SUB(xxx) addq $3, BI addq $4, %rax - jl .L6_27 + jl L(6_27) ALIGN_4 -.L6_29: +L(6_29): vmovddup ALPHA, %xmm0 @@ -1333,13 +1333,13 @@ ALIGN_4 -.L6_30: +L(6_30): testq $2, M - jz .L6_40 + jz L(6_40) ALIGN_4 -.L6_31: +L(6_31): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1349,7 +1349,7 @@ movq K, %rax andq $-8, %rax - je .L6_36 + je L(6_36) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1360,7 +1360,7 @@ negq %rax ALIGN_4 -.L6_32: +L(6_32): prefetcht0 B_PR1(BO,BI,8) KERNEL2x3_1(xxx) @@ -1375,7 +1375,7 @@ KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L6_36 + je L(6_36) prefetcht0 B_PR1(BO,BI,8) KERNEL2x3_1(xxx) @@ -1390,16 +1390,16 @@ KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L6_36 + je L(6_36) - jmp .L6_32 + jmp L(6_32) ALIGN_4 -.L6_36: +L(6_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_39 + je L(6_39) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1411,16 +1411,16 @@ negq %rax ALIGN_4 -.L6_37: +L(6_37): KERNEL2x3_SUB(xxx) addq $3, BI addq $2, %rax - jl .L6_37 + jl L(6_37) ALIGN_4 -.L6_39: +L(6_39): vmovddup ALPHA, %xmm0 @@ -1438,13 +1438,13 @@ addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L6_40: +L(6_40): testq $1, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N ALIGN_4 -.L6_41: +L(6_41): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1453,7 +1453,7 @@ movq K, %rax andq $-8, %rax - je .L6_46 + je L(6_46) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1463,7 +1463,7 @@ negq %rax ALIGN_4 -.L6_42: +L(6_42): prefetcht0 B_PR1(BO,BI,8) KERNEL1x3_1(xxx) @@ -1478,7 +1478,7 @@ KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L6_46 + je L(6_46) prefetcht0 B_PR1(BO,BI,8) KERNEL1x3_1(xxx) @@ -1493,16 +1493,16 @@ KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L6_46 + je L(6_46) - jmp .L6_42 + jmp L(6_42) ALIGN_4 -.L6_46: +L(6_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_49 + je L(6_49) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1513,16 +1513,16 @@ negq %rax ALIGN_4 -.L6_47: +L(6_47): KERNEL1x3_SUB(xxx) addq $3, BI addq $1, %rax - jl .L6_47 + jl L(6_47) ALIGN_4 -.L6_49: +L(6_49): vmovddup ALPHA, %xmm0 @@ -1544,7 +1544,7 @@ /***************************************************************************************************************/ -.L7_10: +L(7_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -1555,10 +1555,10 @@ movq M, I sarq $3, I // i = (m >> 3) - je .L7_20 + je L(7_20) ALIGN_4 -.L7_11: +L(7_11): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -1569,7 +1569,7 @@ andq $-8, %rax - je .L7_16 + je L(7_16) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1581,7 +1581,7 @@ ALIGN_4 -.L7_12: +L(7_12): prefetcht0 B_PR1(BO,BI,8) KERNEL8x3_1(xxx) @@ -1596,7 +1596,7 @@ KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L7_16 + je L(7_16) prefetcht0 B_PR1(BO,BI,8) KERNEL8x3_1(xxx) @@ -1611,16 +1611,16 @@ KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L7_16 + je L(7_16) - jmp .L7_12 + jmp L(7_12) ALIGN_4 -.L7_16: +L(7_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_19 + je L(7_19) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1632,16 +1632,16 @@ negq %rax ALIGN_4 -.L7_17: +L(7_17): KERNEL8x3_SUB(xxx) addq $3, BI addq $8, %rax - jl .L7_17 + jl L(7_17) ALIGN_4 -.L7_19: +L(7_19): vmovddup ALPHA, %xmm0 @@ -1681,21 +1681,21 @@ addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L7_11 + jg L(7_11) ALIGN_4 -.L7_20: +L(7_20): // Test rest of M testq $7, M - jz .L7_60 // to next 6 lines of N + jz L(7_60) // to next 6 lines of N testq $4, M - jz .L7_30 + jz L(7_30) ALIGN_4 -.L7_21: +L(7_21): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -1705,7 +1705,7 @@ movq K, %rax andq $-8, %rax - je .L7_26 + je L(7_26) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1716,7 +1716,7 @@ negq %rax ALIGN_4 -.L7_22: +L(7_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x3_1(xxx) @@ -1731,7 +1731,7 @@ KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L7_26 + je L(7_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x3_1(xxx) @@ -1746,16 +1746,16 @@ KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L7_26 + je L(7_26) - jmp .L7_22 + jmp L(7_22) ALIGN_4 -.L7_26: +L(7_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_29 + je L(7_29) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1767,16 +1767,16 @@ negq %rax ALIGN_4 -.L7_27: +L(7_27): KERNEL4x3_SUB(xxx) addq $3, BI addq $4, %rax - jl .L7_27 + jl L(7_27) ALIGN_4 -.L7_29: +L(7_29): vmovddup ALPHA, %xmm0 @@ -1805,13 +1805,13 @@ ALIGN_4 -.L7_30: +L(7_30): testq $2, M - jz .L7_40 + jz L(7_40) ALIGN_4 -.L7_31: +L(7_31): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -1820,7 +1820,7 @@ movq K, %rax andq $-8, %rax - je .L7_36 + je L(7_36) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1831,7 +1831,7 @@ negq %rax ALIGN_4 -.L7_32: +L(7_32): prefetcht0 B_PR1(BO,BI,8) KERNEL2x3_1(xxx) @@ -1846,7 +1846,7 @@ KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L7_36 + je L(7_36) prefetcht0 B_PR1(BO,BI,8) KERNEL2x3_1(xxx) @@ -1861,16 +1861,16 @@ KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L7_36 + je L(7_36) - jmp .L7_32 + jmp L(7_32) ALIGN_4 -.L7_36: +L(7_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_39 + je L(7_39) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1882,16 +1882,16 @@ negq %rax ALIGN_4 -.L7_37: +L(7_37): KERNEL2x3_SUB(xxx) addq $3, BI addq $2, %rax - jl .L7_37 + jl L(7_37) ALIGN_4 -.L7_39: +L(7_39): vmovddup ALPHA, %xmm0 @@ -1913,13 +1913,13 @@ -.L7_40: +L(7_40): testq $1, M - jz .L7_60 // to next 6 lines of N + jz L(7_60) // to next 6 lines of N ALIGN_4 -.L7_41: +L(7_41): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -1929,7 +1929,7 @@ andq $-8, %rax - je .L7_46 + je L(7_46) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1939,7 +1939,7 @@ negq %rax ALIGN_4 -.L7_42: +L(7_42): prefetcht0 B_PR1(BO,BI,8) KERNEL1x3_1(xxx) @@ -1954,7 +1954,7 @@ KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L7_46 + je L(7_46) prefetcht0 B_PR1(BO,BI,8) KERNEL1x3_1(xxx) @@ -1969,16 +1969,16 @@ KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L7_46 + je L(7_46) - jmp .L7_42 + jmp L(7_42) ALIGN_4 -.L7_46: +L(7_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_49 + je L(7_49) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1989,16 +1989,16 @@ negq %rax ALIGN_4 -.L7_47: +L(7_47): KERNEL1x3_SUB(xxx) addq $3, BI addq $1, %rax - jl .L7_47 + jl L(7_47) ALIGN_4 -.L7_49: +L(7_49): vmovddup ALPHA, %xmm0 @@ -2015,15 +2015,15 @@ addq $1 * SIZE, CO1 # coffset += 1 -.L7_60: +L(7_60): decq J // j -- - jg .L6_01 + jg L(6_01) -.L2_0: +L(2_0): cmpq $0, Nmod6 // N % 6 == 0 - je .L999 + je L(999) /************************************************************************************************ * Loop for Nmod6 / 2 > 0 @@ -2031,30 +2031,30 @@ movq Nmod6, J sarq $1, J // j = j / 2 - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -2064,11 +2064,11 @@ movq M, I sarq $3, I // i = (m >> 3) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2078,7 +2078,7 @@ movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2089,7 +2089,7 @@ negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1(xxx) @@ -2103,7 +2103,7 @@ KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1(xxx) @@ -2117,16 +2117,16 @@ KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2138,16 +2138,16 @@ negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL8x2_SUB(xxx) addq $2, BI addq $8, %rax - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vmovddup ALPHA, %xmm0 @@ -2174,24 +2174,24 @@ addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $7, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2200,7 +2200,7 @@ movq K, %rax andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2211,7 +2211,7 @@ negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1(xxx) @@ -2225,7 +2225,7 @@ KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1(xxx) @@ -2239,16 +2239,16 @@ KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2260,16 +2260,16 @@ negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB(xxx) addq $2, BI addq $4, %rax - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): vmovddup ALPHA, %xmm0 @@ -2289,13 +2289,13 @@ ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2304,7 +2304,7 @@ movq K, %rax andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2315,7 +2315,7 @@ negq %rax ALIGN_4 -.L2_32: +L(2_32): prefetcht0 B_PR1(BO,BI,8) KERNEL2x2_1(xxx) @@ -2329,7 +2329,7 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) prefetcht0 B_PR1(BO,BI,8) KERNEL2x2_1(xxx) @@ -2343,16 +2343,16 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2364,16 +2364,16 @@ negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB(xxx) addq $2, BI addq $2, %rax - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): vmovddup ALPHA, %xmm0 @@ -2387,13 +2387,13 @@ ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2402,7 +2402,7 @@ movq K, %rax andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2412,7 +2412,7 @@ negq %rax ALIGN_4 -.L2_42: +L(2_42): prefetcht0 B_PR1(BO,BI,8) KERNEL1x2_1(xxx) @@ -2426,7 +2426,7 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) prefetcht0 B_PR1(BO,BI,8) KERNEL1x2_1(xxx) @@ -2440,16 +2440,16 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2460,16 +2460,16 @@ negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) addq $2, BI addq $1, %rax - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vmovddup ALPHA, %xmm0 @@ -2485,14 +2485,14 @@ -.L2_60: +L(2_60): decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -2500,30 +2500,30 @@ movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -2532,11 +2532,11 @@ movq M, I sarq $3, I // i = (m >> 3) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -2545,7 +2545,7 @@ movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -2555,7 +2555,7 @@ negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1(xxx) @@ -2568,7 +2568,7 @@ KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_16 + je L(1_16) prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1(xxx) @@ -2581,16 +2581,16 @@ KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -2601,16 +2601,16 @@ negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL8x1_SUB(xxx) addq $1, BI addq $8, %rax - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vmovddup ALPHA, %xmm0 @@ -2626,24 +2626,24 @@ addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $7, M - jz .L999 + jz L(999) testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -2653,7 +2653,7 @@ movq K, %rax andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -2663,7 +2663,7 @@ negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1(xxx) @@ -2676,7 +2676,7 @@ KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1(xxx) @@ -2689,16 +2689,16 @@ KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -2709,16 +2709,16 @@ negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB(xxx) addq $1, BI addq $4, %rax - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): vmovddup ALPHA, %xmm0 @@ -2732,13 +2732,13 @@ ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -2748,7 +2748,7 @@ movq K, %rax andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -2758,7 +2758,7 @@ negq %rax ALIGN_4 -.L1_32: +L(1_32): prefetcht0 B_PR1(BO,BI,8) KERNEL2x1_1(xxx) @@ -2771,7 +2771,7 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) KERNEL2x1_1(xxx) KERNEL2x1_2(xxx) @@ -2783,16 +2783,16 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -2803,16 +2803,16 @@ negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB(xxx) addq $1, BI addq $2, %rax - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): vmovddup ALPHA, %xmm0 @@ -2824,13 +2824,13 @@ ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -2839,7 +2839,7 @@ movq K, %rax andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, 8), AO @@ -2848,7 +2848,7 @@ negq %rax ALIGN_4 -.L1_42: +L(1_42): prefetcht0 B_PR1(BO,BI,8) KERNEL1x1_1(xxx) @@ -2861,7 +2861,7 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) prefetcht0 B_PR1(BO,BI,8) KERNEL1x1_1(xxx) @@ -2874,16 +2874,16 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -2893,16 +2893,16 @@ negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) addq $1, BI addq $1, %rax - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vmovddup ALPHA, %xmm0 @@ -2914,7 +2914,7 @@ ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp @@ -3002,13 +3002,13 @@ STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -3037,32 +3037,32 @@ movq Ndiv6, J cmpq $0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_0: +L(2_0): -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -3076,11 +3076,11 @@ movq M, I sarq $3, I // i = (m >> 3) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3118,7 +3118,7 @@ andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3129,7 +3129,7 @@ negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1(xxx) @@ -3143,7 +3143,7 @@ KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI,8) KERNEL8x2_1(xxx) @@ -3157,12 +3157,12 @@ KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3170,7 +3170,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3182,16 +3182,16 @@ negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL8x2_SUB(xxx) addq $2, BI addq $8, %rax - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vmovddup ALPHA, %xmm0 @@ -3249,24 +3249,24 @@ addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $7, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3303,7 +3303,7 @@ andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3314,7 +3314,7 @@ negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1(xxx) @@ -3328,7 +3328,7 @@ KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x2_1(xxx) @@ -3342,12 +3342,12 @@ KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -3355,7 +3355,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3367,16 +3367,16 @@ negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB(xxx) addq $2, BI addq $4, %rax - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): vmovddup ALPHA, %xmm0 @@ -3424,13 +3424,13 @@ ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3468,7 +3468,7 @@ andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3479,7 +3479,7 @@ negq %rax ALIGN_4 -.L2_32: +L(2_32): prefetcht0 B_PR1(BO,BI,8) KERNEL2x2_1(xxx) @@ -3493,7 +3493,7 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) prefetcht0 B_PR1(BO,BI,8) KERNEL2x2_1(xxx) @@ -3507,12 +3507,12 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -3520,7 +3520,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3532,16 +3532,16 @@ negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB(xxx) addq $2, BI addq $2, %rax - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): vmovddup ALPHA, %xmm0 @@ -3579,13 +3579,13 @@ ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3620,7 +3620,7 @@ #endif andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3630,7 +3630,7 @@ negq %rax ALIGN_4 -.L2_42: +L(2_42): prefetcht0 B_PR1(BO,BI,8) KERNEL1x2_1(xxx) @@ -3644,7 +3644,7 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) prefetcht0 B_PR1(BO,BI,8) KERNEL1x2_1(xxx) @@ -3658,12 +3658,12 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -3671,7 +3671,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3682,16 +3682,16 @@ negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) addq $2, BI addq $1, %rax - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vmovddup ALPHA, %xmm0 @@ -3730,17 +3730,17 @@ -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -3748,30 +3748,30 @@ movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -3785,11 +3785,11 @@ movq M, I sarq $3, I // i = (m >> 3) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3825,7 +3825,7 @@ #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -3835,7 +3835,7 @@ negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1(xxx) @@ -3848,7 +3848,7 @@ KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_16 + je L(1_16) prefetcht0 B_PR1(BO,BI,8) KERNEL8x1_1(xxx) @@ -3861,12 +3861,12 @@ KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3874,7 +3874,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -3885,16 +3885,16 @@ negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL8x1_SUB(xxx) addq $1, BI addq $8, %rax - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vmovddup ALPHA, %xmm0 @@ -3934,24 +3934,24 @@ #endif addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $7, M - jz .L999 + jz L(999) testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3988,7 +3988,7 @@ andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -3998,7 +3998,7 @@ negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1(xxx) @@ -4011,7 +4011,7 @@ KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) prefetcht0 B_PR1(BO,BI,8) KERNEL4x1_1(xxx) @@ -4024,12 +4024,12 @@ KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4037,7 +4037,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -4048,16 +4048,16 @@ negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB(xxx) addq $1, BI addq $4, %rax - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): vmovddup ALPHA, %xmm0 @@ -4094,13 +4094,13 @@ ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4137,7 +4137,7 @@ andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -4147,7 +4147,7 @@ negq %rax ALIGN_4 -.L1_32: +L(1_32): prefetcht0 B_PR1(BO,BI,8) KERNEL2x1_1(xxx) @@ -4160,7 +4160,7 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) KERNEL2x1_1(xxx) KERNEL2x1_2(xxx) @@ -4172,12 +4172,12 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -4185,7 +4185,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -4196,16 +4196,16 @@ negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB(xxx) addq $1, BI addq $2, %rax - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): vmovddup ALPHA, %xmm0 @@ -4238,13 +4238,13 @@ ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4279,7 +4279,7 @@ #endif andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, 8), AO @@ -4288,7 +4288,7 @@ negq %rax ALIGN_4 -.L1_42: +L(1_42): prefetcht0 B_PR1(BO,BI,8) KERNEL1x1_1(xxx) @@ -4301,7 +4301,7 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) prefetcht0 B_PR1(BO,BI,8) KERNEL1x1_1(xxx) @@ -4314,12 +4314,12 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -4327,7 +4327,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -4337,16 +4337,16 @@ negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) addq $1, BI addq $1, %rax - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vmovddup ALPHA, %xmm0 @@ -4379,7 +4379,7 @@ ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/dgemm_kernel_8x2_piledriver.S b/kernel/x86_64/dgemm_kernel_8x2_piledriver.S index 48eb1bcbe1..b403895ef7 100644 --- a/kernel/x86_64/dgemm_kernel_8x2_piledriver.S +++ b/kernel/x86_64/dgemm_kernel_8x2_piledriver.S @@ -1105,13 +1105,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -1130,10 +1130,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv6, J cmpq $0, J - je .L2_0 + je L(2_0) ALIGN_4 -.L6_01: +L(6_01): // copy to sub buffer movq K, %rax salq $1,%rax // K * 2 @@ -1142,10 +1142,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L6_02a + jz L(6_02a) ALIGN_4 -.L6_02: +L(6_02): prefetcht0 B_PR1(BO1) prefetcht0 B_PR1(BO2) prefetchw B_PR1(BO) @@ -1169,16 +1169,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 8*SIZE,BO2 addq $ 12*SIZE,BO decq %rax - jnz .L6_02 + jnz L(6_02) -.L6_02a: +L(6_02a): movq K, %rax andq $3, %rax // K % 4 - jz .L6_02c + jz L(6_02c) ALIGN_4 -.L6_02b: +L(6_02b): vmovups (BO1), %xmm0 vmovsd (BO2), %xmm1 @@ -1188,9 +1188,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 2*SIZE,BO2 addq $ 3*SIZE,BO decq %rax - jnz .L6_02b + jnz L(6_02b) -.L6_02c: +L(6_02c): movq K, %rax salq $1,%rax // K * 2 @@ -1199,11 +1199,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. leaq BUFFER2, BO // second buffer to BO movq K, %rax sarq $2, %rax // k / 4 - jz .L6_03a + jz L(6_03a) ALIGN_4 -.L6_03: +L(6_03): prefetcht0 B_PR1(BO2) prefetchw B_PR1(BO) @@ -1227,17 +1227,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 8*SIZE,BO2 addq $ 12*SIZE,BO decq %rax - jnz .L6_03 + jnz L(6_03) -.L6_03a: +L(6_03a): movq K, %rax andq $3, %rax // K % 4 - jz .L6_03c + jz L(6_03c) ALIGN_4 -.L6_03b: +L(6_03b): vmovsd 1*SIZE(BO1), %xmm0 vmovups (BO2), %xmm1 @@ -1247,14 +1247,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 2*SIZE,BO2 addq $ 3*SIZE,BO decq %rax - jnz .L6_03b + jnz L(6_03b) -.L6_03c: +L(6_03c): movq BO2, B // next offset of B -.L6_10: +L(6_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -1265,18 +1265,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $3, I // i = (m >> 3) - je .L6_20 + je L(6_20) ALIGN_4 -.L6_11: +L(6_11): leaq BUFFER1, BO // first buffer to BO addq $12 * SIZE, BO movq K, %rax sarq $3, %rax // K / 8 cmpq $3, %rax - jl .L6_13 + jl L(6_13) prefetcht0 B_PR1(BO) prefetcht0 B_PR1+64(BO) @@ -1294,7 +1294,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_5 -.L6_12: +L(6_12): prefetcht0 B_PR1-24(BO) prefetcht0 B_PR1+40(BO) @@ -1309,9 +1309,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_M8 dec %rax - jne .L6_12 + jne L(6_12) -.L6_12_E: +L(6_12_E): prefetcht0 B_PR1(BO) prefetcht0 B_PR1+64(BO) @@ -1324,12 +1324,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_M7 KERNEL8x3_E - jmp .L6_16 + jmp L(6_16) -.L6_13: +L(6_13): test $2, %rax - jz .L6_14 + jz L(6_14) KERNEL8x3_INIT KERNEL8x3_M2 @@ -1349,13 +1349,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_M7 KERNEL8x3_E - jmp .L6_16 + jmp L(6_16) -.L6_14: +L(6_14): test $1, %rax - jz .L6_15 + jz L(6_15) KERNEL8x3_INIT KERNEL8x3_M2 @@ -1367,50 +1367,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_E - jmp .L6_16 + jmp L(6_16) -.L6_15: +L(6_15): INIT8x3 -.L6_16: +L(6_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_19 + je L(6_19) ALIGN_4 -.L6_17: +L(6_17): KERNEL8x3_SUBN dec %rax - jne .L6_17 + jne L(6_17) ALIGN_4 -.L6_19: +L(6_19): SAVE8x3 decq I # i -- - jg .L6_11 + jg L(6_11) /************************************************************************** * Rest of M ***************************************************************************/ -.L6_20: +L(6_20): // Test rest of M testq $7, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N testq $4, M - jz .L6_30 + jz L(6_30) ALIGN_4 -.L6_21: +L(6_21): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1419,7 +1419,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_26 + je L(6_26) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1430,7 +1430,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_22: +L(6_22): KERNEL4x3_1(xxx) KERNEL4x3_2(xxx) @@ -1442,7 +1442,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L6_26 + je L(6_26) KERNEL4x3_1(xxx) KERNEL4x3_2(xxx) @@ -1454,16 +1454,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L6_26 + je L(6_26) - jmp .L6_22 + jmp L(6_22) ALIGN_4 -.L6_26: +L(6_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_29 + je L(6_29) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1475,16 +1475,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_27: +L(6_27): KERNEL4x3_SUB(xxx) addq $3, BI addq $4, %rax - jl .L6_27 + jl L(6_27) ALIGN_4 -.L6_29: +L(6_29): vmovddup ALPHA, %xmm0 @@ -1513,13 +1513,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L6_30: +L(6_30): testq $2, M - jz .L6_40 + jz L(6_40) ALIGN_4 -.L6_31: +L(6_31): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1529,7 +1529,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_36 + je L(6_36) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1540,7 +1540,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_32: +L(6_32): KERNEL2x3_1(xxx) KERNEL2x3_2(xxx) @@ -1552,7 +1552,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L6_36 + je L(6_36) KERNEL2x3_1(xxx) KERNEL2x3_2(xxx) @@ -1564,16 +1564,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L6_36 + je L(6_36) - jmp .L6_32 + jmp L(6_32) ALIGN_4 -.L6_36: +L(6_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_39 + je L(6_39) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1585,16 +1585,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_37: +L(6_37): KERNEL2x3_SUB(xxx) addq $3, BI addq $2, %rax - jl .L6_37 + jl L(6_37) ALIGN_4 -.L6_39: +L(6_39): vmovddup ALPHA, %xmm0 @@ -1612,13 +1612,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L6_40: +L(6_40): testq $1, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N ALIGN_4 -.L6_41: +L(6_41): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1627,7 +1627,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_46 + je L(6_46) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1637,7 +1637,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_42: +L(6_42): KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -1649,7 +1649,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L6_46 + je L(6_46) KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -1661,16 +1661,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L6_46 + je L(6_46) - jmp .L6_42 + jmp L(6_42) ALIGN_4 -.L6_46: +L(6_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_49 + je L(6_49) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1681,16 +1681,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_47: +L(6_47): KERNEL1x3_SUB(xxx) addq $3, BI addq $1, %rax - jl .L6_47 + jl L(6_47) ALIGN_4 -.L6_49: +L(6_49): vmovddup ALPHA, %xmm0 @@ -1712,7 +1712,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /***************************************************************************************************************/ -.L7_10: +L(7_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -1723,17 +1723,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $3, I // i = (m >> 3) - je .L7_20 + je L(7_20) ALIGN_4 -.L7_11: +L(7_11): leaq BUFFER2, BO // first buffer to BO addq $12 * SIZE, BO movq K, %rax sarq $3, %rax // K / 8 cmpq $3, %rax - jl .L7_13 + jl L(7_13) prefetcht0 B_PR1(BO) prefetcht0 B_PR1+64(BO) @@ -1751,7 +1751,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_5 -.L7_12: +L(7_12): prefetcht0 B_PR1-24(BO) prefetcht0 B_PR1+40(BO) @@ -1766,9 +1766,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_M8 dec %rax - jne .L7_12 + jne L(7_12) -.L7_12_E: +L(7_12_E): prefetcht0 B_PR1(BO) prefetcht0 B_PR1+64(BO) @@ -1781,14 +1781,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_M7 KERNEL8x3_E - jmp .L7_16 + jmp L(7_16) -.L7_13: +L(7_13): test $2, %rax - jz .L7_14 + jz L(7_14) KERNEL8x3_INIT KERNEL8x3_M2 @@ -1808,13 +1808,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_M7 KERNEL8x3_E - jmp .L7_16 + jmp L(7_16) -.L7_14: +L(7_14): test $1, %rax - jz .L7_15 + jz L(7_15) KERNEL8x3_INIT KERNEL8x3_M2 @@ -1825,51 +1825,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_M7 KERNEL8x3_E - jmp .L7_16 + jmp L(7_16) -.L7_15: +L(7_15): INIT8x3 -.L7_16: +L(7_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_19 + je L(7_19) ALIGN_4 -.L7_17: +L(7_17): KERNEL8x3_SUBN dec %rax - jne .L7_17 + jne L(7_17) ALIGN_4 -.L7_19: +L(7_19): SAVE8x3 decq I # i -- - jg .L7_11 + jg L(7_11) ALIGN_4 -.L7_20: +L(7_20): // Test rest of M testq $7, M - jz .L7_60 // to next 6 lines of N + jz L(7_60) // to next 6 lines of N testq $4, M - jz .L7_30 + jz L(7_30) ALIGN_4 -.L7_21: +L(7_21): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -1879,7 +1879,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_26 + je L(7_26) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1890,7 +1890,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_22: +L(7_22): KERNEL4x3_1(xxx) KERNEL4x3_2(xxx) @@ -1902,7 +1902,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L7_26 + je L(7_26) KERNEL4x3_1(xxx) KERNEL4x3_2(xxx) @@ -1914,16 +1914,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L7_26 + je L(7_26) - jmp .L7_22 + jmp L(7_22) ALIGN_4 -.L7_26: +L(7_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_29 + je L(7_29) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1935,16 +1935,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_27: +L(7_27): KERNEL4x3_SUB(xxx) addq $3, BI addq $4, %rax - jl .L7_27 + jl L(7_27) ALIGN_4 -.L7_29: +L(7_29): vmovddup ALPHA, %xmm0 @@ -1973,13 +1973,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L7_30: +L(7_30): testq $2, M - jz .L7_40 + jz L(7_40) ALIGN_4 -.L7_31: +L(7_31): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -1988,7 +1988,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_36 + je L(7_36) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1999,7 +1999,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_32: +L(7_32): KERNEL2x3_1(xxx) KERNEL2x3_2(xxx) @@ -2011,7 +2011,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L7_36 + je L(7_36) KERNEL2x3_1(xxx) KERNEL2x3_2(xxx) @@ -2023,16 +2023,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L7_36 + je L(7_36) - jmp .L7_32 + jmp L(7_32) ALIGN_4 -.L7_36: +L(7_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_39 + je L(7_39) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2044,16 +2044,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_37: +L(7_37): KERNEL2x3_SUB(xxx) addq $3, BI addq $2, %rax - jl .L7_37 + jl L(7_37) ALIGN_4 -.L7_39: +L(7_39): vmovddup ALPHA, %xmm0 @@ -2075,13 +2075,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L7_40: +L(7_40): testq $1, M - jz .L7_60 // to next 6 lines of N + jz L(7_60) // to next 6 lines of N ALIGN_4 -.L7_41: +L(7_41): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -2091,7 +2091,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L7_46 + je L(7_46) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2101,7 +2101,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_42: +L(7_42): KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -2113,7 +2113,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L7_46 + je L(7_46) KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -2125,16 +2125,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L7_46 + je L(7_46) - jmp .L7_42 + jmp L(7_42) ALIGN_4 -.L7_46: +L(7_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_49 + je L(7_49) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2145,16 +2145,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_47: +L(7_47): KERNEL1x3_SUB(xxx) addq $3, BI addq $1, %rax - jl .L7_47 + jl L(7_47) ALIGN_4 -.L7_49: +L(7_49): vmovddup ALPHA, %xmm0 @@ -2171,15 +2171,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $1 * SIZE, CO1 # coffset += 1 -.L7_60: +L(7_60): decq J // j -- - jg .L6_01 + jg L(6_01) -.L2_0: +L(2_0): cmpq $0, Nmod6 // N % 6 == 0 - je .L999 + je L(999) /************************************************************************************************ * Loop for Nmod6 / 2 > 0 @@ -2187,30 +2187,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J sarq $1, J // j = j / 2 - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -2220,11 +2220,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $3, I // i = (m >> 3) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2234,7 +2234,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2245,7 +2245,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_12: +L(2_12): KERNEL8x2_1(xxx) KERNEL8x2_2(xxx) @@ -2257,7 +2257,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_16 + je L(2_16) KERNEL8x2_1(xxx) KERNEL8x2_2(xxx) @@ -2269,16 +2269,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2290,16 +2290,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL8x2_SUB(xxx) addq $2, BI addq $8, %rax - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vmovddup ALPHA, %xmm0 @@ -2326,24 +2326,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $7, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2352,7 +2352,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2363,7 +2363,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_22: +L(2_22): KERNEL4x2_1(xxx) KERNEL4x2_2(xxx) @@ -2375,7 +2375,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) KERNEL4x2_1(xxx) KERNEL4x2_2(xxx) @@ -2387,16 +2387,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2408,16 +2408,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB(xxx) addq $2, BI addq $4, %rax - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): vmovddup ALPHA, %xmm0 @@ -2437,13 +2437,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2452,7 +2452,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2463,7 +2463,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_1(xxx) KERNEL2x2_2(xxx) @@ -2475,7 +2475,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) KERNEL2x2_1(xxx) KERNEL2x2_2(xxx) @@ -2487,16 +2487,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2508,16 +2508,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB(xxx) addq $2, BI addq $2, %rax - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): vmovddup ALPHA, %xmm0 @@ -2531,13 +2531,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2546,7 +2546,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2556,7 +2556,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -2568,7 +2568,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -2580,16 +2580,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2600,16 +2600,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) addq $2, BI addq $1, %rax - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vmovddup ALPHA, %xmm0 @@ -2625,14 +2625,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_60: +L(2_60): decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -2640,30 +2640,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -2672,11 +2672,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $3, I // i = (m >> 3) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -2685,7 +2685,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -2695,7 +2695,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_12: +L(1_12): KERNEL8x1_1(xxx) KERNEL8x1_2(xxx) @@ -2707,7 +2707,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_16 + je L(1_16) KERNEL8x1_1(xxx) KERNEL8x1_2(xxx) @@ -2719,16 +2719,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -2739,16 +2739,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL8x1_SUB(xxx) addq $1, BI addq $8, %rax - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vmovddup ALPHA, %xmm0 @@ -2764,24 +2764,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $7, M - jz .L999 + jz L(999) testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -2791,7 +2791,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -2801,7 +2801,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_22: +L(1_22): KERNEL4x1_1(xxx) KERNEL4x1_2(xxx) @@ -2813,7 +2813,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) KERNEL4x1_1(xxx) KERNEL4x1_2(xxx) @@ -2825,16 +2825,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -2845,16 +2845,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB(xxx) addq $1, BI addq $4, %rax - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): vmovddup ALPHA, %xmm0 @@ -2868,13 +2868,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -2884,7 +2884,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -2894,7 +2894,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_1(xxx) KERNEL2x1_2(xxx) @@ -2906,7 +2906,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) KERNEL2x1_1(xxx) KERNEL2x1_2(xxx) @@ -2918,16 +2918,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -2938,16 +2938,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB(xxx) addq $1, BI addq $2, %rax - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): vmovddup ALPHA, %xmm0 @@ -2959,13 +2959,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -2974,7 +2974,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, 8), AO @@ -2983,7 +2983,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -2995,7 +2995,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -3007,16 +3007,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -3026,16 +3026,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) addq $1, BI addq $1, %rax - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vmovddup ALPHA, %xmm0 @@ -3047,7 +3047,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp @@ -3135,13 +3135,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -3170,32 +3170,32 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv6, J cmpq $0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_0: +L(2_0): -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -3209,11 +3209,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $3, I // i = (m >> 3) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3251,7 +3251,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3262,7 +3262,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_12: +L(2_12): KERNEL8x2_1(xxx) KERNEL8x2_2(xxx) @@ -3274,7 +3274,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_16 + je L(2_16) KERNEL8x2_1(xxx) KERNEL8x2_2(xxx) @@ -3286,12 +3286,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3299,7 +3299,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3311,16 +3311,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL8x2_SUB(xxx) addq $2, BI addq $8, %rax - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vmovddup ALPHA, %xmm0 @@ -3378,24 +3378,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $7, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3432,7 +3432,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3443,7 +3443,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_22: +L(2_22): KERNEL4x2_1(xxx) KERNEL4x2_2(xxx) @@ -3455,7 +3455,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) KERNEL4x2_1(xxx) KERNEL4x2_2(xxx) @@ -3467,12 +3467,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -3480,7 +3480,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3492,16 +3492,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB(xxx) addq $2, BI addq $4, %rax - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): vmovddup ALPHA, %xmm0 @@ -3549,13 +3549,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3593,7 +3593,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3604,7 +3604,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_1(xxx) KERNEL2x2_2(xxx) @@ -3616,7 +3616,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) KERNEL2x2_1(xxx) KERNEL2x2_2(xxx) @@ -3628,12 +3628,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -3641,7 +3641,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3653,16 +3653,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB(xxx) addq $2, BI addq $2, %rax - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): vmovddup ALPHA, %xmm0 @@ -3700,13 +3700,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3741,7 +3741,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3751,7 +3751,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -3763,7 +3763,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -3775,12 +3775,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -3788,7 +3788,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3799,16 +3799,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) addq $2, BI addq $1, %rax - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vmovddup ALPHA, %xmm0 @@ -3847,17 +3847,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -3865,30 +3865,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -3902,11 +3902,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $3, I // i = (m >> 3) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3942,7 +3942,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -3952,7 +3952,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_12: +L(1_12): KERNEL8x1_1(xxx) KERNEL8x1_2(xxx) @@ -3964,7 +3964,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_16 + je L(1_16) KERNEL8x1_1(xxx) KERNEL8x1_2(xxx) @@ -3976,12 +3976,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3989,7 +3989,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -4000,16 +4000,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL8x1_SUB(xxx) addq $1, BI addq $8, %rax - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vmovddup ALPHA, %xmm0 @@ -4049,24 +4049,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif addq $8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $7, M - jz .L999 + jz L(999) testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4103,7 +4103,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -4113,7 +4113,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_22: +L(1_22): KERNEL4x1_1(xxx) KERNEL4x1_2(xxx) @@ -4125,7 +4125,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) KERNEL4x1_1(xxx) KERNEL4x1_2(xxx) @@ -4137,12 +4137,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4150,7 +4150,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -4161,16 +4161,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB(xxx) addq $1, BI addq $4, %rax - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): vmovddup ALPHA, %xmm0 @@ -4207,13 +4207,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4250,7 +4250,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -4260,7 +4260,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_1(xxx) KERNEL2x1_2(xxx) @@ -4272,7 +4272,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) KERNEL2x1_1(xxx) KERNEL2x1_2(xxx) @@ -4284,12 +4284,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -4297,7 +4297,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -4308,16 +4308,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB(xxx) addq $1, BI addq $2, %rax - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): vmovddup ALPHA, %xmm0 @@ -4350,13 +4350,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4391,7 +4391,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, 8), AO @@ -4400,7 +4400,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -4412,7 +4412,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -4424,12 +4424,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -4437,7 +4437,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -4447,16 +4447,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) addq $1, BI addq $1, %rax - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vmovddup ALPHA, %xmm0 @@ -4489,7 +4489,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/dgemm_ncopy_2.S b/kernel/x86_64/dgemm_ncopy_2.S index 532fddf75f..4a734f024f 100644 --- a/kernel/x86_64/dgemm_ncopy_2.S +++ b/kernel/x86_64/dgemm_ncopy_2.S @@ -112,20 +112,20 @@ cmovne %rax, MM testq $SIZE, LDA - jne .L50 + jne L(50) movq N, J sarq $1, J - jle .L30 + jle L(30) ALIGN_4 -.L21: +L(21): movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), A testq $SIZE, A - je .L22 + je L(22) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -139,13 +139,13 @@ subq $-2 * SIZE, B ALIGN_3 -.L22: +L(22): movq MM, I sarq $3, I - jle .L24 + jle L(24) ALIGN_4 -.L23: +L(23): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 2 * SIZE(AO1) #endif @@ -203,12 +203,12 @@ subq $-16 * SIZE, B decq I - jg .L23 + jg L(23) ALIGN_4 -.L24: +L(24): testq $4, MM - jle .L26 + jle L(26) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 0 * SIZE(AO2), %xmm1 @@ -233,9 +233,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L26: +L(26): testq $2, MM - jle .L28 + jle L(28) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 0 * SIZE(AO2), %xmm1 @@ -252,9 +252,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L28: +L(28): testq $1, MM - jle .L29 + jle L(29) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -265,27 +265,27 @@ subq $-2 * SIZE, B ALIGN_4 -.L29: +L(29): decq J - jg .L21 + jg L(21) ALIGN_4 -.L30: +L(30): testq $1, N - jle .L999 + jle L(999) -.L30x: +L(30x): movq A, AO1 testq $SIZE, A - jne .L35 + jne L(35) movq M, I sarq $3, I - jle .L32 + jle L(32) ALIGN_4 -.L31: +L(31): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 4 * SIZE(AO1) #endif @@ -308,12 +308,12 @@ addq $8 * SIZE, B decq I - jg .L31 + jg L(31) ALIGN_4 -.L32: +L(32): testq $4, M - jle .L33 + jle L(33) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 2 * SIZE(AO1), %xmm1 @@ -325,9 +325,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L33: +L(33): testq $2, M - jle .L34 + jle L(34) MOVAPS 0 * SIZE(AO1), %xmm0 @@ -337,24 +337,24 @@ subq $-2 * SIZE, B ALIGN_4 -.L34: +L(34): testq $1, M - jle .L999 + jle L(999) movsd 0 * SIZE(AO1), %xmm0 movlpd %xmm0, -16 * SIZE(B) - jmp .L999 + jmp L(999) ALIGN_4 -.L35: +L(35): movaps -1 * SIZE(AO1), %xmm0 movq M, I sarq $3, I - jle .L36 + jle L(36) ALIGN_4 -.L36: +L(36): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 4 * SIZE(AO1) #endif @@ -384,12 +384,12 @@ subq $-8 * SIZE, B decq I - jg .L36 + jg L(36) ALIGN_4 -.L37: +L(37): testq $4, M - jle .L38 + jle L(38) MOVAPS 1 * SIZE(AO1), %xmm1 MOVAPS 3 * SIZE(AO1), %xmm2 @@ -406,9 +406,9 @@ addq $4 * SIZE, B ALIGN_4 -.L38: +L(38): testq $2, M - jle .L39 + jle L(39) MOVAPS 1 * SIZE(AO1), %xmm1 @@ -422,27 +422,27 @@ subq $-2 * SIZE, B ALIGN_4 -.L39: +L(39): testq $1, M - jle .L999 + jle L(999) movhpd %xmm0, -16 * SIZE(B) - jmp .L999 + jmp L(999) ALIGN_4 -.L50: +L(50): movq N, J sarq $1, J - jle .L30 + jle L(30) ALIGN_4 -.L61: +L(61): movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), A testq $SIZE, A - je .L62 + je L(62) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -456,15 +456,15 @@ subq $-2 * SIZE, B ALIGN_3 -.L62: +L(62): MOVAPS -1 * SIZE(AO2), %xmm5 movq MM, I sarq $3, I - jle .L64 + jle L(64) ALIGN_4 -.L63: +L(63): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 2 * SIZE(AO1) #endif @@ -516,12 +516,12 @@ subq $-16 * SIZE, B decq I - jg .L63 + jg L(63) ALIGN_4 -.L64: +L(64): testq $4, MM - jle .L66 + jle L(66) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 1 * SIZE(AO2), %xmm1 @@ -545,9 +545,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L66: +L(66): testq $2, MM - jle .L68 + jle L(68) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 1 * SIZE(AO2), %xmm1 @@ -563,9 +563,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L68: +L(68): testq $1, MM - jle .L69 + jle L(69) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -576,15 +576,15 @@ subq $-2 * SIZE, B ALIGN_4 -.L69: +L(69): decq J - jg .L61 + jg L(61) testq $1, N - jne .L30 + jne L(30) ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 diff --git a/kernel/x86_64/dgemm_ncopy_4.S b/kernel/x86_64/dgemm_ncopy_4.S index 41eac95970..3e6ef0600f 100644 --- a/kernel/x86_64/dgemm_ncopy_4.S +++ b/kernel/x86_64/dgemm_ncopy_4.S @@ -133,20 +133,20 @@ cmovne %rax, MM testq $SIZE, LDA - jne .L50 + jne L(50) movq N, J sarq $2, J - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): movq A, AO1 leaq (A, LDA, 2), AO2 leaq (A, LDA, 4), A testq $SIZE, A - je .L12 + je L(12) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -164,13 +164,13 @@ subq $-4 * SIZE, B ALIGN_3 -.L12: +L(12): movq MM, I sarq $3, I - jle .L14 + jle L(14) ALIGN_4 -.L13: +L(13): #ifdef PREFETCH PREFETCH PREFETCHSIZE * SIZE(AO1) #endif @@ -280,12 +280,12 @@ subq $-32 * SIZE, B decq I - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): testq $4, MM - jle .L16 + jle L(16) movapd 0 * SIZE(AO1), %xmm0 movapd 0 * SIZE(AO1, LDA), %xmm1 @@ -328,9 +328,9 @@ subq $-16 * SIZE, B ALIGN_4 -.L16: +L(16): testq $2, MM - jle .L18 + jle L(18) movapd 0 * SIZE(AO1), %xmm0 movapd 0 * SIZE(AO1, LDA), %xmm1 @@ -355,9 +355,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L18: +L(18): testq $1, MM - jle .L19 + jle L(19) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -372,21 +372,21 @@ subq $-4 * SIZE, B ALIGN_4 -.L19: +L(19): decq J - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, N - jle .L30 + jle L(30) movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), A testq $SIZE, A - je .L22 + je L(22) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -400,13 +400,13 @@ subq $-2 * SIZE, B ALIGN_3 -.L22: +L(22): movq MM, I sarq $3, I - jle .L24 + jle L(24) ALIGN_4 -.L23: +L(23): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 2 * SIZE(AO1) #endif @@ -464,12 +464,12 @@ subq $-16 * SIZE, B decq I - jg .L23 + jg L(23) ALIGN_4 -.L24: +L(24): testq $4, MM - jle .L26 + jle L(26) movapd 0 * SIZE(AO1), %xmm0 movapd 0 * SIZE(AO2), %xmm1 @@ -494,9 +494,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L26: +L(26): testq $2, MM - jle .L28 + jle L(28) movapd 0 * SIZE(AO1), %xmm0 movapd 0 * SIZE(AO2), %xmm1 @@ -513,9 +513,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L28: +L(28): testq $1, MM - jle .L30 + jle L(30) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -526,21 +526,21 @@ subq $-2 * SIZE, B ALIGN_4 -.L30: +L(30): testq $1, N - jle .L999 + jle L(999) movq A, AO1 testq $SIZE, A - jne .L35 + jne L(35) movq MM, I sarq $3, I - jle .L32 + jle L(32) ALIGN_4 -.L31: +L(31): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 4 * SIZE(AO1) #endif @@ -563,12 +563,12 @@ subq $-8 * SIZE, B decq I - jg .L31 + jg L(31) ALIGN_4 -.L32: +L(32): testq $4, MM - jle .L33 + jle L(33) movapd 0 * SIZE(AO1), %xmm0 movapd 2 * SIZE(AO1), %xmm1 @@ -580,9 +580,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L33: +L(33): testq $2, MM - jle .L34 + jle L(34) movapd 0 * SIZE(AO1), %xmm0 @@ -592,25 +592,25 @@ subq $-2 * SIZE, B ALIGN_4 -.L34: +L(34): testq $1, MM - jle .L999 + jle L(999) movsd 0 * SIZE(AO1), %xmm0 movlpd %xmm0, -16 * SIZE(B) - jmp .L999 + jmp L(999) ALIGN_4 -.L35: +L(35): movapd -1 * SIZE(AO1), %xmm0 movq MM, I sarq $3, I - jle .L36 + jle L(36) ALIGN_4 -.L36: +L(36): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 4 * SIZE(AO1) #endif @@ -640,12 +640,12 @@ subq $-8 * SIZE, B decq I - jg .L36 + jg L(36) ALIGN_4 -.L37: +L(37): testq $4, MM - jle .L38 + jle L(38) movapd 1 * SIZE(AO1), %xmm1 movapd 3 * SIZE(AO1), %xmm2 @@ -662,9 +662,9 @@ addq $4 * SIZE, B ALIGN_4 -.L38: +L(38): testq $2, MM - jle .L39 + jle L(39) movapd 1 * SIZE(AO1), %xmm1 @@ -678,29 +678,29 @@ subq $-2 * SIZE, B ALIGN_4 -.L39: +L(39): testq $1, MM - jle .L999 + jle L(999) shufpd $1, %xmm0, %xmm0 movlpd %xmm0, -16 * SIZE(B) - jmp .L999 + jmp L(999) ALIGN_4 -.L50: +L(50): movq N, J sarq $2, J - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): movq A, AO1 leaq (A, LDA, 2), AO2 leaq (A, LDA, 4), A testq $SIZE, A - je .L52 + je L(52) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -718,16 +718,16 @@ subq $-4 * SIZE, B ALIGN_3 -.L52: +L(52): movapd -1 * SIZE(AO1, LDA), %xmm5 movapd -1 * SIZE(AO2, LDA), %xmm7 movq MM, I sarq $3, I - jle .L54 + jle L(54) ALIGN_4 -.L53: +L(53): #ifdef PREFETCH PREFETCH PREFETCHSIZE * SIZE(AO1) #endif @@ -825,12 +825,12 @@ subq $-32 * SIZE, B decq I - jg .L53 + jg L(53) ALIGN_4 -.L54: +L(54): testq $4, MM - jle .L56 + jle L(56) movapd 0 * SIZE(AO1), %xmm0 movapd 1 * SIZE(AO1, LDA), %xmm1 @@ -867,9 +867,9 @@ subq $-16 * SIZE, B ALIGN_4 -.L56: +L(56): testq $2, MM - jle .L58 + jle L(58) movapd 0 * SIZE(AO1), %xmm0 movapd 1 * SIZE(AO1, LDA), %xmm1 @@ -891,9 +891,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L58: +L(58): testq $1, MM - jle .L59 + jle L(59) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -908,21 +908,21 @@ subq $-4 * SIZE, B ALIGN_4 -.L59: +L(59): decq J - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, N - jle .L70 + jle L(70) movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), A testq $SIZE, A - je .L62 + je L(62) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -936,15 +936,15 @@ subq $-2 * SIZE, B ALIGN_3 -.L62: +L(62): movapd -1 * SIZE(AO2), %xmm5 movq MM, I sarq $3, I - jle .L64 + jle L(64) ALIGN_4 -.L63: +L(63): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 2 * SIZE(AO1) #endif @@ -996,12 +996,12 @@ subq $-16 * SIZE, B decq I - jg .L63 + jg L(63) ALIGN_4 -.L64: +L(64): testq $4, MM - jle .L66 + jle L(66) movapd 0 * SIZE(AO1), %xmm0 movapd 1 * SIZE(AO2), %xmm1 @@ -1025,9 +1025,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L66: +L(66): testq $2, MM - jle .L68 + jle L(68) movapd 0 * SIZE(AO1), %xmm0 movapd 1 * SIZE(AO2), %xmm1 @@ -1043,9 +1043,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L68: +L(68): testq $1, MM - jle .L70 + jle L(70) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -1056,21 +1056,21 @@ subq $-2 * SIZE, B ALIGN_4 -.L70: +L(70): testq $1, N - jle .L999 + jle L(999) movq A, AO1 testq $SIZE, A - jne .L75 + jne L(75) movq MM, I sarq $3, I - jle .L72 + jle L(72) ALIGN_4 -.L71: +L(71): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 4 * SIZE(AO1) #endif @@ -1093,12 +1093,12 @@ subq $-8 * SIZE, B decq I - jg .L71 + jg L(71) ALIGN_4 -.L72: +L(72): testq $4, MM - jle .L73 + jle L(73) movapd 0 * SIZE(AO1), %xmm0 movapd 2 * SIZE(AO1), %xmm2 @@ -1110,9 +1110,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L73: +L(73): testq $2, MM - jle .L74 + jle L(74) movapd 0 * SIZE(AO1), %xmm0 @@ -1122,25 +1122,25 @@ subq $-2 * SIZE, B ALIGN_4 -.L74: +L(74): testq $1, MM - jle .L999 + jle L(999) movsd 0 * SIZE(AO1), %xmm0 movlpd %xmm0, -16 * SIZE(B) - jmp .L999 + jmp L(999) ALIGN_4 -.L75: +L(75): movapd -1 * SIZE(AO1), %xmm0 movq MM, I sarq $3, I - jle .L76 + jle L(76) ALIGN_4 -.L76: +L(76): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 4 * SIZE(AO1) #endif @@ -1170,12 +1170,12 @@ subq $-8 * SIZE, B decq I - jg .L76 + jg L(76) ALIGN_4 -.L77: +L(77): testq $4, MM - jle .L78 + jle L(78) movapd 1 * SIZE(AO1), %xmm1 movapd 3 * SIZE(AO1), %xmm2 @@ -1192,9 +1192,9 @@ addq $4 * SIZE, B ALIGN_4 -.L78: +L(78): testq $2, MM - jle .L79 + jle L(79) movapd 1 * SIZE(AO1), %xmm1 @@ -1208,16 +1208,16 @@ subq $-2 * SIZE, B ALIGN_4 -.L79: +L(79): testq $1, MM - jle .L999 + jle L(999) shufpd $1, %xmm0, %xmm0 movlpd %xmm0, -16 * SIZE(B) ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 movups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/dgemm_ncopy_8.S b/kernel/x86_64/dgemm_ncopy_8.S index 7600c9a567..2b83ac6432 100644 --- a/kernel/x86_64/dgemm_ncopy_8.S +++ b/kernel/x86_64/dgemm_ncopy_8.S @@ -125,20 +125,20 @@ cmovne %rax, MM testq $SIZE, LDA - jne .L50 + jne L(50) movq N, J sarq $3, J - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): movq A, AO1 leaq (A, LDA, 4), AO2 leaq (A, LDA, 8), A testq $SIZE, A - je .L12 + je L(12) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -165,13 +165,13 @@ subq $-8 * SIZE, B ALIGN_3 -.L12: +L(12): movq MM, I sarq $3, I - jle .L14 + jle L(14) ALIGN_4 -.L13: +L(13): #ifdef PREFETCH PREFETCH PREFETCHSIZE * SIZE(AO1) #endif @@ -381,12 +381,12 @@ subq $-64 * SIZE, B decq I - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): testq $4, MM - jle .L16 + jle L(16) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 0 * SIZE(AO1, LDA), %xmm1 @@ -463,9 +463,9 @@ subq $-32 * SIZE, B ALIGN_4 -.L16: +L(16): testq $2, MM - jle .L18 + jle L(18) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 0 * SIZE(AO1, LDA), %xmm1 @@ -507,9 +507,9 @@ subq $-16 * SIZE, B ALIGN_4 -.L18: +L(18): testq $1, MM - jle .L19 + jle L(19) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -534,21 +534,21 @@ subq $-8 * SIZE, B ALIGN_4 -.L19: +L(19): decq J - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, N - jle .L30 + jle L(30) movq A, AO1 leaq (A, LDA, 2), AO2 leaq (A, LDA, 4), A testq $SIZE, A - je .L22 + je L(22) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -566,13 +566,13 @@ subq $-4 * SIZE, B ALIGN_3 -.L22: +L(22): movq MM, I sarq $3, I - jle .L24 + jle L(24) ALIGN_4 -.L23: +L(23): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 2 * SIZE(AO1) #endif @@ -682,12 +682,12 @@ subq $-32 * SIZE, B decq I - jg .L23 + jg L(23) ALIGN_4 -.L24: +L(24): testq $4, MM - jle .L26 + jle L(26) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 0 * SIZE(AO1, LDA), %xmm1 @@ -730,9 +730,9 @@ subq $-16 * SIZE, B ALIGN_4 -.L26: +L(26): testq $2, MM - jle .L28 + jle L(28) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 0 * SIZE(AO1, LDA), %xmm1 @@ -757,9 +757,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L28: +L(28): testq $1, MM - jle .L30 + jle L(30) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -774,16 +774,16 @@ subq $-4 * SIZE, B ALIGN_4 -.L30: +L(30): testq $2, N - jle .L40 + jle L(40) movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), A testq $SIZE, A - je .L32 + je L(32) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -797,13 +797,13 @@ subq $-2 * SIZE, B ALIGN_3 -.L32: +L(32): movq MM, I sarq $3, I - jle .L34 + jle L(34) ALIGN_4 -.L33: +L(33): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 4 * SIZE(AO1) #endif @@ -861,12 +861,12 @@ subq $-16 * SIZE, B decq I - jg .L33 + jg L(33) ALIGN_4 -.L34: +L(34): testq $4, MM - jle .L36 + jle L(36) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 0 * SIZE(AO2), %xmm1 @@ -891,9 +891,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L36: +L(36): testq $2, MM - jle .L38 + jle L(38) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 0 * SIZE(AO2), %xmm1 @@ -910,9 +910,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L38: +L(38): testq $1, MM - jle .L40 + jle L(40) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -923,21 +923,21 @@ subq $-2 * SIZE, B ALIGN_4 -.L40: +L(40): testq $1, N - jle .L999 + jle L(999) movq A, AO1 testq $SIZE, A - jne .L45 + jne L(45) movq MM, I sarq $3, I - jle .L42 + jle L(42) ALIGN_4 -.L41: +L(41): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 8 * SIZE(AO1) #endif @@ -960,12 +960,12 @@ subq $-8 * SIZE, B decq I - jg .L41 + jg L(41) ALIGN_4 -.L42: +L(42): testq $4, MM - jle .L43 + jle L(43) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 2 * SIZE(AO1), %xmm1 @@ -977,9 +977,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L43: +L(43): testq $2, MM - jle .L44 + jle L(44) MOVAPS 0 * SIZE(AO1), %xmm0 @@ -989,25 +989,25 @@ subq $-2 * SIZE, B ALIGN_4 -.L44: +L(44): testq $1, MM - jle .L999 + jle L(999) movsd 0 * SIZE(AO1), %xmm0 movlpd %xmm0, -16 * SIZE(B) - jmp .L999 + jmp L(999) ALIGN_4 -.L45: +L(45): MOVAPS -1 * SIZE(AO1), %xmm0 movq M, I sarq $3, I - jle .L46 + jle L(46) ALIGN_4 -.L46: +L(46): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 8 * SIZE(AO1) #endif @@ -1037,12 +1037,12 @@ subq $-8 * SIZE, B decq I - jg .L46 + jg L(46) ALIGN_4 -.L47: +L(47): testq $4, M - jle .L48 + jle L(48) MOVAPS 1 * SIZE(AO1), %xmm1 MOVAPS 3 * SIZE(AO1), %xmm2 @@ -1059,9 +1059,9 @@ addq $4 * SIZE, B ALIGN_4 -.L48: +L(48): testq $2, M - jle .L49 + jle L(49) MOVAPS 1 * SIZE(AO1), %xmm1 @@ -1075,29 +1075,29 @@ subq $-2 * SIZE, B ALIGN_4 -.L49: +L(49): testq $1, M - jle .L999 + jle L(999) shufpd $1, %xmm0, %xmm0 movlpd %xmm0, -16 * SIZE(B) - jmp .L999 + jmp L(999) ALIGN_4 -.L50: +L(50): movq N, J sarq $3, J - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): movq A, AO1 leaq (A, LDA, 4), AO2 leaq (A, LDA, 8), A testq $SIZE, A - je .L52 + je L(52) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -1123,7 +1123,7 @@ subq $-8 * SIZE, B ALIGN_3 -.L52: +L(52): MOVAPS -1 * SIZE(AO1, LDA), %xmm9 MOVAPS -1 * SIZE(AO1, LDA3), %xmm10 MOVAPS -1 * SIZE(AO2, LDA), %xmm11 @@ -1131,10 +1131,10 @@ movq MM, I sarq $3, I - jle .L54 + jle L(54) ALIGN_4 -.L53: +L(53): #ifdef PREFETCH PREFETCH PREFETCHSIZE * SIZE(AO1) #endif @@ -1323,12 +1323,12 @@ subq $-64 * SIZE, B decq I - jg .L53 + jg L(53) ALIGN_4 -.L54: +L(54): testq $4, MM - jle .L56 + jle L(56) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 1 * SIZE(AO1, LDA), %xmm1 @@ -1393,9 +1393,9 @@ subq $-32 * SIZE, B ALIGN_4 -.L56: +L(56): testq $2, MM - jle .L58 + jle L(58) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 1 * SIZE(AO1, LDA), %xmm1 @@ -1431,9 +1431,9 @@ subq $-16 * SIZE, B ALIGN_4 -.L58: +L(58): testq $1, MM - jle .L59 + jle L(59) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -1457,21 +1457,21 @@ subq $-8 * SIZE, B ALIGN_4 -.L59: +L(59): decq J - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $4, N - jle .L70 + jle L(70) movq A, AO1 leaq (A, LDA, 2), AO2 leaq (A, LDA, 4), A testq $SIZE, A - je .L62 + je L(62) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -1489,16 +1489,16 @@ subq $-4 * SIZE, B ALIGN_3 -.L62: +L(62): movaps -1 * SIZE(AO1, LDA), %xmm5 movaps -1 * SIZE(AO2, LDA), %xmm7 movq MM, I sarq $3, I - jle .L64 + jle L(64) ALIGN_4 -.L63: +L(63): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 2 * SIZE(AO1) #endif @@ -1596,12 +1596,12 @@ subq $-32 * SIZE, B decq I - jg .L63 + jg L(63) ALIGN_4 -.L64: +L(64): testq $4, MM - jle .L66 + jle L(66) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 1 * SIZE(AO1, LDA), %xmm1 @@ -1638,9 +1638,9 @@ subq $-16 * SIZE, B ALIGN_4 -.L66: +L(66): testq $2, MM - jle .L68 + jle L(68) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 1 * SIZE(AO1, LDA), %xmm1 @@ -1662,9 +1662,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L68: +L(68): testq $1, MM - jle .L70 + jle L(70) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -1679,16 +1679,16 @@ subq $-4 * SIZE, B ALIGN_4 -.L70: +L(70): testq $2, N - jle .L80 + jle L(80) movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), A testq $SIZE, A - je .L72 + je L(72) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -1702,15 +1702,15 @@ subq $-2 * SIZE, B ALIGN_3 -.L72: +L(72): MOVAPS -1 * SIZE(AO2), %xmm5 movq MM, I sarq $3, I - jle .L74 + jle L(74) ALIGN_4 -.L73: +L(73): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 4 * SIZE(AO1) #endif @@ -1762,12 +1762,12 @@ subq $-16 * SIZE, B decq I - jg .L73 + jg L(73) ALIGN_4 -.L74: +L(74): testq $4, MM - jle .L76 + jle L(76) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 1 * SIZE(AO2), %xmm1 @@ -1791,9 +1791,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L76: +L(76): testq $2, MM - jle .L78 + jle L(78) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 1 * SIZE(AO2), %xmm1 @@ -1809,9 +1809,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L78: +L(78): testq $1, MM - jle .L80 + jle L(80) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -1822,21 +1822,21 @@ subq $-2 * SIZE, B ALIGN_4 -.L80: +L(80): testq $1, N - jle .L999 + jle L(999) movq A, AO1 testq $SIZE, A - jne .L85 + jne L(85) movq MM, I sarq $3, I - jle .L82 + jle L(82) ALIGN_4 -.L81: +L(81): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 8 * SIZE(AO1) #endif @@ -1859,12 +1859,12 @@ subq $-8 * SIZE, B decq I - jg .L81 + jg L(81) ALIGN_4 -.L82: +L(82): testq $4, MM - jle .L83 + jle L(83) MOVAPS 0 * SIZE(AO1), %xmm0 MOVAPS 2 * SIZE(AO1), %xmm2 @@ -1876,9 +1876,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L83: +L(83): testq $2, MM - jle .L84 + jle L(84) MOVAPS 0 * SIZE(AO1), %xmm0 @@ -1888,25 +1888,25 @@ subq $-2 * SIZE, B ALIGN_4 -.L84: +L(84): testq $1, MM - jle .L999 + jle L(999) movsd 0 * SIZE(AO1), %xmm0 movlpd %xmm0, -16 * SIZE(B) - jmp .L999 + jmp L(999) ALIGN_4 -.L85: +L(85): MOVAPS -1 * SIZE(AO1), %xmm0 movq M, I sarq $3, I - jle .L86 + jle L(86) ALIGN_4 -.L86: +L(86): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 8 * SIZE(AO1) #endif @@ -1936,12 +1936,12 @@ subq $-8 * SIZE, B decq I - jg .L86 + jg L(86) ALIGN_4 -.L87: +L(87): testq $4, M - jle .L88 + jle L(88) MOVAPS 1 * SIZE(AO1), %xmm1 MOVAPS 3 * SIZE(AO1), %xmm2 @@ -1958,9 +1958,9 @@ addq $4 * SIZE, B ALIGN_4 -.L88: +L(88): testq $2, M - jle .L89 + jle L(89) MOVAPS 1 * SIZE(AO1), %xmm1 @@ -1974,16 +1974,16 @@ subq $-2 * SIZE, B ALIGN_4 -.L89: +L(89): testq $1, M - jle .L999 + jle L(999) shufpd $1, %xmm0, %xmm0 movlpd %xmm0, -16 * SIZE(B) ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 movups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/dgemm_ncopy_8_bulldozer.S b/kernel/x86_64/dgemm_ncopy_8_bulldozer.S index 43f9cd2e3c..dfccce8cfe 100644 --- a/kernel/x86_64/dgemm_ncopy_8_bulldozer.S +++ b/kernel/x86_64/dgemm_ncopy_8_bulldozer.S @@ -113,20 +113,20 @@ cmovne %rax, MM testq $SIZE, LDA - jne .L50 + jne L(50) movq N, J sarq $3, J - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): movq A, AO1 leaq (A, LDA, 4), AO2 leaq (A, LDA, 8), A testq $SIZE, A - je .L12 + je L(12) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO1, LDA), %xmm1 @@ -153,13 +153,13 @@ subq $-8 * SIZE, B ALIGN_3 -.L12: +L(12): movq MM, I sarq $3, I - jle .L14 + jle L(14) ALIGN_4 -.L13: +L(13): prefetchnta A_PRE(AO1) vmovups 0 * SIZE(AO1), %xmm0 @@ -338,12 +338,12 @@ subq $-64 * SIZE, B decq I - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): testq $4, MM - jle .L16 + jle L(16) vmovups 0 * SIZE(AO1), %xmm0 vmovups 0 * SIZE(AO1, LDA), %xmm1 @@ -420,9 +420,9 @@ subq $-32 * SIZE, B ALIGN_4 -.L16: +L(16): testq $2, MM - jle .L18 + jle L(18) vmovups 0 * SIZE(AO1), %xmm0 vmovups 0 * SIZE(AO1, LDA), %xmm1 @@ -464,9 +464,9 @@ subq $-16 * SIZE, B ALIGN_4 -.L18: +L(18): testq $1, MM - jle .L19 + jle L(19) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO1, LDA), %xmm1 @@ -491,21 +491,21 @@ subq $-8 * SIZE, B ALIGN_4 -.L19: +L(19): decq J - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, N - jle .L30 + jle L(30) movq A, AO1 leaq (A, LDA, 2), AO2 leaq (A, LDA, 4), A testq $SIZE, A - je .L22 + je L(22) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO1, LDA), %xmm1 @@ -523,13 +523,13 @@ subq $-4 * SIZE, B ALIGN_3 -.L22: +L(22): movq MM, I sarq $3, I - jle .L24 + jle L(24) ALIGN_4 -.L23: +L(23): vmovups 0 * SIZE(AO1), %xmm0 vmovups 0 * SIZE(AO1, LDA), %xmm1 @@ -615,12 +615,12 @@ subq $-32 * SIZE, B decq I - jg .L23 + jg L(23) ALIGN_4 -.L24: +L(24): testq $4, MM - jle .L26 + jle L(26) vmovups 0 * SIZE(AO1), %xmm0 vmovups 0 * SIZE(AO1, LDA), %xmm1 @@ -663,9 +663,9 @@ subq $-16 * SIZE, B ALIGN_4 -.L26: +L(26): testq $2, MM - jle .L28 + jle L(28) vmovups 0 * SIZE(AO1), %xmm0 vmovups 0 * SIZE(AO1, LDA), %xmm1 @@ -690,9 +690,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L28: +L(28): testq $1, MM - jle .L30 + jle L(30) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO1, LDA), %xmm1 @@ -707,16 +707,16 @@ subq $-4 * SIZE, B ALIGN_4 -.L30: +L(30): testq $2, N - jle .L40 + jle L(40) movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), A testq $SIZE, A - je .L32 + je L(32) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO2), %xmm1 @@ -730,13 +730,13 @@ subq $-2 * SIZE, B ALIGN_3 -.L32: +L(32): movq MM, I sarq $3, I - jle .L34 + jle L(34) ALIGN_4 -.L33: +L(33): vmovups 0 * SIZE(AO1), %xmm0 vmovups 0 * SIZE(AO2), %xmm1 @@ -782,12 +782,12 @@ subq $-16 * SIZE, B decq I - jg .L33 + jg L(33) ALIGN_4 -.L34: +L(34): testq $4, MM - jle .L36 + jle L(36) vmovups 0 * SIZE(AO1), %xmm0 vmovups 0 * SIZE(AO2), %xmm1 @@ -812,9 +812,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L36: +L(36): testq $2, MM - jle .L38 + jle L(38) vmovups 0 * SIZE(AO1), %xmm0 vmovups 0 * SIZE(AO2), %xmm1 @@ -831,9 +831,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L38: +L(38): testq $1, MM - jle .L40 + jle L(40) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO2), %xmm1 @@ -844,21 +844,21 @@ subq $-2 * SIZE, B ALIGN_4 -.L40: +L(40): testq $1, N - jle .L999 + jle L(999) movq A, AO1 testq $SIZE, A - jne .L45 + jne L(45) movq MM, I sarq $3, I - jle .L42 + jle L(42) ALIGN_4 -.L41: +L(41): vmovups 0 * SIZE(AO1), %xmm0 vmovups 2 * SIZE(AO1), %xmm1 @@ -875,12 +875,12 @@ subq $-8 * SIZE, B decq I - jg .L41 + jg L(41) ALIGN_4 -.L42: +L(42): testq $4, MM - jle .L43 + jle L(43) vmovups 0 * SIZE(AO1), %xmm0 vmovups 2 * SIZE(AO1), %xmm1 @@ -892,9 +892,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L43: +L(43): testq $2, MM - jle .L44 + jle L(44) vmovups 0 * SIZE(AO1), %xmm0 @@ -904,25 +904,25 @@ subq $-2 * SIZE, B ALIGN_4 -.L44: +L(44): testq $1, MM - jle .L999 + jle L(999) vmovsd 0 * SIZE(AO1), %xmm0 vmovlpd %xmm0, -16 * SIZE(B) - jmp .L999 + jmp L(999) ALIGN_4 -.L45: +L(45): vmovups -1 * SIZE(AO1), %xmm0 movq M, I sarq $3, I - jle .L46 + jle L(46) ALIGN_4 -.L46: +L(46): vmovups 1 * SIZE(AO1), %xmm1 vmovups 3 * SIZE(AO1), %xmm2 @@ -946,12 +946,12 @@ subq $-8 * SIZE, B decq I - jg .L46 + jg L(46) ALIGN_4 -.L47: +L(47): testq $4, M - jle .L48 + jle L(48) vmovups 1 * SIZE(AO1), %xmm1 vmovups 3 * SIZE(AO1), %xmm2 @@ -968,9 +968,9 @@ addq $4 * SIZE, B ALIGN_4 -.L48: +L(48): testq $2, M - jle .L49 + jle L(49) vmovups 1 * SIZE(AO1), %xmm1 @@ -984,29 +984,29 @@ subq $-2 * SIZE, B ALIGN_4 -.L49: +L(49): testq $1, M - jle .L999 + jle L(999) vshufpd $1, %xmm0, %xmm0 , %xmm0 vmovlpd %xmm0, -16 * SIZE(B) - jmp .L999 + jmp L(999) ALIGN_4 -.L50: +L(50): movq N, J sarq $3, J - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): movq A, AO1 leaq (A, LDA, 4), AO2 leaq (A, LDA, 8), A testq $SIZE, A - je .L52 + je L(52) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO1, LDA), %xmm1 @@ -1032,7 +1032,7 @@ subq $-8 * SIZE, B ALIGN_3 -.L52: +L(52): vmovups -1 * SIZE(AO1, LDA), %xmm9 vmovups -1 * SIZE(AO1, LDA3), %xmm10 vmovups -1 * SIZE(AO2, LDA), %xmm11 @@ -1040,10 +1040,10 @@ movq MM, I sarq $3, I - jle .L54 + jle L(54) ALIGN_4 -.L53: +L(53): vmovups 0 * SIZE(AO1), %xmm0 vmovups 1 * SIZE(AO1, LDA), %xmm1 @@ -1184,12 +1184,12 @@ subq $-64 * SIZE, B decq I - jg .L53 + jg L(53) ALIGN_4 -.L54: +L(54): testq $4, MM - jle .L56 + jle L(56) vmovups 0 * SIZE(AO1), %xmm0 vmovups 1 * SIZE(AO1, LDA), %xmm1 @@ -1254,9 +1254,9 @@ subq $-32 * SIZE, B ALIGN_4 -.L56: +L(56): testq $2, MM - jle .L58 + jle L(58) vmovups 0 * SIZE(AO1), %xmm0 vmovups 1 * SIZE(AO1, LDA), %xmm1 @@ -1292,9 +1292,9 @@ subq $-16 * SIZE, B ALIGN_4 -.L58: +L(58): testq $1, MM - jle .L59 + jle L(59) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO1, LDA), %xmm1 @@ -1318,21 +1318,21 @@ subq $-8 * SIZE, B ALIGN_4 -.L59: +L(59): decq J - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $4, N - jle .L70 + jle L(70) movq A, AO1 leaq (A, LDA, 2), AO2 leaq (A, LDA, 4), A testq $SIZE, A - je .L62 + je L(62) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO1, LDA), %xmm1 @@ -1350,16 +1350,16 @@ subq $-4 * SIZE, B ALIGN_3 -.L62: +L(62): vmovups -1 * SIZE(AO1, LDA), %xmm5 vmovups -1 * SIZE(AO2, LDA), %xmm7 movq MM, I sarq $3, I - jle .L64 + jle L(64) ALIGN_4 -.L63: +L(63): vmovups 0 * SIZE(AO1), %xmm0 vmovups 1 * SIZE(AO1, LDA), %xmm1 @@ -1433,12 +1433,12 @@ subq $-32 * SIZE, B decq I - jg .L63 + jg L(63) ALIGN_4 -.L64: +L(64): testq $4, MM - jle .L66 + jle L(66) vmovups 0 * SIZE(AO1), %xmm0 vmovups 1 * SIZE(AO1, LDA), %xmm1 @@ -1475,9 +1475,9 @@ subq $-16 * SIZE, B ALIGN_4 -.L66: +L(66): testq $2, MM - jle .L68 + jle L(68) vmovups 0 * SIZE(AO1), %xmm0 vmovups 1 * SIZE(AO1, LDA), %xmm1 @@ -1499,9 +1499,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L68: +L(68): testq $1, MM - jle .L70 + jle L(70) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO1, LDA), %xmm1 @@ -1516,16 +1516,16 @@ subq $-4 * SIZE, B ALIGN_4 -.L70: +L(70): testq $2, N - jle .L80 + jle L(80) movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), A testq $SIZE, A - je .L72 + je L(72) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO2), %xmm1 @@ -1539,15 +1539,15 @@ subq $-2 * SIZE, B ALIGN_3 -.L72: +L(72): vmovups -1 * SIZE(AO2), %xmm5 movq MM, I sarq $3, I - jle .L74 + jle L(74) ALIGN_4 -.L73: +L(73): vmovups 0 * SIZE(AO1), %xmm0 vmovups 1 * SIZE(AO2), %xmm1 @@ -1587,12 +1587,12 @@ subq $-16 * SIZE, B decq I - jg .L73 + jg L(73) ALIGN_4 -.L74: +L(74): testq $4, MM - jle .L76 + jle L(76) vmovups 0 * SIZE(AO1), %xmm0 vmovups 1 * SIZE(AO2), %xmm1 @@ -1616,9 +1616,9 @@ subq $-8 * SIZE, B ALIGN_4 -.L76: +L(76): testq $2, MM - jle .L78 + jle L(78) vmovups 0 * SIZE(AO1), %xmm0 vmovups 1 * SIZE(AO2), %xmm1 @@ -1634,9 +1634,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L78: +L(78): testq $1, MM - jle .L80 + jle L(80) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO2), %xmm1 @@ -1647,21 +1647,21 @@ subq $-2 * SIZE, B ALIGN_4 -.L80: +L(80): testq $1, N - jle .L999 + jle L(999) movq A, AO1 testq $SIZE, A - jne .L85 + jne L(85) movq MM, I sarq $3, I - jle .L82 + jle L(82) ALIGN_4 -.L81: +L(81): vmovups 0 * SIZE(AO1), %xmm0 vmovups 2 * SIZE(AO1), %xmm2 @@ -1678,12 +1678,12 @@ subq $-8 * SIZE, B decq I - jg .L81 + jg L(81) ALIGN_4 -.L82: +L(82): testq $4, MM - jle .L83 + jle L(83) vmovups 0 * SIZE(AO1), %xmm0 vmovups 2 * SIZE(AO1), %xmm2 @@ -1695,9 +1695,9 @@ subq $-4 * SIZE, B ALIGN_4 -.L83: +L(83): testq $2, MM - jle .L84 + jle L(84) vmovups 0 * SIZE(AO1), %xmm0 @@ -1707,25 +1707,25 @@ subq $-2 * SIZE, B ALIGN_4 -.L84: +L(84): testq $1, MM - jle .L999 + jle L(999) vmovsd 0 * SIZE(AO1), %xmm0 vmovlpd %xmm0, -16 * SIZE(B) - jmp .L999 + jmp L(999) ALIGN_4 -.L85: +L(85): vmovups -1 * SIZE(AO1), %xmm0 movq M, I sarq $3, I - jle .L86 + jle L(86) ALIGN_4 -.L86: +L(86): vmovups 1 * SIZE(AO1), %xmm1 vmovups 3 * SIZE(AO1), %xmm2 @@ -1749,12 +1749,12 @@ subq $-8 * SIZE, B decq I - jg .L86 + jg L(86) ALIGN_4 -.L87: +L(87): testq $4, M - jle .L88 + jle L(88) vmovups 1 * SIZE(AO1), %xmm1 vmovups 3 * SIZE(AO1), %xmm2 @@ -1771,9 +1771,9 @@ addq $4 * SIZE, B ALIGN_4 -.L88: +L(88): testq $2, M - jle .L89 + jle L(89) vmovups 1 * SIZE(AO1), %xmm1 @@ -1787,16 +1787,16 @@ subq $-2 * SIZE, B ALIGN_4 -.L89: +L(89): testq $1, M - jle .L999 + jle L(999) vshufpd $1, %xmm0, %xmm0 , %xmm0 vmovlpd %xmm0, -16 * SIZE(B) ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI vmovups 0(%rsp), %xmm6 vmovups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/dgemm_tcopy_2.S b/kernel/x86_64/dgemm_tcopy_2.S index 9881610d7d..33504368e1 100644 --- a/kernel/x86_64/dgemm_tcopy_2.S +++ b/kernel/x86_64/dgemm_tcopy_2.S @@ -142,10 +142,10 @@ leaq (, N, SIZE), M8 cmpq $2, N - jl .L40 + jl L(40) ALIGN_4 -.L31: +L(31): subq $2, N movq A, AO1 @@ -157,10 +157,10 @@ movq M, I sarq $3, I - jle .L34 + jle L(34) ALIGN_4 -.L33: +L(33): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 2 * SIZE(AO1) #endif @@ -201,12 +201,12 @@ addq $8 * SIZE, AO2 decq I - jg .L33 + jg L(33) ALIGN_4 -.L34: +L(34): testq $4, M - jle .L36 + jle L(36) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -223,9 +223,9 @@ leaq (B0, M8, 4), B0 ALIGN_4 -.L36: +L(36): testq $2, M - jle .L38 + jle L(38) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(0 * SIZE, AO2, %xmm1) @@ -238,9 +238,9 @@ leaq (B0, M8, 2), B0 ALIGN_4 -.L38: +L(38): testq $1, M - jle .L39 + jle L(39) movsd 0 * SIZE(AO1), %xmm0 movhpd 0 * SIZE(AO2), %xmm0 @@ -249,24 +249,24 @@ subq $-2 * SIZE, B3 ALIGN_4 -.L39: +L(39): cmpq $2, N - jge .L31 + jge L(31) ALIGN_4 -.L40: +L(40): cmpq $1, N - jl .L999 + jl L(999) movq A, AO1 movq B, B0 movq M, I sarq $3, I - jle .L44 + jle L(44) ALIGN_4 -.L43: +L(43): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 4 * SIZE(AO1) #endif @@ -290,12 +290,12 @@ leaq (B0, M8, 4), B0 decq I - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): testq $4, M - jle .L45 + jle L(45) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -307,9 +307,9 @@ leaq (B0, M8, 4), B0 ALIGN_4 -.L45: +L(45): testq $2, M - jle .L46 + jle L(46) MOVUPS_A1(0 * SIZE, AO1, %xmm0) @@ -318,16 +318,16 @@ addq $2 * SIZE, AO1 ALIGN_4 -.L46: +L(46): testq $1, M - jle .L999 + jle L(999) movsd 0 * SIZE(AO1), %xmm0 movlpd %xmm0, -16 * SIZE(B3) ALIGN_4 -.L999: +L(999): popq %rbp popq %r13 popq %r12 diff --git a/kernel/x86_64/dgemm_tcopy_4.S b/kernel/x86_64/dgemm_tcopy_4.S index 98ba6473c5..46c4e3bdd0 100644 --- a/kernel/x86_64/dgemm_tcopy_4.S +++ b/kernel/x86_64/dgemm_tcopy_4.S @@ -152,10 +152,10 @@ leaq (, N, SIZE), M8 cmpq $4, N - jl .L30 + jl L(30) ALIGN_4 -.L21: +L(21): subq $4, N movq A, AO1 @@ -167,10 +167,10 @@ movq M, I sarq $3, I - jle .L24 + jle L(24) ALIGN_4 -.L23: +L(23): #ifdef PREFETCH PREFETCH PREFETCHSIZE * SIZE(AO1) #endif @@ -248,12 +248,12 @@ leaq (B0, M8, 8), B0 decq I - jg .L23 + jg L(23) ALIGN_4 -.L24: +L(24): testq $4, M - jle .L26 + jle L(26) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -280,9 +280,9 @@ leaq (B0, M8, 4), B0 ALIGN_4 -.L26: +L(26): testq $2, M - jle .L28 + jle L(28) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A2(0 * SIZE, AO1, LDA, 1, %xmm1) @@ -299,9 +299,9 @@ subq $-8 * SIZE, B2 ALIGN_4 -.L28: +L(28): testq $1, M - jle .L29 + jle L(29) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -316,14 +316,14 @@ subq $-4 * SIZE, B3 ALIGN_4 -.L29: +L(29): cmpq $4, N - jge .L21 + jge L(21) ALIGN_4 -.L30: +L(30): cmpq $2, N - jl .L40 + jl L(40) subq $2, N @@ -336,10 +336,10 @@ movq M, I sarq $3, I - jle .L34 + jle L(34) ALIGN_4 -.L33: +L(33): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 2 * SIZE(AO1) #endif @@ -381,12 +381,12 @@ leaq (B0, M8, 8), B0 decq I - jg .L33 + jg L(33) ALIGN_4 -.L34: +L(34): testq $4, M - jle .L36 + jle L(36) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -403,9 +403,9 @@ leaq (B0, M8, 4), B0 ALIGN_4 -.L36: +L(36): testq $2, M - jle .L38 + jle L(38) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(0 * SIZE, AO2, %xmm1) @@ -418,9 +418,9 @@ subq $-4 * SIZE, B2 ALIGN_4 -.L38: +L(38): testq $1, M - jle .L40 + jle L(40) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -431,9 +431,9 @@ subq $-2 * SIZE, B3 ALIGN_4 -.L40: +L(40): cmpq $1, N - jl .L999 + jl L(999) movq A, AO1 @@ -441,10 +441,10 @@ movq M, I sarq $3, I - jle .L44 + jle L(44) ALIGN_4 -.L43: +L(43): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 4 * SIZE(AO1) #endif @@ -467,12 +467,12 @@ leaq (B0, M8, 8), B0 decq I - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): testq $4, M - jle .L45 + jle L(45) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -484,9 +484,9 @@ leaq (B0, M8, 4), B0 ALIGN_4 -.L45: +L(45): testq $2, M - jle .L46 + jle L(46) MOVUPS_A1(0 * SIZE, AO1, %xmm0) @@ -496,17 +496,17 @@ subq $-2 * SIZE, B2 ALIGN_4 -.L46: +L(46): testq $1, M - jle .L999 + jle L(999) movsd 0 * SIZE(AO1), %xmm0 movlpd %xmm0, -16 * SIZE(B3) - jmp .L999 + jmp L(999) ALIGN_4 -.L999: +L(999): popq %rbp popq %r12 popq %r13 diff --git a/kernel/x86_64/dgemm_tcopy_8.S b/kernel/x86_64/dgemm_tcopy_8.S index db97db7f18..42240dc404 100644 --- a/kernel/x86_64/dgemm_tcopy_8.S +++ b/kernel/x86_64/dgemm_tcopy_8.S @@ -139,10 +139,10 @@ leaq (, N, SIZE), M8 cmpq $8, N - jl .L20 + jl L(20) ALIGN_4 -.L11: +L(11): subq $8, N movq A, AO1 @@ -154,10 +154,10 @@ movq M, I sarq $3, I - jle .L14 + jle L(14) ALIGN_4 -.L13: +L(13): #ifdef PREFETCH PREFETCH PREFETCHSIZE * SIZE(AO1) #endif @@ -307,12 +307,12 @@ leaq (B0, M8, 8), B0 decq I - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): testq $4, M - jle .L16 + jle L(16) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -359,9 +359,9 @@ subq $-32 * SIZE, B1 ALIGN_4 -.L16: +L(16): testq $2, M - jle .L18 + jle L(18) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A2(0 * SIZE, AO1, LDA, 1, %xmm1) @@ -388,9 +388,9 @@ subq $-16 * SIZE, B2 ALIGN_4 -.L18: +L(18): testq $1, M - jle .L19 + jle L(19) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -417,14 +417,14 @@ subq $-8 * SIZE, B3 ALIGN_4 -.L19: +L(19): cmpq $8, N - jge .L11 + jge L(11) ALIGN_4 -.L20: +L(20): cmpq $4, N - jl .L30 + jl L(30) subq $4, N @@ -437,10 +437,10 @@ movq M, I sarq $3, I - jle .L24 + jle L(24) ALIGN_4 -.L23: +L(23): #ifdef PREFETCH PREFETCH PREFETCHSIZE * SIZE(AO1) #endif @@ -518,12 +518,12 @@ leaq (B0, M8, 8), B0 decq I - jg .L23 + jg L(23) ALIGN_4 -.L24: +L(24): testq $4, M - jle .L26 + jle L(26) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -550,9 +550,9 @@ subq $-16 * SIZE, B1 ALIGN_4 -.L26: +L(26): testq $2, M - jle .L28 + jle L(28) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A2(0 * SIZE, AO1, LDA, 1, %xmm1) @@ -569,9 +569,9 @@ subq $-8 * SIZE, B2 ALIGN_4 -.L28: +L(28): testq $1, M - jle .L30 + jle L(30) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO1, LDA), %xmm1 @@ -586,9 +586,9 @@ subq $-4 * SIZE, B3 ALIGN_4 -.L30: +L(30): cmpq $2, N - jl .L40 + jl L(40) subq $2, N @@ -601,10 +601,10 @@ movq M, I sarq $3, I - jle .L34 + jle L(34) ALIGN_4 -.L33: +L(33): #ifdef PREFETCH PREFETCH PREFETCHSIZE * SIZE(AO1) #endif @@ -646,12 +646,12 @@ leaq (B0, M8, 8), B0 decq I - jg .L33 + jg L(33) ALIGN_4 -.L34: +L(34): testq $4, M - jle .L36 + jle L(36) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -668,9 +668,9 @@ subq $-8 * SIZE, B1 ALIGN_4 -.L36: +L(36): testq $2, M - jle .L38 + jle L(38) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(0 * SIZE, AO2, %xmm1) @@ -683,9 +683,9 @@ subq $-4 * SIZE, B2 ALIGN_4 -.L38: +L(38): testq $1, M - jle .L40 + jle L(40) movsd 0 * SIZE(AO1), %xmm0 movsd 0 * SIZE(AO2), %xmm1 @@ -696,9 +696,9 @@ subq $-2 * SIZE, B3 ALIGN_4 -.L40: +L(40): cmpq $1, N - jl .L999 + jl L(999) movq A, AO1 @@ -706,10 +706,10 @@ movq M, I sarq $3, I - jle .L44 + jle L(44) ALIGN_4 -.L43: +L(43): #ifdef PREFETCH PREFETCH PREFETCHSIZE * 8 * SIZE(AO1) #endif @@ -732,12 +732,12 @@ leaq (B0, M8, 8), B0 decq I - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): testq $4, M - jle .L45 + jle L(45) MOVUPS_A1(0 * SIZE, AO1, %xmm0) MOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -749,9 +749,9 @@ subq $-4 * SIZE, B1 ALIGN_4 -.L45: +L(45): testq $2, M - jle .L46 + jle L(46) MOVUPS_A1(0 * SIZE, AO1, %xmm0) @@ -761,17 +761,17 @@ subq $-2 * SIZE, B2 ALIGN_4 -.L46: +L(46): testq $1, M - jle .L999 + jle L(999) movsd 0 * SIZE(AO1), %xmm0 movlpd %xmm0, -16 * SIZE(B3) - jmp .L999 + jmp L(999) ALIGN_4 -.L999: +L(999): popq %rbp popq %r12 popq %r13 diff --git a/kernel/x86_64/dgemm_tcopy_8_bulldozer.S b/kernel/x86_64/dgemm_tcopy_8_bulldozer.S index a9dd25389f..1f4a8fb75f 100644 --- a/kernel/x86_64/dgemm_tcopy_8_bulldozer.S +++ b/kernel/x86_64/dgemm_tcopy_8_bulldozer.S @@ -122,10 +122,10 @@ leaq (, N, SIZE), M8 cmpq $8, N - jl .L20 + jl L(20) ALIGN_4 -.L11: +L(11): subq $8, N movq A, AO1 @@ -137,10 +137,10 @@ movq M, I sarq $3, I - jle .L14 + jle L(14) ALIGN_4 -.L13: +L(13): prefetchnta A_PRE(AO1) VMOVUPS_A1(0 * SIZE, AO1, %xmm0) @@ -239,12 +239,12 @@ leaq (B0, M8, 8), B0 decq I - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): testq $4, M - jle .L16 + jle L(16) VMOVUPS_A1(0 * SIZE, AO1, %xmm0) VMOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -291,9 +291,9 @@ subq $-32 * SIZE, B1 ALIGN_4 -.L16: +L(16): testq $2, M - jle .L18 + jle L(18) VMOVUPS_A1(0 * SIZE, AO1, %xmm0) VMOVUPS_A2(0 * SIZE, AO1, LDA, 1, %xmm1) @@ -320,9 +320,9 @@ subq $-16 * SIZE, B2 ALIGN_4 -.L18: +L(18): testq $1, M - jle .L19 + jle L(19) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO1, LDA), %xmm1 @@ -349,14 +349,14 @@ subq $-8 * SIZE, B3 ALIGN_4 -.L19: +L(19): cmpq $8, N - jge .L11 + jge L(11) ALIGN_4 -.L20: +L(20): cmpq $4, N - jl .L30 + jl L(30) subq $4, N @@ -369,10 +369,10 @@ movq M, I sarq $3, I - jle .L24 + jle L(24) ALIGN_4 -.L23: +L(23): VMOVUPS_A1(0 * SIZE, AO1, %xmm0) VMOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -420,12 +420,12 @@ leaq (B0, M8, 8), B0 decq I - jg .L23 + jg L(23) ALIGN_4 -.L24: +L(24): testq $4, M - jle .L26 + jle L(26) VMOVUPS_A1(0 * SIZE, AO1, %xmm0) VMOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -452,9 +452,9 @@ subq $-16 * SIZE, B1 ALIGN_4 -.L26: +L(26): testq $2, M - jle .L28 + jle L(28) VMOVUPS_A1(0 * SIZE, AO1, %xmm0) VMOVUPS_A2(0 * SIZE, AO1, LDA, 1, %xmm1) @@ -471,9 +471,9 @@ subq $-8 * SIZE, B2 ALIGN_4 -.L28: +L(28): testq $1, M - jle .L30 + jle L(30) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO1, LDA), %xmm1 @@ -488,9 +488,9 @@ subq $-4 * SIZE, B3 ALIGN_4 -.L30: +L(30): cmpq $2, N - jl .L40 + jl L(40) subq $2, N @@ -503,10 +503,10 @@ movq M, I sarq $3, I - jle .L34 + jle L(34) ALIGN_4 -.L33: +L(33): VMOVUPS_A1(0 * SIZE, AO1, %xmm0) VMOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -533,12 +533,12 @@ leaq (B0, M8, 8), B0 decq I - jg .L33 + jg L(33) ALIGN_4 -.L34: +L(34): testq $4, M - jle .L36 + jle L(36) VMOVUPS_A1(0 * SIZE, AO1, %xmm0) VMOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -555,9 +555,9 @@ subq $-8 * SIZE, B1 ALIGN_4 -.L36: +L(36): testq $2, M - jle .L38 + jle L(38) VMOVUPS_A1(0 * SIZE, AO1, %xmm0) VMOVUPS_A1(0 * SIZE, AO2, %xmm1) @@ -570,9 +570,9 @@ subq $-4 * SIZE, B2 ALIGN_4 -.L38: +L(38): testq $1, M - jle .L40 + jle L(40) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO2), %xmm1 @@ -583,9 +583,9 @@ subq $-2 * SIZE, B3 ALIGN_4 -.L40: +L(40): cmpq $1, N - jl .L999 + jl L(999) movq A, AO1 @@ -593,10 +593,10 @@ movq M, I sarq $3, I - jle .L44 + jle L(44) ALIGN_4 -.L43: +L(43): VMOVUPS_A1(0 * SIZE, AO1, %xmm0) VMOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -612,12 +612,12 @@ leaq (B0, M8, 8), B0 decq I - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): testq $4, M - jle .L45 + jle L(45) VMOVUPS_A1(0 * SIZE, AO1, %xmm0) VMOVUPS_A1(2 * SIZE, AO1, %xmm1) @@ -629,9 +629,9 @@ subq $-4 * SIZE, B1 ALIGN_4 -.L45: +L(45): testq $2, M - jle .L46 + jle L(46) VMOVUPS_A1(0 * SIZE, AO1, %xmm0) @@ -641,17 +641,17 @@ subq $-2 * SIZE, B2 ALIGN_4 -.L46: +L(46): testq $1, M - jle .L999 + jle L(999) vmovsd 0 * SIZE(AO1), %xmm0 vmovsd %xmm0, -16 * SIZE(B3) - jmp .L999 + jmp L(999) ALIGN_4 -.L999: +L(999): popq %rbp popq %r12 popq %r13 diff --git a/kernel/x86_64/dgemv_n.S b/kernel/x86_64/dgemv_n.S index 58dd43bbb1..aa139a042a 100644 --- a/kernel/x86_64/dgemv_n.S +++ b/kernel/x86_64/dgemv_n.S @@ -164,20 +164,20 @@ movq LDA,LDAX movq X,XX -.L0t: +L(0t): xorq I,I addq $1,I salq $21,I subq I,MMM movq I,M - jge .L00t + jge L(00t) movq MMM,M addq M, I - jle .L999x + jle L(999x) movq I, M -.L00t: +L(00t): movq XX,X movq AA,A movq NN,N @@ -205,9 +205,9 @@ #endif testq N, N # if n <= 0 goto END - jle .L999 + jle L(999) testq M, M # if n <= 0 goto END - jle .L999 + jle L(999) #if !defined(COPY_FORCE) && !defined(ALIGNED_ACCESS) #ifndef NOCOPY_UNALIGNED @@ -217,7 +217,7 @@ #endif testq %rax, %rax cmoveq Y, BUFFER - je .L10 + je L(10) #endif movq BUFFER, Y1 @@ -229,7 +229,7 @@ sarq $4, %rax ALIGN_3 -.L01: +L(01): movapd %xmm4, 0 * SIZE(Y1) movapd %xmm4, 2 * SIZE(Y1) movapd %xmm4, 4 * SIZE(Y1) @@ -240,10 +240,10 @@ movapd %xmm4, 14 * SIZE(Y1) subq $-16 * SIZE, Y1 decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L10: +L(10): #ifdef ALIGNED_ACCESS leaq SIZE(BUFFER), %rax @@ -251,16 +251,16 @@ cmovne %rax, BUFFER testq $SIZE, LDA - jne .L50 + jne L(50) #endif #if GEMV_UNROLL >= 8 cmpq $8, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $8, N leaq 16 * SIZE(BUFFER), Y1 @@ -328,7 +328,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L1X + je L(1X) movsd -16 * SIZE(A1), %xmm4 movsd -16 * SIZE(A1, LDA), %xmm5 @@ -366,12 +366,12 @@ addq $SIZE, Y1 ALIGN_3 -.L1X: +L(1X): #endif movq MM, I sarq $3, I - jle .L15 + jle L(15) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm5) @@ -384,10 +384,10 @@ MOVUPS_YL1(-10 * SIZE, Y1, %xmm3) decq I - jle .L14 + jle L(14) ALIGN_3 -.L13: +L(13): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -552,10 +552,10 @@ subq $1, I BRANCH - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): mulpd %xmm8, %xmm4 addpd %xmm4, %xmm0 MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm4) @@ -673,9 +673,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L15: +L(15): testq $4, MM - je .L16 + je L(16) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm5) @@ -744,9 +744,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L16: +L(16): testq $2, MM - je .L17 + je L(17) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm5) @@ -785,9 +785,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L17: +L(17): testq $1, MM - je .L18 + je L(18) movsd -16 * SIZE(A1), %xmm4 movsd -16 * SIZE(A1, LDA), %xmm5 @@ -821,23 +821,23 @@ movsd %xmm0, -16 * SIZE(Y1) ALIGN_3 -.L18: +L(18): cmpq $8, N - jge .L11 + jge L(11) ALIGN_3 -.L20: +L(20): #endif #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 4 ALIGN_3 -.L21: +L(21): #endif subq $4, N @@ -883,7 +883,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L2X + je L(2X) movsd -16 * SIZE(A1), %xmm4 movsd -16 * SIZE(A1, LDA), %xmm5 @@ -908,12 +908,12 @@ addq $SIZE, Y1 ALIGN_3 -.L2X: +L(2X): #endif movq MM, I sarq $3, I - jle .L25 + jle L(25) MOVUPS_A1(-16 * SIZE, A1, %xmm0) MOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -931,10 +931,10 @@ MOVUPS_A2(-10 * SIZE, A1, LDA, 1, %xmm7) decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -1027,10 +1027,10 @@ subq $1, I BRANCH - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): mulpd %xmm12, %xmm0 addpd %xmm0, %xmm8 MOVUPS_A1(-16 * SIZE, A2, %xmm0) @@ -1088,9 +1088,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L25: +L(25): testq $4, MM - je .L26 + je L(26) MOVUPS_A1(-16 * SIZE, A1, %xmm0) MOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -1135,9 +1135,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L26: +L(26): testq $2, MM - je .L27 + je L(27) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm9) @@ -1162,12 +1162,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L27: +L(27): testq $1, MM #if GEMV_UNROLL == 4 - je .L28 + je L(28) #else - je .L30 + je L(30) #endif movsd -16 * SIZE(Y1), %xmm0 @@ -1190,25 +1190,25 @@ ALIGN_3 #if GEMV_UNROLL == 4 -.L28: +L(28): cmpq $4, N - jge .L21 + jge L(21) ALIGN_3 #endif -.L30: +L(30): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L40 + jl L(40) #if GEMV_UNROLL == 2 ALIGN_3 -.L31: +L(31): #endif subq $2, N @@ -1242,7 +1242,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L3X + je L(3X) movsd -16 * SIZE(A1), %xmm4 movsd -16 * SIZE(A2), %xmm5 @@ -1261,12 +1261,12 @@ addq $SIZE, Y1 ALIGN_3 -.L3X: +L(3X): #endif movq MM, I sarq $3, I - jle .L35 + jle L(35) MOVUPS_A1(-16 * SIZE, A1, %xmm0) MOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -1284,10 +1284,10 @@ MOVUPS_A1(-10 * SIZE, A2, %xmm7) decq I - jle .L34 + jle L(34) ALIGN_3 -.L33: +L(33): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -1344,10 +1344,10 @@ subq $1, I BRANCH - jg .L33 + jg L(33) ALIGN_3 -.L34: +L(34): mulpd %xmm12, %xmm0 addpd %xmm0, %xmm8 mulpd %xmm12, %xmm1 @@ -1375,9 +1375,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L35: +L(35): testq $4, MM - je .L36 + je L(36) MOVUPS_A1(-16 * SIZE, A1, %xmm0) @@ -1406,9 +1406,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L36: +L(36): testq $2, MM - je .L37 + je L(37) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A1(-16 * SIZE, A2, %xmm9) @@ -1427,12 +1427,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L37: +L(37): testq $1, MM #if GEMV_UNROLL == 2 - je .L38 + je L(38) #else - je .L40 + je L(40) #endif movsd -16 * SIZE(Y1), %xmm0 @@ -1449,16 +1449,16 @@ ALIGN_3 #if GEMV_UNROLL == 2 -.L38: +L(38): cmpq $2, N - jge .L31 + jge L(31) ALIGN_3 #endif -.L40: +L(40): cmpq $1, N - jl .L900 + jl L(900) #endif leaq 16 * SIZE(BUFFER), Y1 @@ -1482,7 +1482,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L4X + je L(4X) movsd -16 * SIZE(A1), %xmm4 movsd -16 * SIZE(Y1), %xmm0 @@ -1496,12 +1496,12 @@ addq $SIZE, Y1 ALIGN_3 -.L4X: +L(4X): #endif movq MM, I sarq $3, I - jle .L45 + jle L(45) MOVUPS_A1(-16 * SIZE, A1, %xmm0) MOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -1514,10 +1514,10 @@ MOVUPS_YL1(-10 * SIZE, Y1, %xmm11) decq I - jle .L44 + jle L(44) ALIGN_3 -.L43: +L(43): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -1555,10 +1555,10 @@ subq $1, I BRANCH - jg .L43 + jg L(43) ALIGN_3 -.L44: +L(44): mulpd %xmm12, %xmm0 addpd %xmm0, %xmm8 MOVUPS_YS1(-16 * SIZE, Y1, %xmm8) @@ -1576,9 +1576,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L45: +L(45): testq $4, MM - je .L46 + je L(46) MOVUPS_A1(-16 * SIZE, A1, %xmm0) MOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -1597,9 +1597,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L46: +L(46): testq $2, MM - je .L47 + je L(47) MOVUPS_A1(-16 * SIZE, A1, %xmm8) @@ -1614,9 +1614,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L47: +L(47): testq $1, MM - je .L900 + je L(900) movsd -16 * SIZE(Y1), %xmm0 movsd -16 * SIZE(A1), %xmm8 @@ -1628,17 +1628,17 @@ ALIGN_3 #ifdef ALIGNED_ACCESS - jmp .L900 + jmp L(900) ALIGN_3 -.L50: +L(50): #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L60 + jl L(60) ALIGN_3 -.L51: +L(51): subq $4, N @@ -1682,7 +1682,7 @@ mulpd %xmm0, %xmm15 testq $SIZE, A - je .L5X + je L(5X) movsd -16 * SIZE(A1), %xmm4 movsd -16 * SIZE(A1, LDA), %xmm5 @@ -1707,13 +1707,13 @@ addq $SIZE, Y1 ALIGN_3 -.L5X: +L(5X): movhpd -16 * SIZE(A1, LDA), %xmm8 movhpd -16 * SIZE(A2, LDA), %xmm9 movq MM, I sarq $3, I - jle .L55 + jle L(55) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm5) @@ -1725,10 +1725,10 @@ MOVUPS_YL1(-10 * SIZE, Y1, %xmm3) decq I - jle .L54 + jle L(54) ALIGN_3 -.L53: +L(53): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -1829,10 +1829,10 @@ subq $1, I BRANCH - jg .L53 + jg L(53) ALIGN_3 -.L54: +L(54): mulpd %xmm12, %xmm4 addpd %xmm4, %xmm0 MOVUPS_A1(-10 * SIZE, A1, %xmm7) @@ -1904,9 +1904,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L55: +L(55): testq $4, MM - je .L56 + je L(56) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm5) @@ -1957,9 +1957,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L56: +L(56): testq $2, MM - je .L57 + je L(57) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A2(-15 * SIZE, A1, LDA, 1, %xmm5) @@ -1988,9 +1988,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L57: +L(57): testq $1, MM - je .L58 + je L(58) movsd -16 * SIZE(Y1), %xmm0 @@ -2011,23 +2011,23 @@ movsd %xmm0, -16 * SIZE(Y1) ALIGN_3 -.L58: +L(58): cmpq $4, N - jge .L51 + jge L(51) ALIGN_3 -.L60: +L(60): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L70 + jl L(70) #if GEMV_UNROLL == 2 ALIGN_3 -.L61: +L(61): #endif subq $2, N @@ -2060,7 +2060,7 @@ mulpd %xmm0, %xmm13 testq $SIZE, A - je .L6X + je L(6X) movsd -16 * SIZE(A1), %xmm4 movsd -16 * SIZE(A2), %xmm5 @@ -2079,12 +2079,12 @@ addq $SIZE, Y1 ALIGN_3 -.L6X: +L(6X): movhpd -16 * SIZE(A2), %xmm8 movq MM, I sarq $3, I - jle .L65 + jle L(65) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm5) @@ -2096,10 +2096,10 @@ MOVUPS_YL1(-10 * SIZE, Y1, %xmm3) decq I - jle .L64 + jle L(64) ALIGN_3 -.L63: +L(63): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -2160,10 +2160,10 @@ subq $1, I BRANCH - jg .L63 + jg L(63) ALIGN_3 -.L64: +L(64): mulpd %xmm12, %xmm4 addpd %xmm4, %xmm0 MOVUPS_A1(-10 * SIZE, A1, %xmm7) @@ -2203,9 +2203,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L65: +L(65): testq $4, MM - je .L66 + je L(66) MOVUPS_A1(-16 * SIZE, A1, %xmm4) @@ -2238,9 +2238,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L66: +L(66): testq $2, MM - je .L67 + je L(67) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-15 * SIZE, A2, %xmm5) @@ -2261,12 +2261,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L67: +L(67): testq $1, MM #if GEMV_UNROLL == 2 - je .L68 + je L(68) #else - je .L70 + je L(70) #endif movsd -16 * SIZE(Y1), %xmm0 @@ -2283,16 +2283,16 @@ ALIGN_3 #if GEMV_UNROLL == 2 -.L68: +L(68): cmpq $2, N - jge .L61 + jge L(61) ALIGN_3 #endif -.L70: +L(70): cmpq $1, N - jl .L900 + jl L(900) #endif @@ -2316,7 +2316,7 @@ mulpd %xmm0, %xmm12 testq $SIZE, A - je .L7X + je L(7X) movsd -16 * SIZE(A1), %xmm4 movsd -16 * SIZE(Y1), %xmm0 @@ -2330,11 +2330,11 @@ addq $SIZE, Y1 ALIGN_3 -.L7X: +L(7X): movq MM, I sarq $3, I - jle .L75 + jle L(75) MOVUPS_A1(-16 * SIZE, A1, %xmm0) MOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -2347,10 +2347,10 @@ MOVUPS_YL1(-10 * SIZE, Y1, %xmm11) decq I - jle .L74 + jle L(74) ALIGN_3 -.L73: +L(73): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -2388,10 +2388,10 @@ subq $1, I BRANCH - jg .L73 + jg L(73) ALIGN_3 -.L74: +L(74): mulpd %xmm12, %xmm0 addpd %xmm0, %xmm8 MOVUPS_YS1(-16 * SIZE, Y1, %xmm8) @@ -2409,9 +2409,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L75: +L(75): testq $4, MM - je .L76 + je L(76) MOVUPS_A1(-16 * SIZE, A1, %xmm0) MOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -2430,9 +2430,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L76: +L(76): testq $2, MM - je .L77 + je L(77) MOVUPS_A1(-16 * SIZE, A1, %xmm8) @@ -2447,9 +2447,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L77: +L(77): testq $1, MM - je .L900 + je L(900) movsd -16 * SIZE(Y1), %xmm0 movsd -16 * SIZE(A1), %xmm8 @@ -2462,19 +2462,19 @@ ALIGN_3 -.L900: +L(900): #ifndef COPY_FORCE cmpq Y, BUFFER - je .L999 + je L(999) #endif movq M, TMP_M movq Y, Y1 cmpq $SIZE, INCY - jne .L950 + jne L(950) testq $SIZE, Y1 - je .L910 + je L(910) movsd (Y1), %xmm0 addsd (BUFFER), %xmm0 @@ -2484,19 +2484,19 @@ addq $SIZE, BUFFER decq TMP_M - jle .L999 + jle L(999) ALIGN_4 -.L910: +L(910): testq $SIZE, BUFFER - jne .L920 + jne L(920) movq TMP_M, %rax sarq $3, %rax - jle .L914 + jle L(914) ALIGN_3 -.L912: +L(912): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE) * 4 + PREOFFSET(Y1) #endif @@ -2529,15 +2529,15 @@ addq $8 * SIZE, BUFFER decq %rax - jg .L912 + jg L(912) ALIGN_3 -.L914: +L(914): testq $7, TMP_M - jle .L999 + jle L(999) testq $4, TMP_M - jle .L915 + jle L(915) movapd 0 * SIZE(Y1), %xmm0 movapd 2 * SIZE(Y1), %xmm1 @@ -2555,9 +2555,9 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L915: +L(915): testq $2, TMP_M - jle .L916 + jle L(916) movapd (Y1), %xmm0 @@ -2571,9 +2571,9 @@ addq $2 * SIZE, BUFFER ALIGN_3 -.L916: +L(916): testq $1, TMP_M - jle .L999 + jle L(999) movsd (Y1), %xmm0 @@ -2584,18 +2584,18 @@ movlpd %xmm0, (Y1) ALIGN_3 - jmp .L999 + jmp L(999) ALIGN_4 -.L920: +L(920): movapd -1 * SIZE(BUFFER), %xmm4 movq TMP_M, %rax sarq $3, %rax - jle .L924 + jle L(924) ALIGN_3 -.L922: +L(922): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE) * 4 + PREOFFSET(Y1) #endif @@ -2635,15 +2635,15 @@ addq $8 * SIZE, BUFFER decq %rax - jg .L922 + jg L(922) ALIGN_3 -.L924: +L(924): testq $7, TMP_M - jle .L999 + jle L(999) testq $4, TMP_M - jle .L925 + jle L(925) movapd 0 * SIZE(Y1), %xmm0 movapd 2 * SIZE(Y1), %xmm1 @@ -2666,9 +2666,9 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L925: +L(925): testq $2, TMP_M - jle .L926 + jle L(926) movapd (Y1), %xmm0 @@ -2686,9 +2686,9 @@ addq $2 * SIZE, BUFFER ALIGN_3 -.L926: +L(926): testq $1, TMP_M - jle .L999 + jle L(999) movsd (Y1), %xmm0 @@ -2699,12 +2699,12 @@ movlpd %xmm0, (Y1) ALIGN_3 - jmp .L999 + jmp L(999) ALIGN_4 -.L950: +L(950): testq $SIZE, BUFFER - je .L960 + je L(960) movsd (Y1), %xmm0 addsd (BUFFER), %xmm0 @@ -2714,18 +2714,18 @@ addq $SIZE, BUFFER decq TMP_M - jle .L999 + jle L(999) ALIGN_4 -.L960: +L(960): movq Y1, Y2 movq TMP_M, %rax sarq $3, %rax - jle .L964 + jle L(964) ALIGN_3 -.L962: +L(962): movsd (Y2), %xmm0 addq INCY, Y2 movhpd (Y2), %xmm0 @@ -2783,15 +2783,15 @@ addq $8 * SIZE, BUFFER decq %rax - jg .L962 + jg L(962) ALIGN_3 -.L964: +L(964): testq $7, TMP_M - jle .L999 + jle L(999) testq $4, TMP_M - jle .L965 + jle L(965) movsd (Y2), %xmm0 addq INCY, Y2 @@ -2822,9 +2822,9 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L965: +L(965): testq $2, TMP_M - jle .L966 + jle L(966) movsd (Y2), %xmm0 addq INCY, Y2 @@ -2843,9 +2843,9 @@ addq $2 * SIZE, BUFFER ALIGN_3 -.L966: +L(966): testq $1, TMP_M - jle .L999 + jle L(999) movsd (Y2), %xmm0 @@ -2856,16 +2856,16 @@ movlpd %xmm0, (Y1) ALIGN_3 -.L999: +L(999): leaq (, M, SIZE), %rax addq %rax,AA movq STACK_INCY, INCY imulq INCY, %rax addq %rax, Y - jmp .L0t + jmp L(0t) ALIGN_4 -.L999x: +L(999x): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/dgemv_n_atom.S b/kernel/x86_64/dgemv_n_atom.S index ed6a585795..1e0957b661 100644 --- a/kernel/x86_64/dgemv_n_atom.S +++ b/kernel/x86_64/dgemv_n_atom.S @@ -143,13 +143,13 @@ leaq (,LDA, SIZE), LDA testq N, N - jle .L999 + jle L(999) testq M, M - jle .L999 + jle L(999) cmpq $SIZE, INCY cmoveq Y, BUFFER - je .L10 + je L(10) movq BUFFER, Y1 xorps %xmm4, %xmm4 @@ -159,7 +159,7 @@ sarq $3, %rax ALIGN_3 -.L01: +L(01): movsd %xmm4, 0 * SIZE(Y1) movsd %xmm4, 1 * SIZE(Y1) movsd %xmm4, 2 * SIZE(Y1) @@ -171,16 +171,16 @@ addq $8 * SIZE, Y1 decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L10: +L(10): movq N, J sarq $1, J - jle .L20 + jle L(20) ALIGN_3 -.L11: +L(11): movq BUFFER, Y1 movq A, A1 @@ -199,7 +199,7 @@ movq M, I sarq $3, I - jle .L15 + jle L(15) movsd 0 * SIZE(A1), %xmm0 movsd 1 * SIZE(A1), %xmm1 @@ -221,10 +221,10 @@ mulsd %xmm14, %xmm3 decq I - jle .L14 + jle L(14) ALIGN_3 -.L13: +L(13): PREFETCH PREFETCH_SIZE * SIZE(A1) mulsd %xmm15, %xmm4 PREFETCH PREFETCH_SIZE * SIZE(A2) @@ -314,10 +314,10 @@ movsd 2 * SIZE(Y1), %xmm10 movsd %xmm11,-1 * SIZE(Y1) movsd 3 * SIZE(Y1), %xmm11 - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): mulsd %xmm15, %xmm4 addsd %xmm0, %xmm8 movsd 4 * SIZE(A1), %xmm0 @@ -393,9 +393,9 @@ movsd %xmm11,-1 * SIZE(Y1) ALIGN_3 -.L15: +L(15): testq $4, M - je .L17 + je L(17) movsd 0 * SIZE(A1), %xmm0 movsd 1 * SIZE(A1), %xmm1 @@ -440,9 +440,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L17: +L(17): testq $2, M - je .L18 + je L(18) movsd 0 * SIZE(A1), %xmm0 movsd 1 * SIZE(A1), %xmm1 @@ -470,9 +470,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L18: +L(18): testq $1, M - je .L19 + je L(19) movsd 0 * SIZE(Y1), %xmm8 @@ -488,14 +488,14 @@ movsd %xmm8, 0 * SIZE(Y1) ALIGN_3 -.L19: +L(19): decq J - jg .L11 + jg L(11) ALIGN_3 -.L20: +L(20): testq $1, N - je .L990 + je L(990) movq BUFFER, Y1 movq A, A1 @@ -505,7 +505,7 @@ movq M, I sarq $3, I - jle .L25 + jle L(25) movsd 0 * SIZE(A1), %xmm0 movsd 1 * SIZE(A1), %xmm1 @@ -527,10 +527,10 @@ mulsd %xmm14, %xmm3 decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): PREFETCH PREFETCH_SIZE * SIZE(A1) addsd %xmm0, %xmm8 @@ -583,10 +583,10 @@ addq $8 * SIZE, A1 decq I - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): addsd %xmm0, %xmm8 addsd %xmm1, %xmm9 addsd %xmm2, %xmm10 @@ -622,9 +622,9 @@ addq $8 * SIZE, A1 ALIGN_3 -.L25: +L(25): testq $4, M - je .L27 + je L(27) movsd 0 * SIZE(A1), %xmm0 movsd 1 * SIZE(A1), %xmm1 @@ -655,9 +655,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L27: +L(27): testq $2, M - je .L28 + je L(28) movsd 0 * SIZE(A1), %xmm0 movsd 1 * SIZE(A1), %xmm1 @@ -677,9 +677,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L28: +L(28): testq $1, M - je .L990 + je L(990) movsd 0 * SIZE(Y1), %xmm8 movsd 0 * SIZE(A1), %xmm0 @@ -690,18 +690,18 @@ movsd %xmm8, 0 * SIZE(Y1) ALIGN_3 -.L990: +L(990): cmpq $SIZE, INCY - je .L999 + je L(999) movq Y, Y1 movq M, %rax sarq $2, %rax - jle .L994 + jle L(994) ALIGN_3 -.L992: +L(992): movsd (Y), %xmm0 addq INCY, Y movsd (Y), %xmm1 @@ -727,12 +727,12 @@ addq INCY, Y1 decq %rax - jg .L992 + jg L(992) ALIGN_3 -.L994: +L(994): testq $2, M - jle .L996 + jle L(996) movsd (Y), %xmm0 addq INCY, Y @@ -749,9 +749,9 @@ addq INCY, Y1 ALIGN_3 -.L996: +L(996): testq $1, M - jle .L999 + jle L(999) movsd (Y), %xmm0 @@ -760,7 +760,7 @@ movsd %xmm0, (Y1) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/dgemv_n_bulldozer.S b/kernel/x86_64/dgemv_n_bulldozer.S index bc00d67fc5..24ea327a27 100644 --- a/kernel/x86_64/dgemv_n_bulldozer.S +++ b/kernel/x86_64/dgemv_n_bulldozer.S @@ -174,9 +174,9 @@ #endif testq N, N # if n <= 0 goto END - jle .L999 + jle L(999) testq M, M # if n <= 0 goto END - jle .L999 + jle L(999) #if !defined(COPY_FORCE) && !defined(ALIGNED_ACCESS) #ifndef NOCOPY_UNALIGNED @@ -186,7 +186,7 @@ #endif testq %rax, %rax cmoveq Y, BUFFER - je .L10 + je L(10) #endif movq BUFFER, Y1 @@ -198,7 +198,7 @@ sarq $4, %rax ALIGN_3 -.L01: +L(01): vmovups %xmm4, 0 * SIZE(Y1) vmovups %xmm4, 2 * SIZE(Y1) vmovups %xmm4, 4 * SIZE(Y1) @@ -209,10 +209,10 @@ vmovups %xmm4, 14 * SIZE(Y1) subq $-16 * SIZE, Y1 decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L10: +L(10): #ifdef ALIGNED_ACCESS leaq SIZE(BUFFER), %rax @@ -220,16 +220,16 @@ cmovne %rax, BUFFER testq $SIZE, LDA - jne .L50 + jne L(50) #endif #if GEMV_UNROLL >= 8 cmpq $8, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $8, N leaq 16 * SIZE(BUFFER), Y1 @@ -267,7 +267,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L1X + je L(1X) vmovsd -16 * SIZE(Y1), %xmm0 @@ -299,12 +299,12 @@ addq $SIZE, Y1 ALIGN_3 -.L1X: +L(1X): #endif movq MM, I sarq $3, I - jle .L15 + jle L(15) VMOVUPS_YL1(-16 * SIZE, Y1, %xmm0) VMOVUPS_YL1(-14 * SIZE, Y1, %xmm1) @@ -312,10 +312,10 @@ VMOVUPS_YL1(-10 * SIZE, Y1, %xmm3) decq I - jle .L14 + jle L(14) ALIGN_5 -.L13: +L(13): prefetchnta A_PRE(A1) @@ -387,10 +387,10 @@ subq $1, I BRANCH - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm8, %xmm0 vfmaddpd %xmm1 , -14 * SIZE(A1) , %xmm8, %xmm1 @@ -445,9 +445,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L15: +L(15): testq $4, MM - je .L16 + je L(16) VMOVUPS_A1(-16 * SIZE, A1, %xmm4) VMOVUPS_A1(-14 * SIZE, A1, %xmm5) @@ -501,9 +501,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L16: +L(16): testq $2, MM - je .L17 + je L(17) VMOVUPS_A1(-16 * SIZE, A1, %xmm4) VMOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm5) @@ -534,9 +534,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L17: +L(17): testq $1, MM - je .L18 + je L(18) vmovsd -16 * SIZE(A1), %xmm4 vmovsd -16 * SIZE(A1, LDA), %xmm5 @@ -565,23 +565,23 @@ vmovsd %xmm0, -16 * SIZE(Y1) ALIGN_3 -.L18: +L(18): cmpq $8, N - jge .L11 + jge L(11) ALIGN_3 -.L20: +L(20): #endif #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 4 ALIGN_3 -.L21: +L(21): #endif subq $4, N @@ -609,7 +609,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L2X + je L(2X) vmovsd -16 * SIZE(A1), %xmm4 vmovsd -16 * SIZE(A1, LDA), %xmm5 @@ -630,12 +630,12 @@ addq $SIZE, Y1 ALIGN_3 -.L2X: +L(2X): #endif movq MM, I sarq $3, I - jle .L25 + jle L(25) VMOVUPS_A1(-16 * SIZE, A1, %xmm0) VMOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -653,10 +653,10 @@ VMOVUPS_A2(-10 * SIZE, A1, LDA, 1, %xmm7) decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): @@ -722,10 +722,10 @@ subq $1, I BRANCH - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): vfmaddpd %xmm8 , %xmm12, %xmm0 , %xmm8 vfmaddpd %xmm9 , %xmm12, %xmm1 , %xmm9 @@ -767,9 +767,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L25: +L(25): testq $4, MM - je .L26 + je L(26) VMOVUPS_A1(-16 * SIZE, A1, %xmm0) VMOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -806,9 +806,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L26: +L(26): testq $2, MM - je .L27 + je L(27) VMOVUPS_A1(-16 * SIZE, A1, %xmm8) VMOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm9) @@ -829,12 +829,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L27: +L(27): testq $1, MM #if GEMV_UNROLL == 4 - je .L28 + je L(28) #else - je .L30 + je L(30) #endif vmovsd -16 * SIZE(Y1), %xmm0 @@ -853,25 +853,25 @@ ALIGN_3 #if GEMV_UNROLL == 4 -.L28: +L(28): cmpq $4, N - jge .L21 + jge L(21) ALIGN_3 #endif -.L30: +L(30): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L40 + jl L(40) #if GEMV_UNROLL == 2 ALIGN_3 -.L31: +L(31): #endif subq $2, N @@ -893,7 +893,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L3X + je L(3X) vmovsd -16 * SIZE(A1), %xmm4 vmovsd -16 * SIZE(A2), %xmm5 @@ -910,12 +910,12 @@ addq $SIZE, Y1 ALIGN_3 -.L3X: +L(3X): #endif movq MM, I sarq $3, I - jle .L35 + jle L(35) VMOVUPS_A1(-16 * SIZE, A1, %xmm0) VMOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -933,10 +933,10 @@ VMOVUPS_A1(-10 * SIZE, A2, %xmm7) decq I - jle .L34 + jle L(34) ALIGN_3 -.L33: +L(33): vfmaddpd %xmm8 , %xmm12 , %xmm0 , %xmm8 @@ -980,10 +980,10 @@ subq $1, I BRANCH - jg .L33 + jg L(33) ALIGN_3 -.L34: +L(34): vfmaddpd %xmm8 , %xmm12 , %xmm0 , %xmm8 vfmaddpd %xmm9 , %xmm12 , %xmm1 , %xmm9 vfmaddpd %xmm10, %xmm12 , %xmm2 , %xmm10 @@ -1004,9 +1004,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L35: +L(35): testq $4, MM - je .L36 + je L(36) VMOVUPS_A1(-16 * SIZE, A1, %xmm0) @@ -1032,9 +1032,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L36: +L(36): testq $2, MM - je .L37 + je L(37) VMOVUPS_A1(-16 * SIZE, A1, %xmm8) VMOVUPS_A1(-16 * SIZE, A2, %xmm9) @@ -1051,12 +1051,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L37: +L(37): testq $1, MM #if GEMV_UNROLL == 2 - je .L38 + je L(38) #else - je .L40 + je L(40) #endif vmovsd -16 * SIZE(Y1), %xmm0 @@ -1071,16 +1071,16 @@ ALIGN_3 #if GEMV_UNROLL == 2 -.L38: +L(38): cmpq $2, N - jge .L31 + jge L(31) ALIGN_3 #endif -.L40: +L(40): cmpq $1, N - jl .L900 + jl L(900) #endif leaq 16 * SIZE(BUFFER), Y1 @@ -1095,7 +1095,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L4X + je L(4X) vmovsd -16 * SIZE(A1), %xmm4 vmovsd -16 * SIZE(Y1), %xmm0 @@ -1108,12 +1108,12 @@ addq $SIZE, Y1 ALIGN_3 -.L4X: +L(4X): #endif movq MM, I sarq $3, I - jle .L45 + jle L(45) VMOVUPS_A1(-16 * SIZE, A1, %xmm0) VMOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -1126,10 +1126,10 @@ VMOVUPS_YL1(-10 * SIZE, Y1, %xmm11) decq I - jle .L44 + jle L(44) ALIGN_3 -.L43: +L(43): vfmaddpd %xmm8 , %xmm12 , %xmm0 , %xmm8 @@ -1158,10 +1158,10 @@ subq $1, I BRANCH - jg .L43 + jg L(43) ALIGN_3 -.L44: +L(44): vfmaddpd %xmm8 , %xmm12 , %xmm0 , %xmm8 vfmaddpd %xmm9 , %xmm12 , %xmm1 , %xmm9 @@ -1177,9 +1177,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L45: +L(45): testq $4, MM - je .L46 + je L(46) VMOVUPS_A1(-16 * SIZE, A1, %xmm0) VMOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -1197,9 +1197,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L46: +L(46): testq $2, MM - je .L47 + je L(47) VMOVUPS_A1(-16 * SIZE, A1, %xmm8) @@ -1213,9 +1213,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L47: +L(47): testq $1, MM - je .L900 + je L(900) vmovsd -16 * SIZE(Y1), %xmm0 vmovsd -16 * SIZE(A1), %xmm8 @@ -1226,17 +1226,17 @@ ALIGN_3 #ifdef ALIGNED_ACCESS - jmp .L900 + jmp L(900) ALIGN_3 -.L50: +L(50): #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L60 + jl L(60) ALIGN_3 -.L51: +L(51): subq $4, N @@ -1262,7 +1262,7 @@ vmulpd %xmm0, %xmm15 , %xmm15 testq $SIZE, A - je .L5X + je L(5X) vmovsd -16 * SIZE(A1), %xmm4 vmovsd -16 * SIZE(A1, LDA), %xmm5 @@ -1283,13 +1283,13 @@ addq $SIZE, Y1 ALIGN_3 -.L5X: +L(5X): vmovhpd -16 * SIZE(A1, LDA), %xmm8, %xmm8 vmovhpd -16 * SIZE(A2, LDA), %xmm9, %xmm9 movq MM, I sarq $3, I - jle .L55 + jle L(55) VMOVUPS_A1(-16 * SIZE, A1, %xmm4) VMOVUPS_A1(-14 * SIZE, A1, %xmm5) @@ -1301,10 +1301,10 @@ VMOVUPS_YL1(-10 * SIZE, Y1, %xmm3) decq I - jle .L54 + jle L(54) ALIGN_3 -.L53: +L(53): @@ -1381,11 +1381,11 @@ subq $1, I BRANCH - jg .L53 + jg L(53) ALIGN_3 -.L54: +L(54): vfmaddpd %xmm0 , %xmm12 , %xmm4 , %xmm0 VMOVUPS_A1(-10 * SIZE, A1, %xmm7) @@ -1442,9 +1442,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L55: +L(55): testq $4, MM - je .L56 + je L(56) VMOVUPS_A1(-16 * SIZE, A1, %xmm4) VMOVUPS_A1(-14 * SIZE, A1, %xmm5) @@ -1487,9 +1487,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L56: +L(56): testq $2, MM - je .L57 + je L(57) VMOVUPS_A1(-16 * SIZE, A1, %xmm4) VMOVUPS_A2(-15 * SIZE, A1, LDA, 1, %xmm5) @@ -1514,9 +1514,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L57: +L(57): testq $1, MM - je .L58 + je L(58) vmovsd -16 * SIZE(Y1), %xmm0 @@ -1533,23 +1533,23 @@ vmovsd %xmm0, -16 * SIZE(Y1) ALIGN_3 -.L58: +L(58): cmpq $4, N - jge .L51 + jge L(51) ALIGN_3 -.L60: +L(60): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L70 + jl L(70) #if GEMV_UNROLL == 2 ALIGN_3 -.L61: +L(61): #endif subq $2, N @@ -1570,7 +1570,7 @@ vmulpd %xmm0, %xmm13 , %xmm13 testq $SIZE, A - je .L6X + je L(6X) vmovsd -16 * SIZE(A1), %xmm4 vmovsd -16 * SIZE(A2), %xmm5 @@ -1587,12 +1587,12 @@ addq $SIZE, Y1 ALIGN_3 -.L6X: +L(6X): vmovhpd -16 * SIZE(A2), %xmm8, %xmm8 movq MM, I sarq $3, I - jle .L65 + jle L(65) VMOVUPS_A1(-16 * SIZE, A1, %xmm4) VMOVUPS_A1(-14 * SIZE, A1, %xmm5) @@ -1604,10 +1604,10 @@ VMOVUPS_YL1(-10 * SIZE, Y1, %xmm3) decq I - jle .L64 + jle L(64) ALIGN_3 -.L63: +L(63): vfmaddpd %xmm0 , %xmm12 , %xmm4 , %xmm0 VMOVUPS_A1(-10 * SIZE, A1, %xmm7) @@ -1654,10 +1654,10 @@ subq $1, I BRANCH - jg .L63 + jg L(63) ALIGN_3 -.L64: +L(64): vfmaddpd %xmm0 , %xmm12 , %xmm4 , %xmm0 VMOVUPS_A1(-10 * SIZE, A1, %xmm7) @@ -1690,9 +1690,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L65: +L(65): testq $4, MM - je .L66 + je L(66) VMOVUPS_A1(-16 * SIZE, A1, %xmm4) @@ -1721,9 +1721,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L66: +L(66): testq $2, MM - je .L67 + je L(67) VMOVUPS_A1(-16 * SIZE, A1, %xmm4) VMOVUPS_A1(-15 * SIZE, A2, %xmm5) @@ -1742,12 +1742,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L67: +L(67): testq $1, MM #if GEMV_UNROLL == 2 - je .L68 + je L(68) #else - je .L70 + je L(70) #endif vmovsd -16 * SIZE(Y1), %xmm0 @@ -1762,16 +1762,16 @@ ALIGN_3 #if GEMV_UNROLL == 2 -.L68: +L(68): cmpq $2, N - jge .L61 + jge L(61) ALIGN_3 #endif -.L70: +L(70): cmpq $1, N - jl .L900 + jl L(900) #endif @@ -1786,7 +1786,7 @@ vmulpd %xmm0, %xmm12 , %xmm12 testq $SIZE, A - je .L7X + je L(7X) vmovsd -16 * SIZE(A1), %xmm4 vmovsd -16 * SIZE(Y1), %xmm0 @@ -1799,11 +1799,11 @@ addq $SIZE, Y1 ALIGN_3 -.L7X: +L(7X): movq MM, I sarq $3, I - jle .L75 + jle L(75) VMOVUPS_A1(-16 * SIZE, A1, %xmm0) VMOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -1816,10 +1816,10 @@ VMOVUPS_YL1(-10 * SIZE, Y1, %xmm11) decq I - jle .L74 + jle L(74) ALIGN_3 -.L73: +L(73): vfmaddpd %xmm8 , %xmm12 , %xmm0 , %xmm8 VMOVUPS_A1( -8 * SIZE, A1, %xmm0) @@ -1849,10 +1849,10 @@ subq $1, I BRANCH - jg .L73 + jg L(73) ALIGN_3 -.L74: +L(74): vfmaddpd %xmm8 , %xmm12 , %xmm0 , %xmm8 VMOVUPS_YS1(-16 * SIZE, Y1, %xmm8) @@ -1867,9 +1867,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L75: +L(75): testq $4, MM - je .L76 + je L(76) VMOVUPS_A1(-16 * SIZE, A1, %xmm0) VMOVUPS_A1(-14 * SIZE, A1, %xmm1) @@ -1886,9 +1886,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L76: +L(76): testq $2, MM - je .L77 + je L(77) VMOVUPS_A1(-16 * SIZE, A1, %xmm8) @@ -1902,9 +1902,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L77: +L(77): testq $1, MM - je .L900 + je L(900) vmovsd -16 * SIZE(Y1), %xmm0 vmovsd -16 * SIZE(A1), %xmm8 @@ -1916,17 +1916,17 @@ ALIGN_3 -.L900: +L(900): #ifndef COPY_FORCE cmpq Y, BUFFER - je .L999 + je L(999) #endif cmpq $SIZE, INCY - jne .L950 + jne L(950) testq $SIZE, Y - je .L910 + je L(910) vmovsd (Y), %xmm0 vaddsd (BUFFER), %xmm0, %xmm0 @@ -1936,19 +1936,19 @@ addq $SIZE, BUFFER decq M - jle .L999 + jle L(999) ALIGN_4 -.L910: +L(910): testq $SIZE, BUFFER - jne .L920 + jne L(920) movq M, %rax sarq $3, %rax - jle .L914 + jle L(914) ALIGN_3 -.L912: +L(912): vmovups 0 * SIZE(Y), %xmm0 vmovups 2 * SIZE(Y), %xmm1 @@ -1975,15 +1975,15 @@ addq $8 * SIZE, BUFFER decq %rax - jg .L912 + jg L(912) ALIGN_3 -.L914: +L(914): testq $7, M - jle .L999 + jle L(999) testq $4, M - jle .L915 + jle L(915) vmovups 0 * SIZE(Y), %xmm0 vmovups 2 * SIZE(Y), %xmm1 @@ -2001,9 +2001,9 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L915: +L(915): testq $2, M - jle .L916 + jle L(916) vmovups (Y), %xmm0 @@ -2017,9 +2017,9 @@ addq $2 * SIZE, BUFFER ALIGN_3 -.L916: +L(916): testq $1, M - jle .L999 + jle L(999) vmovsd (Y), %xmm0 @@ -2030,18 +2030,18 @@ vmovsd %xmm0, (Y) ALIGN_3 - jmp .L999 + jmp L(999) ALIGN_4 -.L920: +L(920): vmovups -1 * SIZE(BUFFER), %xmm4 movq M, %rax sarq $3, %rax - jle .L924 + jle L(924) ALIGN_3 -.L922: +L(922): vmovups 0 * SIZE(Y), %xmm0 vmovups 2 * SIZE(Y), %xmm1 @@ -2075,15 +2075,15 @@ addq $8 * SIZE, BUFFER decq %rax - jg .L922 + jg L(922) ALIGN_3 -.L924: +L(924): testq $7, M - jle .L999 + jle L(999) testq $4, M - jle .L925 + jle L(925) vmovups 0 * SIZE(Y), %xmm0 vmovups 2 * SIZE(Y), %xmm1 @@ -2106,9 +2106,9 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L925: +L(925): testq $2, M - jle .L926 + jle L(926) vmovups (Y), %xmm0 @@ -2126,9 +2126,9 @@ addq $2 * SIZE, BUFFER ALIGN_3 -.L926: +L(926): testq $1, M - jle .L999 + jle L(999) vmovsd (Y), %xmm0 @@ -2139,12 +2139,12 @@ vmovsd %xmm0, (Y) ALIGN_3 - jmp .L999 + jmp L(999) ALIGN_4 -.L950: +L(950): testq $SIZE, BUFFER - je .L960 + je L(960) vmovsd (Y), %xmm0 vaddsd (BUFFER), %xmm0, %xmm0 @@ -2154,18 +2154,18 @@ addq $SIZE, BUFFER decq M - jle .L999 + jle L(999) ALIGN_4 -.L960: +L(960): movq Y, Y1 movq M, %rax sarq $3, %rax - jle .L964 + jle L(964) ALIGN_3 -.L962: +L(962): vmovsd (Y), %xmm0 addq INCY, Y vmovhpd (Y), %xmm0, %xmm0 @@ -2223,15 +2223,15 @@ addq $8 * SIZE, BUFFER decq %rax - jg .L962 + jg L(962) ALIGN_3 -.L964: +L(964): testq $7, M - jle .L999 + jle L(999) testq $4, M - jle .L965 + jle L(965) vmovsd (Y), %xmm0 addq INCY, Y @@ -2262,9 +2262,9 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L965: +L(965): testq $2, M - jle .L966 + jle L(966) vmovsd (Y), %xmm0 addq INCY, Y @@ -2283,9 +2283,9 @@ addq $2 * SIZE, BUFFER ALIGN_3 -.L966: +L(966): testq $1, M - jle .L999 + jle L(999) vmovsd (Y), %xmm0 @@ -2296,7 +2296,7 @@ vmovsd %xmm0, (Y1) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/dgemv_t.S b/kernel/x86_64/dgemv_t.S index 9277774161..5bdb1bfb0c 100644 --- a/kernel/x86_64/dgemv_t.S +++ b/kernel/x86_64/dgemv_t.S @@ -168,19 +168,19 @@ -.L0x: +L(0x): xorq M,M addq $1,M salq $21,M subq M,MMM - jge .L00 + jge L(00) movq MMM,%rax addq M,%rax - jle .L999x + jle L(999x) movq %rax,M -.L00: +L(00): movq LDAX,LDA movq NN,N movq AA,A @@ -200,15 +200,15 @@ subq $-16 * SIZE, A testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) movq BUFFER, X1 #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L01 + je L(01) movsd (X), %xmm0 addq INCX, X @@ -217,18 +217,18 @@ addq $1 * SIZE, BUFFER addq $2 * SIZE, X1 decq M - jle .L10 + jle L(10) ALIGN_4 -.L01: +L(01): #endif movq M, I sarq $3, I - jle .L05 + jle L(05) ALIGN_4 -.L02: +L(02): movsd (X), %xmm0 addq INCX, X movhpd (X), %xmm0 @@ -256,38 +256,38 @@ addq $8 * SIZE, X1 decq I - jg .L02 + jg L(02) ALIGN_4 -.L05: +L(05): movq M, I andq $7, I - jle .L10 + jle L(10) ALIGN_2 -.L06: +L(06): movsd (X), %xmm0 addq INCX, X movsd %xmm0, 0 * SIZE(X1) addq $SIZE, X1 decq I - jg .L06 + jg L(06) ALIGN_4 -.L10: +L(10): movq Y, Y1 #ifdef ALIGNED_ACCESS testq $SIZE, LDA - jne .L50 + jne L(50) #endif #if GEMV_UNROLL >= 8 cmpq $8, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $8, N leaq 16 * SIZE(BUFFER), X1 @@ -311,7 +311,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L1X + je L(1X) movsd -16 * SIZE(X1), %xmm12 @@ -345,12 +345,12 @@ addq $SIZE, X1 ALIGN_3 -.L1X: +L(1X): #endif movq M, I sarq $3, I - jle .L15 + jle L(15) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm9) @@ -361,10 +361,10 @@ MOVUPS_XL1(-14 * SIZE, X1, %xmm13) decq I - jle .L13 + jle L(13) ALIGN_4 -.L12: +L(12): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -515,10 +515,10 @@ addq $8 * SIZE, X1 decq I - jg .L12 + jg L(12) ALIGN_4 -.L13: +L(13): mulpd %xmm12, %xmm8 addpd %xmm8, %xmm0 MOVUPS_A1(-16 * SIZE, A2, %xmm8) @@ -626,9 +626,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L15: +L(15): testq $4, M - jle .L16 + jle L(16) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm9) @@ -691,9 +691,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L16: +L(16): testq $2, M - jle .L17 + jle L(17) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm9) @@ -729,9 +729,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L17: +L(17): testq $1, M - je .L18 + je L(18) movsd -16 * SIZE(X1), %xmm12 @@ -761,7 +761,7 @@ addsd %xmm11, %xmm7 ALIGN_4 -.L18: +L(18): #ifdef HAVE_SSE3 haddpd %xmm1, %xmm0 haddpd %xmm3, %xmm2 @@ -796,7 +796,7 @@ mulpd ALPHA, %xmm6 cmpq $SIZE, INCY - jne .L19 + jne L(19) movsd 0 * SIZE(Y), %xmm8 movhpd 1 * SIZE(Y), %xmm8 @@ -824,11 +824,11 @@ addq $8 * SIZE, Y1 cmpq $8, N - jge .L11 - jmp .L20 + jge L(11) + jmp L(20) ALIGN_4 -.L19: +L(19): movsd (Y), %xmm8 addq INCY, Y movhpd (Y), %xmm8 @@ -869,21 +869,21 @@ addq INCY, Y1 cmpq $8, N - jge .L11 + jge L(11) ALIGN_4 -.L20: +L(20): #endif #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 4 ALIGN_3 -.L21: +L(21): #endif subq $4, N @@ -904,7 +904,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L2X + je L(2X) movsd -16 * SIZE(X1), %xmm12 @@ -926,12 +926,12 @@ addq $SIZE, X1 ALIGN_3 -.L2X: +L(2X): #endif movq M, I sarq $3, I - jle .L25 + jle L(25) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm9) @@ -942,10 +942,10 @@ MOVUPS_XL1(-14 * SIZE, X1, %xmm13) decq I - jle .L23 + jle L(23) ALIGN_4 -.L22: +L(22): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -1028,10 +1028,10 @@ addq $8 * SIZE, X1 decq I - jg .L22 + jg L(22) ALIGN_4 -.L23: +L(23): mulpd %xmm12, %xmm8 addpd %xmm8, %xmm0 MOVUPS_A1(-14 * SIZE, A1, %xmm8) @@ -1087,9 +1087,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L25: +L(25): testq $4, M - jle .L26 + jle L(26) MOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -1126,9 +1126,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L26: +L(26): testq $2, M - jle .L27 + jle L(27) MOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -1150,9 +1150,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L27: +L(27): testq $1, M - je .L28 + je L(28) movsd -16 * SIZE(X1), %xmm12 @@ -1170,7 +1170,7 @@ addsd %xmm11, %xmm3 ALIGN_4 -.L28: +L(28): #ifdef HAVE_SSE3 haddpd %xmm1, %xmm0 haddpd %xmm3, %xmm2 @@ -1191,7 +1191,7 @@ mulpd ALPHA, %xmm2 cmpq $SIZE, INCY - jne .L29 + jne L(29) movsd 0 * SIZE(Y), %xmm4 movhpd 1 * SIZE(Y), %xmm4 @@ -1210,12 +1210,12 @@ #if GEMV_UNROLL == 4 cmpq $4, N - jge .L21 + jge L(21) #endif - jmp .L30 + jmp L(30) ALIGN_4 -.L29: +L(29): movsd (Y), %xmm4 addq INCY, Y movhpd (Y), %xmm4 @@ -1239,22 +1239,22 @@ #if GEMV_UNROLL == 4 cmpq $4, N - jge .L21 + jge L(21) #endif ALIGN_4 -.L30: +L(30): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L40 + jl L(40) #if GEMV_UNROLL == 2 ALIGN_3 -.L31: +L(31): #endif subq $2, N @@ -1276,7 +1276,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L3X + je L(3X) movsd -16 * SIZE(X1), %xmm12 @@ -1292,12 +1292,12 @@ addq $SIZE, X1 ALIGN_3 -.L3X: +L(3X): #endif movq M, I sarq $3, I - jle .L35 + jle L(35) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A1(-16 * SIZE, A2, %xmm9) @@ -1308,10 +1308,10 @@ MOVUPS_XL1(-14 * SIZE, X1, %xmm13) decq I - jle .L33 + jle L(33) ALIGN_4 -.L32: +L(32): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -1361,10 +1361,10 @@ addq $8 * SIZE, X1 decq I - jg .L32 + jg L(32) ALIGN_4 -.L33: +L(33): mulpd %xmm12, %xmm8 addpd %xmm8, %xmm0 MOVUPS_A1(-12 * SIZE, A1, %xmm8) @@ -1396,9 +1396,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L35: +L(35): testq $4, M - jle .L36 + jle L(36) MOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -1423,9 +1423,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L36: +L(36): testq $2, M - jle .L37 + jle L(37) MOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -1441,9 +1441,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L37: +L(37): testq $1, M - je .L38 + je L(38) movsd -16 * SIZE(X1), %xmm12 @@ -1455,7 +1455,7 @@ addsd %xmm9, %xmm1 ALIGN_4 -.L38: +L(38): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -1485,13 +1485,13 @@ #if GEMV_UNROLL == 2 cmpq $2, N - jge .L31 + jge L(31) #endif ALIGN_4 -.L40: +L(40): cmpq $1, N - jl .L999 + jl L(999) #endif @@ -1506,7 +1506,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L4X + je L(4X) movsd -16 * SIZE(X1), %xmm12 movsd -16 * SIZE(A1), %xmm8 @@ -1517,12 +1517,12 @@ addq $SIZE, X1 ALIGN_3 -.L4X: +L(4X): #endif movq M, I sarq $3, I - jle .L45 + jle L(45) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A1(-14 * SIZE, A1, %xmm9) @@ -1533,10 +1533,10 @@ MOVUPS_XL1(-14 * SIZE, X1, %xmm13) decq I - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -1567,10 +1567,10 @@ addq $8 * SIZE, X1 decq I - jg .L42 + jg L(42) ALIGN_4 -.L43: +L(43): mulpd %xmm12, %xmm8 MOVUPS_XL1(-12 * SIZE, X1, %xmm12) addpd %xmm8, %xmm0 @@ -1587,9 +1587,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L45: +L(45): testq $4, M - jle .L46 + jle L(46) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A1(-14 * SIZE, A1, %xmm9) @@ -1606,9 +1606,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L46: +L(46): testq $2, M - jle .L47 + jle L(47) MOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -1620,9 +1620,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L47: +L(47): testq $1, M - je .L48 + je L(48) movsd -16 * SIZE(X1), %xmm12 @@ -1631,7 +1631,7 @@ addsd %xmm8, %xmm0 ALIGN_4 -.L48: +L(48): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -1658,17 +1658,17 @@ addq INCY, Y1 #ifdef ALIGNED_ACCESS - jmp .L999 + jmp L(999) ALIGN_4 -.L50: +L(50): #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L60 + jl L(60) ALIGN_3 -.L51: +L(51): subq $4, N leaq 16 * SIZE(BUFFER), X1 @@ -1688,7 +1688,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L5X + je L(5X) movsd -16 * SIZE(X1), %xmm12 @@ -1710,7 +1710,7 @@ addq $SIZE, X1 ALIGN_3 -.L5X: +L(5X): #endif movhpd -16 * SIZE(A1, LDA), %xmm8 @@ -1718,7 +1718,7 @@ movq M, I sarq $3, I - jle .L55 + jle L(55) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A2(-15 * SIZE, A1, LDA, 1, %xmm5) @@ -1729,10 +1729,10 @@ MOVUPS_XL1(-14 * SIZE, X1, %xmm13) decq I - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -1825,10 +1825,10 @@ addq $8 * SIZE, X1 decq I - jg .L52 + jg L(52) ALIGN_4 -.L53: +L(53): mulpd %xmm12, %xmm4 addpd %xmm4, %xmm0 MOVUPS_A1(-14 * SIZE, A1, %xmm4) @@ -1895,9 +1895,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L55: +L(55): testq $4, M - jle .L56 + jle L(56) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A2(-15 * SIZE, A1, LDA, 1, %xmm5) @@ -1941,9 +1941,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L56: +L(56): testq $2, M - jle .L57 + jle L(57) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A2(-15 * SIZE, A1, LDA, 1, %xmm5) @@ -1970,9 +1970,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L57: +L(57): testq $1, M - je .L58 + je L(58) movsd -16 * SIZE(X1), %xmm12 @@ -1990,7 +1990,7 @@ addsd %xmm9, %xmm3 ALIGN_4 -.L58: +L(58): #ifdef HAVE_SSE3 haddpd %xmm1, %xmm0 haddpd %xmm3, %xmm2 @@ -2011,7 +2011,7 @@ mulpd ALPHA, %xmm2 cmpq $SIZE, INCY - jne .L59 + jne L(59) movsd 0 * SIZE(Y), %xmm4 movhpd 1 * SIZE(Y), %xmm4 @@ -2029,11 +2029,11 @@ addq $4 * SIZE, Y1 cmpq $4, N - jge .L51 - jmp .L60 + jge L(51) + jmp L(60) ALIGN_4 -.L59: +L(59): movsd (Y), %xmm4 addq INCY, Y movhpd (Y), %xmm4 @@ -2055,21 +2055,21 @@ movhpd %xmm2, (Y1) addq INCY, Y1 cmpq $4, N - jge .L51 + jge L(51) ALIGN_4 -.L60: +L(60): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L70 + jl L(70) #if GEMV_UNROLL == 2 ALIGN_3 -.L61: +L(61): #endif subq $2, N @@ -2091,7 +2091,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L6X + je L(6X) movsd -16 * SIZE(X1), %xmm12 @@ -2107,14 +2107,14 @@ addq $SIZE, X1 ALIGN_3 -.L6X: +L(6X): #endif movhpd -16 * SIZE(A2), %xmm8 movq M, I sarq $3, I - jle .L65 + jle L(65) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-15 * SIZE, A2, %xmm5) @@ -2125,10 +2125,10 @@ MOVUPS_XL1(-14 * SIZE, X1, %xmm13) decq I - jle .L63 + jle L(63) ALIGN_4 -.L62: +L(62): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -2182,10 +2182,10 @@ addq $8 * SIZE, X1 decq I - jg .L62 + jg L(62) ALIGN_4 -.L63: +L(63): mulpd %xmm12, %xmm4 addpd %xmm4, %xmm0 MOVUPS_A1(-12 * SIZE, A1, %xmm4) @@ -2221,9 +2221,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L65: +L(65): testq $4, M - jle .L66 + jle L(66) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-15 * SIZE, A2, %xmm5) @@ -2251,9 +2251,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L66: +L(66): testq $2, M - jle .L67 + jle L(67) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-15 * SIZE, A2, %xmm5) @@ -2272,9 +2272,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L67: +L(67): testq $1, M - je .L68 + je L(68) movsd -16 * SIZE(X1), %xmm12 @@ -2286,7 +2286,7 @@ addsd %xmm8, %xmm1 ALIGN_4 -.L68: +L(68): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -2316,13 +2316,13 @@ #if GEMV_UNROLL == 2 cmpq $2, N - jge .L61 + jge L(61) #endif ALIGN_4 -.L70: +L(70): cmpq $1, N - jl .L999 + jl L(999) #endif @@ -2337,7 +2337,7 @@ #ifdef ALIGNED_ACCESS testq $SIZE, A - je .L7X + je L(7X) movsd -16 * SIZE(X1), %xmm12 @@ -2349,11 +2349,11 @@ addq $SIZE, X1 ALIGN_3 -.L7X: +L(7X): #endif movq M, I sarq $3, I - jle .L75 + jle L(75) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm5) @@ -2364,10 +2364,10 @@ MOVUPS_XL1(-14 * SIZE, X1, %xmm13) decq I - jle .L73 + jle L(73) ALIGN_4 -.L72: +L(72): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -2398,10 +2398,10 @@ addq $8 * SIZE, X1 decq I - jg .L72 + jg L(72) ALIGN_4 -.L73: +L(73): mulpd %xmm12, %xmm4 MOVUPS_XL1(-12 * SIZE, X1, %xmm12) addpd %xmm4, %xmm0 @@ -2418,9 +2418,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L75: +L(75): testq $4, M - jle .L76 + jle L(76) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm5) @@ -2437,9 +2437,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L76: +L(76): testq $2, M - jle .L77 + jle L(77) MOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -2451,9 +2451,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L77: +L(77): testq $1, M - je .L78 + je L(78) movsd -16 * SIZE(X1), %xmm12 @@ -2462,7 +2462,7 @@ addsd %xmm4, %xmm0 ALIGN_4 -.L78: +L(78): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -2490,13 +2490,13 @@ #endif ALIGN_4 -.L999: +L(999): leaq (, M, SIZE), %rax addq %rax,AA - jmp .L0x; + jmp L(0x); ALIGN_4 -.L999x: +L(999x): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/dgemv_t_atom.S b/kernel/x86_64/dgemv_t_atom.S index 1e63c427b3..c2cde272ea 100644 --- a/kernel/x86_64/dgemv_t_atom.S +++ b/kernel/x86_64/dgemv_t_atom.S @@ -142,22 +142,22 @@ movq Y, Y1 testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) cmpq $SIZE, INCX cmoveq X, BUFFER - je .L10 + je L(10) movq BUFFER, X1 movq M, I sarq $3, I - jle .L05 + jle L(05) ALIGN_3 -.L02: +L(02): movsd (X), %xmm0 addq INCX, X movsd (X), %xmm1 @@ -189,16 +189,16 @@ addq $8 * SIZE, X1 decq I - jg .L02 + jg L(02) ALIGN_3 -.L05: +L(05): movq M, I andq $7, I - jle .L10 + jle L(10) ALIGN_3 -.L06: +L(06): movsd (X), %xmm0 addq INCX, X @@ -206,16 +206,16 @@ addq $SIZE, X1 decq I - jg .L06 + jg L(06) ALIGN_3 -.L10: +L(10): movq N, J sarq $1, J - jle .L20 + jle L(20) ALIGN_3 -.L11: +L(11): movq A, A1 leaq (A, LDA, 1), A2 leaq (A, LDA, 2), A @@ -228,7 +228,7 @@ movq M, I sarq $3, I - jle .L14 + jle L(14) movsd 0 * SIZE(X1), %xmm4 movsd 0 * SIZE(A1), %xmm8 @@ -254,10 +254,10 @@ movsd 5 * SIZE(X1), %xmm5 decq I - jle .L13 + jle L(13) ALIGN_3 -.L12: +L(12): PREFETCH PREFETCH_SIZE * SIZE(A1) addsd %xmm8, %xmm0 PREFETCH PREFETCH_SIZE * SIZE(A2) @@ -328,10 +328,10 @@ mulsd %xmm5, %xmm13 movsd 5 * SIZE(X1), %xmm5 - jg .L12 + jg L(12) ALIGN_3 -.L13: +L(13): addsd %xmm8, %xmm0 movsd 4 * SIZE(A1), %xmm8 mulsd %xmm6, %xmm10 @@ -381,9 +381,9 @@ addsd %xmm15, %xmm1 ALIGN_4 -.L14: +L(14): testq $4, M - je .L16 + je L(16) movsd 0 * SIZE(X1), %xmm4 movsd 0 * SIZE(A1), %xmm8 @@ -426,9 +426,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L16: +L(16): testq $2, M - je .L17 + je L(17) movsd 0 * SIZE(X1), %xmm4 movsd 0 * SIZE(A1), %xmm8 @@ -454,9 +454,9 @@ ALIGN_4 -.L17: +L(17): testq $1, M - je .L19 + je L(19) movsd 0 * SIZE(X1), %xmm4 @@ -470,7 +470,7 @@ addsd %xmm12, %xmm1 ALIGN_4 -.L19: +L(19): mulsd ALPHA, %xmm0 addsd (Y), %xmm0 addq INCY, Y @@ -485,12 +485,12 @@ addq INCY, Y1 decq J - jg .L11 + jg L(11) ALIGN_3 -.L20: +L(20): testq $1, N - jle .L999 + jle L(999) movq A, A1 movq BUFFER, X1 @@ -500,7 +500,7 @@ movq M, I sarq $3, I - jle .L24 + jle L(24) movsd 0 * SIZE(X1), %xmm4 movsd 0 * SIZE(A1), %xmm8 @@ -522,10 +522,10 @@ movsd 7 * SIZE(X1), %xmm7 decq I - jle .L23 + jle L(23) ALIGN_3 -.L22: +L(22): PREFETCH PREFETCH_SIZE * SIZE(A1) addsd %xmm8, %xmm0 movsd 4 * SIZE(A1), %xmm8 @@ -566,10 +566,10 @@ addq $8 * SIZE, A1 addq $8 * SIZE, X1 decq I - jg .L22 + jg L(22) ALIGN_3 -.L23: +L(23): addsd %xmm8, %xmm0 movsd 4 * SIZE(A1), %xmm8 addsd %xmm9, %xmm1 @@ -592,9 +592,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L24: +L(24): testq $4, M - je .L26 + je L(26) movsd 0 * SIZE(X1), %xmm4 movsd 0 * SIZE(A1), %xmm8 @@ -619,9 +619,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L26: +L(26): testq $2, M - je .L27 + je L(27) movsd 0 * SIZE(X1), %xmm4 movsd 0 * SIZE(A1), %xmm8 @@ -636,9 +636,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L27: +L(27): testq $1, M - je .L29 + je L(29) movsd 0 * SIZE(X1), %xmm4 movsd 0 * SIZE(A1), %xmm8 @@ -647,7 +647,7 @@ addsd %xmm8, %xmm0 ALIGN_4 -.L29: +L(29): addsd %xmm1, %xmm0 mulsd ALPHA, %xmm0 @@ -656,7 +656,7 @@ movsd %xmm0, (Y1) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/dgemv_t_bulldozer.S b/kernel/x86_64/dgemv_t_bulldozer.S index 9cd44ee2f0..279332cf22 100644 --- a/kernel/x86_64/dgemv_t_bulldozer.S +++ b/kernel/x86_64/dgemv_t_bulldozer.S @@ -173,19 +173,19 @@ -.L0x: +L(0x): xorq M,M addq $1,M salq $22,M subq M,MMM - jge .L00 + jge L(00) movq MMM,%rax addq M,%rax - jle .L999x + jle L(999x) movq %rax,M -.L00: +L(00): movq LDAX,LDA movq NN,N movq AA,A @@ -205,18 +205,18 @@ subq $-16 * SIZE, A testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) movq BUFFER, X1 movq M, I sarq $3, I - jle .L05 + jle L(05) ALIGN_4 -.L02: +L(02): vmovsd (X), %xmm0 addq INCX, X vmovhpd (X), %xmm0 , %xmm0 @@ -244,34 +244,34 @@ addq $8 * SIZE, X1 decq I - jg .L02 + jg L(02) ALIGN_4 -.L05: +L(05): movq M, I andq $7, I - jle .L10 + jle L(10) ALIGN_2 -.L06: +L(06): vmovsd (X), %xmm0 addq INCX, X vmovsd %xmm0, 0 * SIZE(X1) addq $SIZE, X1 decq I - jg .L06 + jg L(06) ALIGN_4 -.L10: +L(10): movq Y, Y1 #if GEMV_UNROLL >= 8 cmpq $8, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $8, N leaq 16 * SIZE(BUFFER), X1 @@ -292,17 +292,17 @@ movq M, I sarq $3, I - jle .L15 + jle L(15) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) VMOVUPS_XL1(-14 * SIZE, X1, %xmm13) decq I - jle .L13 + jle L(13) ALIGN_4 -.L12: +L(12): prefetchnta A_PRE(A1) prefetchnta A_PRE(A2) @@ -367,10 +367,10 @@ addq $8 * SIZE, X1 decq I - jg .L12 + jg L(12) ALIGN_4 -.L13: +L(13): vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 vfmaddpd %xmm1 , -16 * SIZE(A1 , LDA , 1) , %xmm12 , %xmm1 @@ -423,9 +423,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L15: +L(15): testq $4, M - jle .L16 + jle L(16) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -458,9 +458,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L16: +L(16): testq $2, M - jle .L17 + jle L(17) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -479,9 +479,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L17: +L(17): testq $1, M - je .L18 + je L(18) vmovsd -16 * SIZE(X1), %xmm12 @@ -507,7 +507,7 @@ ALIGN_4 -.L18: +L(18): vhaddpd %xmm1, %xmm0 , %xmm0 vhaddpd %xmm3, %xmm2 , %xmm2 vhaddpd %xmm5, %xmm4 , %xmm4 @@ -519,7 +519,7 @@ vmulpd ALPHA, %xmm6 , %xmm6 cmpq $SIZE, INCY - jne .L19 + jne L(19) vaddpd 0 * SIZE(Y), %xmm0 , %xmm0 vaddpd 2 * SIZE(Y), %xmm2 , %xmm2 @@ -534,11 +534,11 @@ addq $8 * SIZE, Y1 cmpq $8, N - jge .L11 - jmp .L20 + jge L(11) + jmp L(20) ALIGN_4 -.L19: +L(19): vmovsd (Y), %xmm8 addq INCY, Y @@ -580,21 +580,21 @@ addq INCY, Y1 cmpq $8, N - jge .L11 + jge L(11) ALIGN_4 -.L20: +L(20): #endif #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 4 ALIGN_3 -.L21: +L(21): #endif subq $4, N @@ -612,16 +612,16 @@ movq M, I sarq $3, I - jle .L25 + jle L(25) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) VMOVUPS_XL1(-14 * SIZE, X1, %xmm13) decq I - jle .L23 + jle L(23) ALIGN_4 -.L22: +L(22): prefetchnta A_PRE(A1) prefetchnta A_PRE(A2) @@ -661,10 +661,10 @@ addq $8 * SIZE, X1 decq I - jg .L22 + jg L(22) ALIGN_4 -.L23: +L(23): vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 vfmaddpd %xmm1 , -16 * SIZE(A1 , LDA , 1) , %xmm12 , %xmm1 @@ -696,9 +696,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L25: +L(25): testq $4, M - jle .L26 + jle L(26) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) VMOVUPS_XL1(-14 * SIZE, X1, %xmm13) @@ -719,9 +719,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L26: +L(26): testq $2, M - jle .L27 + jle L(27) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -735,9 +735,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L27: +L(27): testq $1, M - je .L28 + je L(28) vmovsd -16 * SIZE(X1), %xmm12 @@ -753,7 +753,7 @@ ALIGN_4 -.L28: +L(28): vhaddpd %xmm1, %xmm0 , %xmm0 vhaddpd %xmm3, %xmm2 , %xmm2 @@ -761,7 +761,7 @@ vmulpd ALPHA, %xmm2 , %xmm2 cmpq $SIZE, INCY - jne .L29 + jne L(29) vmovups 0 * SIZE(Y), %xmm4 vmovups 2 * SIZE(Y), %xmm5 @@ -776,12 +776,12 @@ #if GEMV_UNROLL == 4 cmpq $4, N - jge .L21 + jge L(21) #endif - jmp .L30 + jmp L(30) ALIGN_4 -.L29: +L(29): vmovsd (Y), %xmm4 addq INCY, Y @@ -806,22 +806,22 @@ #if GEMV_UNROLL == 4 cmpq $4, N - jge .L21 + jge L(21) #endif ALIGN_4 -.L30: +L(30): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L40 + jl L(40) #if GEMV_UNROLL == 2 ALIGN_3 -.L31: +L(31): #endif subq $2, N @@ -840,16 +840,16 @@ movq M, I sarq $3, I - jle .L35 + jle L(35) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) VMOVUPS_XL1(-14 * SIZE, X1, %xmm13) decq I - jle .L33 + jle L(33) ALIGN_4 -.L32: +L(32): prefetchnta A_PRE(A1) prefetchnta A_PRE(A2) @@ -879,10 +879,10 @@ addq $8 * SIZE, X1 decq I - jg .L32 + jg L(32) ALIGN_4 -.L33: +L(33): vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 vfmaddpd %xmm1 , -16 * SIZE(A2) , %xmm12 , %xmm1 @@ -905,9 +905,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L35: +L(35): testq $4, M - jle .L36 + jle L(36) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) VMOVUPS_XL1(-14 * SIZE, X1, %xmm13) @@ -924,9 +924,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L36: +L(36): testq $2, M - jle .L37 + jle L(37) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -938,9 +938,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L37: +L(37): testq $1, M - je .L38 + je L(38) vmovsd -16 * SIZE(X1), %xmm12 @@ -952,7 +952,7 @@ ALIGN_4 -.L38: +L(38): vaddpd %xmm2, %xmm0 , %xmm0 vaddpd %xmm3, %xmm1 , %xmm1 @@ -974,13 +974,13 @@ #if GEMV_UNROLL == 2 cmpq $2, N - jge .L31 + jge L(31) #endif ALIGN_4 -.L40: +L(40): cmpq $1, N - jl .L999 + jl L(999) #endif @@ -997,17 +997,17 @@ movq M, I sarq $3, I - jle .L45 + jle L(45) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) VMOVUPS_XL1(-14 * SIZE, X1, %xmm13) decq I - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): prefetchnta A_PRE(A1) vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 @@ -1027,10 +1027,10 @@ addq $8 * SIZE, X1 decq I - jg .L42 + jg L(42) ALIGN_4 -.L43: +L(43): vfmaddpd %xmm0 , -16 * SIZE(A1) , %xmm12 , %xmm0 vfmaddpd %xmm2 , -14 * SIZE(A1) , %xmm13 , %xmm2 @@ -1047,9 +1047,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L45: +L(45): testq $4, M - jle .L46 + jle L(46) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -1062,9 +1062,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L46: +L(46): testq $2, M - jle .L47 + jle L(47) VMOVUPS_XL1(-16 * SIZE, X1, %xmm12) @@ -1074,9 +1074,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L47: +L(47): testq $1, M - je .L48 + je L(48) vmovsd -16 * SIZE(X1), %xmm12 vmovsd -16 * SIZE(A1), %xmm8 @@ -1084,7 +1084,7 @@ vfmaddpd %xmm0, %xmm8 , %xmm12, %xmm0 ALIGN_4 -.L48: +L(48): vaddpd %xmm2, %xmm0 , %xmm0 vaddpd %xmm3, %xmm1 , %xmm1 @@ -1104,13 +1104,13 @@ ALIGN_4 -.L999: +L(999): leaq (, M, SIZE), %rax addq %rax,AA - jmp .L0x; + jmp L(0x); ALIGN_4 -.L999x: +L(999x): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/dot.S b/kernel/x86_64/dot.S index a11d25e5d1..d1dd3d58bf 100644 --- a/kernel/x86_64/dot.S +++ b/kernel/x86_64/dot.S @@ -63,16 +63,16 @@ fldz cmpq $SIZE, INCX - jne .L14 + jne L(14) cmpq $SIZE, INCY - jne .L14 + jne L(14) movq N, %rax sarq $2, %rax - jle .L15 + jle L(15) ALIGN_3 -.L16: +L(16): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -101,16 +101,16 @@ addq $4 * SIZE, X addq $4 * SIZE, Y decq %rax - jg .L16 + jg L(16) ALIGN_3 -.L15: +L(15): movq N, %rax andq $3, %rax - jle .L27 + jle L(27) ALIGN_3 -.L22: +L(22): FLD (X) addq $SIZE, X FLD (Y) @@ -118,18 +118,18 @@ addq $SIZE, Y faddp %st,%st(1) decq %rax - jg .L22 + jg L(22) - jmp .L27 + jmp L(27) ALIGN_3 -.L14: +L(14): movq N, %rax sarq $2, %rax - jle .L30 + jle L(30) ALIGN_3 -.L31: +L(31): FLD (X) addq INCX, X FLD (Y) @@ -159,16 +159,16 @@ faddp %st,%st(4) decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L30: +L(30): movq N, %rax andq $3, %rax - jle .L27 + jle L(27) ALIGN_3 -.L37: +L(37): FLD (X) addq INCX, X FLD (Y) @@ -176,10 +176,10 @@ addq INCY, Y faddp %st, %st(1) decq %rax - jg .L37 + jg L(37) ALIGN_3 -.L27: +L(27): faddp %st,%st(2) faddp %st,%st(2) faddp %st,%st(1) diff --git a/kernel/x86_64/dot_atom.S b/kernel/x86_64/dot_atom.S index 794cf14220..70cbbcdbc4 100644 --- a/kernel/x86_64/dot_atom.S +++ b/kernel/x86_64/dot_atom.S @@ -68,16 +68,16 @@ pxor %xmm2, %xmm2 cmpq $0, N pxor %xmm3, %xmm3 - jle .L999 + jle L(999) cmpq $SIZE, INCX - jne .L50 + jne L(50) cmpq $SIZE, INCY - jne .L50 + jne L(50) movq N, %rax sarq $3, %rax - jle .L14 + jle L(14) movsd 0 * SIZE(X), %xmm4 movsd 0 * SIZE(Y), %xmm8 @@ -94,11 +94,11 @@ mulsd %xmm11, %xmm7 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -147,10 +147,10 @@ addq $8 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): addsd %xmm4, %xmm0 movsd 4 * SIZE(X), %xmm4 addsd %xmm5, %xmm1 @@ -178,12 +178,12 @@ addq $ 8 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $7, N - jle .L999 + jle L(999) testq $4, N - jle .L16 + jle L(16) movsd 0 * SIZE(X), %xmm4 movsd 0 * SIZE(Y), %xmm8 @@ -208,9 +208,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $2, N - jle .L17 + jle L(17) movsd 0 * SIZE(X), %xmm4 movsd 0 * SIZE(Y), %xmm8 @@ -226,25 +226,25 @@ addq $2 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $1, N - jle .L999 + jle L(999) movsd 0 * SIZE(X), %xmm4 movsd 0 * SIZE(Y), %xmm8 mulsd %xmm8, %xmm4 addsd %xmm4, %xmm0 - jmp .L999 + jmp L(999) ALIGN_3 -.L50: +L(50): movq N, %rax sarq $2, %rax - jle .L55 + jle L(55) ALIGN_3 -.L53: +L(53): movsd 0 * SIZE(X), %xmm4 addq INCX, X mulsd 0 * SIZE(Y), %xmm4 @@ -268,26 +268,26 @@ addsd %xmm7, %xmm3 decq %rax - jg .L53 + jg L(53) ALIGN_3 -.L55: +L(55): movq N, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_3 -.L56: +L(56): movsd 0 * SIZE(X), %xmm4 addq INCX, X mulsd 0 * SIZE(Y), %xmm4 addq INCY, Y addsd %xmm4, %xmm0 decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L999: +L(999): addsd %xmm1, %xmm0 addsd %xmm3, %xmm2 addsd %xmm2, %xmm0 diff --git a/kernel/x86_64/dot_sse.S b/kernel/x86_64/dot_sse.S index 6886222598..69f45e46d0 100644 --- a/kernel/x86_64/dot_sse.S +++ b/kernel/x86_64/dot_sse.S @@ -69,21 +69,21 @@ xorps %xmm3, %xmm3 cmpq $0, N - jle .L999 + jle L(999) cmpq $SIZE, INCX - jne .L50 + jne L(50) cmpq $SIZE, INCY - jne .L50 + jne L(50) subq $-32 * SIZE, X subq $-32 * SIZE, Y cmpq $3, N - jle .L17 + jle L(17) testq $SIZE, Y - je .L05 + je L(05) movss -32 * SIZE(X), %xmm0 mulss -32 * SIZE(Y), %xmm0 @@ -92,9 +92,9 @@ decq N ALIGN_2 -.L05: +L(05): testq $2 * SIZE, Y - je .L10 + je L(10) #ifdef movsd xorps %xmm4, %xmm4 @@ -108,24 +108,24 @@ addq $2 * SIZE, X addq $2 * SIZE, Y subq $2, N - jle .L999 + jle L(999) ALIGN_2 -.L10: +L(10): #ifdef ALIGNED_ACCESS testq $2 * SIZE, X - jne .L30 + jne L(30) testq $SIZE, X - jne .L20 + jne L(20) #else testq $3 * SIZE, X - jne .L20 + jne L(20) #endif movq N, %rax sarq $5, %rax - jle .L14 + jle L(14) movaps -32 * SIZE(X), %xmm4 movaps -28 * SIZE(X), %xmm5 @@ -138,11 +138,11 @@ movaps -4 * SIZE(X), %xmm11 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -195,10 +195,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): mulps -32 * SIZE(Y), %xmm4 addps %xmm4, %xmm0 mulps -28 * SIZE(Y), %xmm5 @@ -221,12 +221,12 @@ subq $-32 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $31, N - jle .L999 + jle L(999) testq $16, N - jle .L15 + jle L(15) movaps -32 * SIZE(X), %xmm4 movaps -28 * SIZE(X), %xmm5 @@ -246,9 +246,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $8, N - jle .L16 + jle L(16) movaps -32 * SIZE(X), %xmm4 movaps -28 * SIZE(X), %xmm5 @@ -262,9 +262,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $4, N - jle .L17 + jle L(17) movaps -32 * SIZE(X), %xmm4 mulps -32 * SIZE(Y), %xmm4 @@ -275,9 +275,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $2, N - jle .L18 + jle L(18) #ifdef movsd xorps %xmm4, %xmm4 @@ -295,17 +295,17 @@ addq $2 * SIZE, Y ALIGN_3 -.L18: +L(18): testq $1, N - jle .L999 + jle L(999) movss -32 * SIZE(X), %xmm4 mulss -32 * SIZE(Y), %xmm4 addss %xmm4, %xmm0 - jmp .L999 + jmp L(999) ALIGN_3 -.L20: +L(20): #ifdef ALIGNED_ACCESS movaps -33 * SIZE(X), %xmm4 @@ -313,7 +313,7 @@ movq N, %rax sarq $5, %rax - jle .L24 + jle L(24) movaps -32 * SIZE(X), %xmm5 movaps -28 * SIZE(X), %xmm6 @@ -325,11 +325,11 @@ movaps -8 * SIZE(X), %xmm11 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -398,10 +398,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): movss %xmm5, %xmm4 pshufd $0x39, %xmm4, %xmm4 mulps -32 * SIZE(Y), %xmm4 @@ -447,12 +447,12 @@ subq $-32 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $31, N - jle .L999 + jle L(999) testq $16, N - jle .L25 + jle L(25) movaps -32 * SIZE(X), %xmm5 movaps -28 * SIZE(X), %xmm6 @@ -483,9 +483,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $8, N - jle .L26 + jle L(26) movaps -32 * SIZE(X), %xmm5 movaps -28 * SIZE(X), %xmm6 @@ -506,9 +506,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $4, N - jle .L27 + jle L(27) movaps -32 * SIZE(X), %xmm5 movss %xmm5, %xmm4 @@ -521,9 +521,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $2, N - jle .L28 + jle L(28) #ifdef movsd xorps %xmm8, %xmm8 @@ -540,26 +540,26 @@ addq $2 * SIZE, Y ALIGN_3 -.L28: +L(28): testq $1, N - jle .L999 + jle L(999) pshufd $0x39, %xmm4, %xmm4 mulss -32 * SIZE(Y), %xmm4 addss %xmm4, %xmm0 - jmp .L999 + jmp L(999) ALIGN_3 -.L30: +L(30): testq $SIZE, X - jne .L40 + jne L(40) movhps -32 * SIZE(X), %xmm4 addq $2 * SIZE, X movq N, %rax sarq $5, %rax - jle .L34 + jle L(34) movaps -32 * SIZE(X), %xmm5 movaps -28 * SIZE(X), %xmm6 @@ -571,11 +571,11 @@ movaps -8 * SIZE(X), %xmm11 decq %rax - jle .L32 + jle L(32) ALIGN_3 -.L31: +L(31): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -636,10 +636,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L32: +L(32): SHUFPD_1 %xmm5, %xmm4 mulps -32 * SIZE(Y), %xmm4 addps %xmm4, %xmm0 @@ -677,12 +677,12 @@ subq $-32 * SIZE, Y ALIGN_3 -.L34: +L(34): testq $31, N - jle .L999 + jle L(999) testq $16, N - jle .L35 + jle L(35) movaps -32 * SIZE(X), %xmm5 movaps -28 * SIZE(X), %xmm6 @@ -709,9 +709,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L35: +L(35): testq $8, N - jle .L36 + jle L(36) movaps -32 * SIZE(X), %xmm5 movaps -28 * SIZE(X), %xmm6 @@ -728,9 +728,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L36: +L(36): testq $4, N - jle .L37 + jle L(37) movaps -32 * SIZE(X), %xmm5 @@ -743,9 +743,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L37: +L(37): testq $2, N - jle .L38 + jle L(38) xorps %xmm5, %xmm5 movhlps %xmm4, %xmm5 @@ -758,23 +758,23 @@ addq $2 * SIZE, Y ALIGN_3 -.L38: +L(38): testq $1, N - jle .L999 + jle L(999) movss -34 * SIZE(X), %xmm4 mulss -32 * SIZE(Y), %xmm4 addss %xmm4, %xmm0 - jmp .L999 + jmp L(999) ALIGN_3 -.L40: +L(40): movaps -35 * SIZE(X), %xmm4 addq $SIZE, X movq N, %rax sarq $5, %rax - jle .L44 + jle L(44) movaps -32 * SIZE(X), %xmm5 movaps -28 * SIZE(X), %xmm6 @@ -786,11 +786,11 @@ movaps -8 * SIZE(X), %xmm11 decq %rax - jle .L42 + jle L(42) ALIGN_3 -.L41: +L(41): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -859,10 +859,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L42: +L(42): movss %xmm5, %xmm4 shufps $0x93, %xmm5, %xmm4 mulps -32 * SIZE(Y), %xmm4 @@ -908,12 +908,12 @@ subq $-32 * SIZE, Y ALIGN_3 -.L44: +L(44): testq $31, N - jle .L999 + jle L(999) testq $16, N - jle .L45 + jle L(45) movaps -32 * SIZE(X), %xmm5 movaps -28 * SIZE(X), %xmm6 @@ -944,9 +944,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L45: +L(45): testq $8, N - jle .L46 + jle L(46) movaps -32 * SIZE(X), %xmm5 movaps -28 * SIZE(X), %xmm6 @@ -967,9 +967,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L46: +L(46): testq $4, N - jle .L47 + jle L(47) movaps -32 * SIZE(X), %xmm5 movss %xmm5, %xmm4 @@ -982,9 +982,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L47: +L(47): testq $2, N - jle .L48 + jle L(48) movaps -32 * SIZE(X), %xmm5 #ifdef movsd @@ -1003,20 +1003,20 @@ addq $2 * SIZE, Y ALIGN_3 -.L48: +L(48): testq $1, N - jle .L999 + jle L(999) pshufd $0x93, %xmm4, %xmm4 mulss -32 * SIZE(Y), %xmm4 addss %xmm4, %xmm0 - jmp .L999 + jmp L(999) ALIGN_4 #else movq N, %rax sarq $5, %rax - jle .L24 + jle L(24) movlps -32 * SIZE(X), %xmm4 movhps -30 * SIZE(X), %xmm4 @@ -1037,11 +1037,11 @@ movhps -2 * SIZE(X), %xmm11 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -1102,10 +1102,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): mulps -32 * SIZE(Y), %xmm4 addps %xmm4, %xmm0 mulps -28 * SIZE(Y), %xmm5 @@ -1128,12 +1128,12 @@ subq $-32 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $31, N - jle .L999 + jle L(999) testq $16, N - jle .L25 + jle L(25) movlps -32 * SIZE(X), %xmm4 movhps -30 * SIZE(X), %xmm4 @@ -1157,9 +1157,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $8, N - jle .L26 + jle L(26) movlps -32 * SIZE(X), %xmm4 movhps -30 * SIZE(X), %xmm4 @@ -1175,9 +1175,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $4, N - jle .L27 + jle L(27) movlps -32 * SIZE(X), %xmm4 movhps -30 * SIZE(X), %xmm4 @@ -1189,9 +1189,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $2, N - jle .L28 + jle L(28) #ifdef movsd xorps %xmm4, %xmm4 @@ -1209,25 +1209,25 @@ addq $2 * SIZE, Y ALIGN_3 -.L28: +L(28): testq $1, N - jle .L999 + jle L(999) movss -32 * SIZE(X), %xmm4 mulss -32 * SIZE(Y), %xmm4 addss %xmm4, %xmm0 - jmp .L999 + jmp L(999) ALIGN_3 #endif -.L50: +L(50): movq N, %rax sarq $2, %rax - jle .L55 + jle L(55) ALIGN_3 -.L53: +L(53): movss 0 * SIZE(X), %xmm4 addq INCX, X mulss 0 * SIZE(Y), %xmm4 @@ -1251,26 +1251,26 @@ addss %xmm7, %xmm3 decq %rax - jg .L53 + jg L(53) ALIGN_3 -.L55: +L(55): movq N, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_3 -.L56: +L(56): movss 0 * SIZE(X), %xmm4 addq INCX, X mulss 0 * SIZE(Y), %xmm4 addq INCY, Y addss %xmm4, %xmm0 decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L999: +L(999): addps %xmm1, %xmm0 addps %xmm3, %xmm2 addps %xmm2, %xmm0 diff --git a/kernel/x86_64/dot_sse2.S b/kernel/x86_64/dot_sse2.S index ceb2d0c29b..49f0246205 100644 --- a/kernel/x86_64/dot_sse2.S +++ b/kernel/x86_64/dot_sse2.S @@ -69,18 +69,18 @@ xorps %xmm3, %xmm3 cmpq $0, N - jle .L999 + jle L(999) cmpq $SIZE, INCX - jne .L50 + jne L(50) cmpq $SIZE, INCY - jne .L50 + jne L(50) subq $-16 * SIZE, X subq $-16 * SIZE, Y testq $SIZE, Y - je .L10 + je L(10) movsd -16 * SIZE(X), %xmm0 mulsd -16 * SIZE(Y), %xmm0 @@ -89,13 +89,13 @@ decq N ALIGN_2 -.L10: +L(10): testq $SIZE, X - jne .L20 + jne L(20) movq N, %rax sarq $4, %rax - jle .L14 + jle L(14) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -108,11 +108,11 @@ movaps -2 * SIZE(X), %xmm11 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -165,10 +165,10 @@ subq $-16 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): mulpd -16 * SIZE(Y), %xmm4 addpd %xmm4, %xmm0 mulpd -14 * SIZE(Y), %xmm5 @@ -191,12 +191,12 @@ subq $-16 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $15, N - jle .L999 + jle L(999) testq $8, N - jle .L15 + jle L(15) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -216,9 +216,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $4, N - jle .L16 + jle L(16) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -232,9 +232,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $2, N - jle .L17 + jle L(17) movaps -16 * SIZE(X), %xmm4 @@ -245,17 +245,17 @@ addq $2 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $1, N - jle .L999 + jle L(999) movsd -16 * SIZE(X), %xmm4 mulsd -16 * SIZE(Y), %xmm4 addsd %xmm4, %xmm0 - jmp .L999 + jmp L(999) ALIGN_3 -.L20: +L(20): #ifdef ALIGNED_ACCESS @@ -264,7 +264,7 @@ movq N, %rax sarq $4, %rax - jle .L24 + jle L(24) movaps -16 * SIZE(X), %xmm5 movaps -14 * SIZE(X), %xmm6 @@ -276,11 +276,11 @@ movaps -4 * SIZE(X), %xmm11 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -341,10 +341,10 @@ subq $-16 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): SHUFPD_1 %xmm5, %xmm4 mulpd -16 * SIZE(Y), %xmm4 addpd %xmm4, %xmm0 @@ -382,12 +382,12 @@ subq $-16 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $15, N - jle .L999 + jle L(999) testq $8, N - jle .L25 + jle L(25) movaps -16 * SIZE(X), %xmm5 movaps -14 * SIZE(X), %xmm6 @@ -414,9 +414,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $4, N - jle .L26 + jle L(26) movaps -16 * SIZE(X), %xmm5 movaps -14 * SIZE(X), %xmm6 @@ -433,9 +433,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $2, N - jle .L27 + jle L(27) movaps -16 * SIZE(X), %xmm5 @@ -448,21 +448,21 @@ addq $2 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $1, N - jle .L999 + jle L(999) SHUFPD_1 %xmm4, %xmm4 mulsd -16 * SIZE(Y), %xmm4 addsd %xmm4, %xmm0 - jmp .L999 + jmp L(999) ALIGN_3 #else movq N, %rax sarq $4, %rax - jle .L24 + jle L(24) movlps -16 * SIZE(X), %xmm4 movhps -15 * SIZE(X), %xmm4 @@ -483,11 +483,11 @@ movhps -1 * SIZE(X), %xmm11 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -548,10 +548,10 @@ subq $-16 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): mulpd -16 * SIZE(Y), %xmm4 addpd %xmm4, %xmm0 mulpd -14 * SIZE(Y), %xmm5 @@ -574,12 +574,12 @@ subq $-16 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $15, N - jle .L999 + jle L(999) testq $8, N - jle .L25 + jle L(25) movlps -16 * SIZE(X), %xmm4 movhps -15 * SIZE(X), %xmm4 @@ -603,9 +603,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $4, N - jle .L26 + jle L(26) movlps -16 * SIZE(X), %xmm4 movhps -15 * SIZE(X), %xmm4 @@ -621,9 +621,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $2, N - jle .L27 + jle L(27) movlps -16 * SIZE(X), %xmm4 movhps -15 * SIZE(X), %xmm4 @@ -635,24 +635,24 @@ addq $2 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $1, N - jle .L999 + jle L(999) movsd -16 * SIZE(X), %xmm4 mulsd -16 * SIZE(Y), %xmm4 addsd %xmm4, %xmm0 - jmp .L999 + jmp L(999) ALIGN_3 #endif -.L50: +L(50): movq N, %rax sarq $2, %rax - jle .L55 + jle L(55) ALIGN_3 -.L53: +L(53): movsd 0 * SIZE(X), %xmm4 addq INCX, X mulsd 0 * SIZE(Y), %xmm4 @@ -676,26 +676,26 @@ addsd %xmm7, %xmm3 decq %rax - jg .L53 + jg L(53) ALIGN_3 -.L55: +L(55): movq N, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_3 -.L56: +L(56): movsd 0 * SIZE(X), %xmm4 addq INCX, X mulsd 0 * SIZE(Y), %xmm4 addq INCY, Y addsd %xmm4, %xmm0 decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L999: +L(999): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 diff --git a/kernel/x86_64/dtrsm_kernel_LT_8x2_bulldozer.S b/kernel/x86_64/dtrsm_kernel_LT_8x2_bulldozer.S index bccf1c9080..14a6fc75ab 100644 --- a/kernel/x86_64/dtrsm_kernel_LT_8x2_bulldozer.S +++ b/kernel/x86_64/dtrsm_kernel_LT_8x2_bulldozer.S @@ -667,10 +667,10 @@ movq N, J sarq $1, J # j = (n >> 1) - jle .L80 + jle L(80) ALIGN_4 -.L01: +L(01): movq A, AO @@ -683,10 +683,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L50_A + jle L(50_A) ALIGN_4 /*********************************************************************************/ -.L51: +L(51): movq B, BO @@ -707,10 +707,10 @@ leaq (BO, %rax, 2), BO negq %rax - je .L56 + je L(56) ALIGN_4 -.L52: +L(52): prefetcht0 A_PR1(AO,%rax,8) prefetcht0 B_PR1(BO,%rax,2) KERNEL8x2_SUB @@ -721,13 +721,13 @@ prefetcht0 A_PR1(AO,%rax,8) KERNEL8x2_SUB - jl .L52 + jl L(52) ALIGN_4 -.L56: +L(56): movq KK, %rax andq $3, %rax # if (k & 1) - je .L59 + je L(59) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 8), AO @@ -735,13 +735,13 @@ negq %rax ALIGN_4 -.L57: +L(57): KERNEL8x2_SUB - jl .L57 + jl L(57) ALIGN_4 -.L59: +L(59): SOLVE_8x2 @@ -757,16 +757,16 @@ addq $8, KK decq I # i -- - jg .L51 + jg L(51) ALIGN_4 /*********************************************************************************/ -.L50_A: +L(50_A): testq $4, M - je .L60 + je L(60) -.L51_A: +L(51_A): movq B, BO @@ -782,23 +782,23 @@ leaq (BO, %rax, 2), BO negq %rax - je .L56_A + je L(56_A) ALIGN_4 -.L52_A: +L(52_A): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB - jl .L52_A + jl L(52_A) ALIGN_4 -.L56_A: +L(56_A): movq KK, %rax andq $3, %rax # if (k & 1) - je .L59_A + je L(59_A) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -806,14 +806,14 @@ negq %rax ALIGN_4 -.L57_A: +L(57_A): KERNEL4x2_SUB - jl .L57_A + jl L(57_A) ALIGN_4 -.L59_A: +L(59_A): SOLVE_4x2 @@ -833,11 +833,11 @@ /*********************************************************************************/ -.L60: +L(60): testq $2, M - je .L70 + je L(70) -.L61: +L(61): movq B, BO vxorpd %xmm8, %xmm8 , %xmm8 @@ -850,23 +850,23 @@ leaq (BO, %rax, 2), BO negq %rax - je .L66 + je L(66) ALIGN_4 -.L62: +L(62): KERNEL2x2_SUB KERNEL2x2_SUB KERNEL2x2_SUB KERNEL2x2_SUB - jl .L62 + jl L(62) ALIGN_4 -.L66: +L(66): movq KK, %rax andq $3, %rax # if (k & 1) - je .L69 + je L(69) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -874,14 +874,14 @@ negq %rax ALIGN_4 -.L67: +L(67): KERNEL2x2_SUB - jl .L67 + jl L(67) ALIGN_4 -.L69: +L(69): SOLVE_2x2 @@ -898,12 +898,12 @@ ALIGN_4 /********************************************************************************/ -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): movq B, BO vxorpd %xmm8, %xmm8 , %xmm8 @@ -915,23 +915,23 @@ leaq (BO, %rax, 2), BO negq %rax - je .L76 + je L(76) ALIGN_4 -.L72: +L(72): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB - jl .L72 + jl L(72) ALIGN_4 -.L76: +L(76): movq KK, %rax andq $3, %rax # if (k & 1) - je .L78 + je L(78) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -939,14 +939,14 @@ negq %rax ALIGN_4 -.L77: +L(77): KERNEL1x2_SUB - jl .L77 + jl L(77) ALIGN_4 -.L78: +L(78): SOLVE_1x2 @@ -963,17 +963,17 @@ ALIGN_4 -.L79: +L(79): movq BO, B decq J # j -- - jg .L01 + jg L(01) ALIGN_4 /***************************************************************************************/ -.L80: +L(80): testq $1, N - je .L999 + je L(999) movq A, AO movq C, CO1 # coffset1 = c @@ -983,10 +983,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L90_A + jle L(90_A) ALIGN_4 /*************************************************************************************/ -.L91: +L(91): movq B, BO @@ -1003,22 +1003,22 @@ leaq (BO, %rax, 1), BO negq %rax - je .L96 + je L(96) ALIGN_4 -.L92: +L(92): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB - jl .L92 + jl L(92) ALIGN_4 -.L96: +L(96): movq KK, %rax andq $3, %rax # if (k & 1) - je .L99 + je L(99) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 8), AO @@ -1026,12 +1026,12 @@ negq %rax ALIGN_4 -.L97: +L(97): KERNEL8x1_SUB - jl .L97 + jl L(97) ALIGN_4 -.L99: +L(99): SOLVE_8x1 @@ -1047,15 +1047,15 @@ decq I # i -- - jg .L91 + jg L(91) ALIGN_4 /*****************************************************************************/ -.L90_A: +L(90_A): testq $4, M - je .L100 + je L(100) -.L91_A: +L(91_A): movq B, BO vxorpd %xmm8, %xmm8 , %xmm8 @@ -1068,23 +1068,23 @@ leaq (BO, %rax, 1), BO negq %rax - je .L96_A + je L(96_A) ALIGN_4 -.L92_A: +L(92_A): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB - jl .L92_A + jl L(92_A) ALIGN_4 -.L96_A: +L(96_A): movq KK, %rax andq $3, %rax # if (k & 1) - je .L99_A + je L(99_A) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -1092,13 +1092,13 @@ negq %rax ALIGN_4 -.L97_A: +L(97_A): KERNEL4x1_SUB - jl .L97_A + jl L(97_A) ALIGN_4 -.L99_A: +L(99_A): SOLVE_4x1 @@ -1116,9 +1116,9 @@ ALIGN_4 /*************************************************************************************/ -.L100: +L(100): testq $2, M - je .L110 + je L(110) @@ -1133,23 +1133,23 @@ leaq (BO, %rax, 1), BO negq %rax - je .L106 + je L(106) ALIGN_4 -.L102: +L(102): KERNEL2x1_SUB KERNEL2x1_SUB KERNEL2x1_SUB KERNEL2x1_SUB - jl .L102 + jl L(102) ALIGN_4 -.L106: +L(106): movq KK, %rax andq $3, %rax # if (k & 1) - je .L109 + je L(109) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -1157,14 +1157,14 @@ negq %rax ALIGN_4 -.L107: +L(107): KERNEL2x1_SUB - jl .L107 + jl L(107) ALIGN_4 -.L109: +L(109): SOLVE_2x1 @@ -1180,12 +1180,12 @@ ALIGN_4 -.L110: +L(110): testq $1, M - je .L119 + je L(119) ALIGN_4 -.L111: +L(111): movq B, BO vxorpd %xmm8, %xmm8 , %xmm8 @@ -1197,23 +1197,23 @@ leaq (BO, %rax, 1), BO negq %rax - je .L116 + je L(116) ALIGN_4 -.L112: +L(112): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB - jl .L112 + jl L(112) ALIGN_4 -.L116: +L(116): movq KK, %rax andq $3, %rax # if (k & 1) - je .L118 + je L(118) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -1221,14 +1221,14 @@ negq %rax ALIGN_4 -.L117: +L(117): KERNEL1x1_SUB - jl .L117 + jl L(117) ALIGN_4 -.L118: +L(118): SOLVE_1x1 @@ -1244,7 +1244,7 @@ ALIGN_4 -.L119: +L(119): movq BO, B @@ -1252,7 +1252,7 @@ ALIGN_4 -.L999: +L(999): movq (%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/dtrsm_kernel_RN_8x2_bulldozer.S b/kernel/x86_64/dtrsm_kernel_RN_8x2_bulldozer.S index 9f693f852e..855bbda292 100644 --- a/kernel/x86_64/dtrsm_kernel_RN_8x2_bulldozer.S +++ b/kernel/x86_64/dtrsm_kernel_RN_8x2_bulldozer.S @@ -480,10 +480,10 @@ movq N, J sarq $1, J # j = (n >> 1) - jle .L80 + jle L(80) ALIGN_4 -.L01: +L(01): movq A, AO @@ -493,10 +493,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L50_A + jle L(50_A) ALIGN_4 /*********************************************************************************/ -.L51: +L(51): movq B, BO @@ -517,10 +517,10 @@ leaq (BO, %rax, 2), BO negq %rax - je .L56 + je L(56) ALIGN_4 -.L52: +L(52): prefetcht0 A_PR1(AO,%rax,8) prefetcht0 B_PR1(BO,%rax,2) KERNEL8x2_SUB @@ -531,13 +531,13 @@ prefetcht0 A_PR1(AO,%rax,8) KERNEL8x2_SUB - jl .L52 + jl L(52) ALIGN_4 -.L56: +L(56): movq KK, %rax andq $3, %rax # if (k & 1) - je .L59 + je L(59) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 8), AO @@ -545,13 +545,13 @@ negq %rax ALIGN_4 -.L57: +L(57): KERNEL8x2_SUB - jl .L57 + jl L(57) ALIGN_4 -.L59: +L(59): SOLVE_8x2 @@ -565,16 +565,16 @@ leaq (BO, %rax, 2), BO decq I # i -- - jg .L51 + jg L(51) ALIGN_4 /*********************************************************************************/ -.L50_A: +L(50_A): testq $4, M - je .L60 + je L(60) -.L51_A: +L(51_A): movq B, BO @@ -590,23 +590,23 @@ leaq (BO, %rax, 2), BO negq %rax - je .L56_A + je L(56_A) ALIGN_4 -.L52_A: +L(52_A): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB - jl .L52_A + jl L(52_A) ALIGN_4 -.L56_A: +L(56_A): movq KK, %rax andq $3, %rax # if (k & 1) - je .L59_A + je L(59_A) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -614,14 +614,14 @@ negq %rax ALIGN_4 -.L57_A: +L(57_A): KERNEL4x2_SUB - jl .L57_A + jl L(57_A) ALIGN_4 -.L59_A: +L(59_A): SOLVE_4x2 @@ -639,11 +639,11 @@ /*********************************************************************************/ -.L60: +L(60): testq $2, M - je .L70 + je L(70) -.L61: +L(61): movq B, BO vxorpd %xmm8, %xmm8 , %xmm8 @@ -656,23 +656,23 @@ leaq (BO, %rax, 2), BO negq %rax - je .L66 + je L(66) ALIGN_4 -.L62: +L(62): KERNEL2x2_SUB KERNEL2x2_SUB KERNEL2x2_SUB KERNEL2x2_SUB - jl .L62 + jl L(62) ALIGN_4 -.L66: +L(66): movq KK, %rax andq $3, %rax # if (k & 1) - je .L69 + je L(69) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -680,14 +680,14 @@ negq %rax ALIGN_4 -.L67: +L(67): KERNEL2x2_SUB - jl .L67 + jl L(67) ALIGN_4 -.L69: +L(69): SOLVE_2x2 @@ -702,12 +702,12 @@ ALIGN_4 /********************************************************************************/ -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): movq B, BO vxorpd %xmm8, %xmm8 , %xmm8 @@ -719,23 +719,23 @@ leaq (BO, %rax, 2), BO negq %rax - je .L76 + je L(76) ALIGN_4 -.L72: +L(72): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB - jl .L72 + jl L(72) ALIGN_4 -.L76: +L(76): movq KK, %rax andq $3, %rax # if (k & 1) - je .L78 + je L(78) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -743,14 +743,14 @@ negq %rax ALIGN_4 -.L77: +L(77): KERNEL1x2_SUB - jl .L77 + jl L(77) ALIGN_4 -.L78: +L(78): SOLVE_1x2 @@ -765,29 +765,29 @@ ALIGN_4 -.L79: +L(79): addq $2, KK // number of values in B # only for RN Kernel movq BO, B decq J # j -- - jg .L01 + jg L(01) ALIGN_4 /***************************************************************************************/ -.L80: +L(80): testq $1, N - je .L999 + je L(999) movq A, AO movq C, CO1 # coffset1 = c movq M, I sarq $3, I # i = (m >> 3) - jle .L90_A + jle L(90_A) ALIGN_4 /*************************************************************************************/ -.L91: +L(91): movq B, BO @@ -804,22 +804,22 @@ leaq (BO, %rax, 1), BO negq %rax - je .L96 + je L(96) ALIGN_4 -.L92: +L(92): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB - jl .L92 + jl L(92) ALIGN_4 -.L96: +L(96): movq KK, %rax andq $3, %rax # if (k & 1) - je .L99 + je L(99) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 8), AO @@ -827,12 +827,12 @@ negq %rax ALIGN_4 -.L97: +L(97): KERNEL8x1_SUB - jl .L97 + jl L(97) ALIGN_4 -.L99: +L(99): SOLVE_8x1 @@ -846,15 +846,15 @@ decq I # i -- - jg .L91 + jg L(91) ALIGN_4 /*****************************************************************************/ -.L90_A: +L(90_A): testq $4, M - je .L100 + je L(100) -.L91_A: +L(91_A): movq B, BO vxorpd %xmm8, %xmm8 , %xmm8 @@ -867,23 +867,23 @@ leaq (BO, %rax, 1), BO negq %rax - je .L96_A + je L(96_A) ALIGN_4 -.L92_A: +L(92_A): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB - jl .L92_A + jl L(92_A) ALIGN_4 -.L96_A: +L(96_A): movq KK, %rax andq $3, %rax # if (k & 1) - je .L99_A + je L(99_A) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -891,13 +891,13 @@ negq %rax ALIGN_4 -.L97_A: +L(97_A): KERNEL4x1_SUB - jl .L97_A + jl L(97_A) ALIGN_4 -.L99_A: +L(99_A): SOLVE_4x1 @@ -912,9 +912,9 @@ ALIGN_4 /*************************************************************************************/ -.L100: +L(100): testq $2, M - je .L110 + je L(110) @@ -929,23 +929,23 @@ leaq (BO, %rax, 1), BO negq %rax - je .L106 + je L(106) ALIGN_4 -.L102: +L(102): KERNEL2x1_SUB KERNEL2x1_SUB KERNEL2x1_SUB KERNEL2x1_SUB - jl .L102 + jl L(102) ALIGN_4 -.L106: +L(106): movq KK, %rax andq $3, %rax # if (k & 1) - je .L109 + je L(109) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -953,14 +953,14 @@ negq %rax ALIGN_4 -.L107: +L(107): KERNEL2x1_SUB - jl .L107 + jl L(107) ALIGN_4 -.L109: +L(109): SOLVE_2x1 @@ -974,12 +974,12 @@ ALIGN_4 -.L110: +L(110): testq $1, M - je .L119 + je L(119) ALIGN_4 -.L111: +L(111): movq B, BO vxorpd %xmm8, %xmm8 , %xmm8 @@ -991,23 +991,23 @@ leaq (BO, %rax, 1), BO negq %rax - je .L116 + je L(116) ALIGN_4 -.L112: +L(112): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB - jl .L112 + jl L(112) ALIGN_4 -.L116: +L(116): movq KK, %rax andq $3, %rax # if (k & 1) - je .L118 + je L(118) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -1015,14 +1015,14 @@ negq %rax ALIGN_4 -.L117: +L(117): KERNEL1x1_SUB - jl .L117 + jl L(117) ALIGN_4 -.L118: +L(118): SOLVE_1x1 @@ -1036,7 +1036,7 @@ ALIGN_4 -.L119: +L(119): addq $1 , KK // number of values in B # only for RN Kernel @@ -1046,7 +1046,7 @@ ALIGN_4 -.L999: +L(999): movq (%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/gemm_beta.S b/kernel/x86_64/gemm_beta.S index 09df2b79d9..9c48feff04 100644 --- a/kernel/x86_64/gemm_beta.S +++ b/kernel/x86_64/gemm_beta.S @@ -93,28 +93,28 @@ pxor %xmm1, %xmm1 test M, M - jle .L999 + jle L(999) test N, N - jle .L999 + jle L(999) #ifdef DOUBLE ucomisd %xmm1, %xmm0 #else ucomiss %xmm1, %xmm0 #endif - jne .L201 + jne L(201) ALIGN_4 -.L101: +L(101): movq C, C1 leaq (C, LDC, SIZE), C movq M, I sarq $3, I - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): #ifdef OPTERON prefetchw 32 * SIZE(C1) #endif @@ -129,37 +129,37 @@ MOVSD %xmm0, 7 * SIZE(C1) addq $8 * SIZE, C1 decq I - jg .L102 + jg L(102) ALIGN_4 -.L103: +L(103): movq M, I andq $7, I - jle .L105 + jle L(105) ALIGN_4 -.L104: +L(104): MOVSD %xmm0, 0 * SIZE(C1) addq $SIZE, C1 decq I - jg .L104 + jg L(104) ALIGN_4 -.L105: +L(105): decq N - jg .L101 - jmp .L999 + jg L(101) + jmp L(999) ALIGN_3 -.L201: +L(201): movq C, C1 # c_offset = c leaq (C, LDC, SIZE), C # c += ldc movq M, I sarq $3, I - jle .L203 + jle L(203) ALIGN_4 -.L202: +L(202): #ifdef OPTERON prefetchw 32 * SIZE(C1) #endif @@ -193,30 +193,30 @@ addq $8 * SIZE, C1 decq I - jg .L202 + jg L(202) ALIGN_4 -.L203: +L(203): movq M, I andq $7, I - jle .L205 + jle L(205) ALIGN_4 -.L204: +L(204): MOVSD 0 * SIZE(C1), %xmm8 MULSD %xmm0, %xmm8 MOVSD %xmm8, 0 * SIZE(C1) addq $SIZE, C1 decq I - jg .L204 + jg L(204) ALIGN_4 -.L205: +L(205): decq N - jg .L201 + jg L(201) ALIGN_3 -.L999: +L(999): xorq %rax, %rax #ifdef WINDOWS_ABI diff --git a/kernel/x86_64/gemm_kernel_2x8_nehalem.S b/kernel/x86_64/gemm_kernel_2x8_nehalem.S index 7e4b0d863b..52caf7d2d1 100644 --- a/kernel/x86_64/gemm_kernel_2x8_nehalem.S +++ b/kernel/x86_64/gemm_kernel_2x8_nehalem.S @@ -163,10 +163,10 @@ movq N, J sarq $3, J NOBRANCH - jle .L30 + jle L(30) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -183,10 +183,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): prefetcht2 -16 * SIZE(BB) subq $-8 * SIZE, BB @@ -253,10 +253,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm12 @@ -382,10 +382,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -395,10 +395,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addpd %xmm1, %xmm12 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm13 @@ -434,10 +434,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): addpd %xmm1, %xmm12 movaps %xmm8, %xmm0 shufpd $2, %xmm9, %xmm8 @@ -470,7 +470,7 @@ orq LDC, %rax testq $15, %rax NOBRANCH - jne .L18x + jne L(18x) leaq (LDC, LDC, 2), %rax @@ -522,11 +522,11 @@ addq $2 * SIZE, CO2 decq I BRANCH - jg .L11 - jmp .L20 + jg L(11) + jmp L(20) ALIGN_4 -.L18x: +L(18x): leaq (LDC, LDC, 2), %rax #ifndef TRMMKERNEL @@ -576,13 +576,13 @@ addq $2 * SIZE, CO2 decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M BRANCH - jle .L29 + jle L(29) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -623,10 +623,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -688,10 +688,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -701,10 +701,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -724,10 +724,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): mulpd %xmm7, %xmm8 mulpd %xmm7, %xmm9 mulpd %xmm7, %xmm10 @@ -775,7 +775,7 @@ #endif ALIGN_4 -.L29: +L(29): #if defined(TRMMKERNEL) && !defined(LEFT) addq $8, KK #endif @@ -786,12 +786,12 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L30: +L(30): testq $4, N - jle .L50 + jle L(50) ALIGN_4 #if defined(TRMMKERNEL) && defined(LEFT) @@ -806,10 +806,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L31: +L(31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -856,10 +856,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -931,10 +931,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -944,10 +944,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): addpd %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm9 @@ -969,10 +969,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): addpd %xmm1, %xmm8 addpd %xmm2, %xmm9 addpd %xmm3, %xmm10 @@ -1036,13 +1036,13 @@ addq $2 * SIZE, CO2 decq I BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L40: +L(40): testq $1, M BRANCH - jle .L49 + jle L(49) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1083,10 +1083,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1124,10 +1124,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1137,10 +1137,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1154,10 +1154,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): addpd %xmm10, %xmm8 mulpd %xmm7, %xmm8 addpd %xmm11, %xmm9 @@ -1192,7 +1192,7 @@ #endif ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif @@ -1202,9 +1202,9 @@ leaq (C, LDC, 4), C ALIGN_4 -.L50: +L(50): testq $2, N - jle .L70 + jle L(70) ALIGN_4 #if defined(TRMMKERNEL) && defined(LEFT) @@ -1219,10 +1219,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1265,10 +1265,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -1308,13 +1308,13 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 ALIGN_3 -.L55: +L(55): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1324,10 +1324,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addpd %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm9 @@ -1341,10 +1341,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): addpd %xmm1, %xmm8 addpd %xmm2, %xmm9 @@ -1386,13 +1386,13 @@ addq $2 * SIZE, CO2 decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L69 + jle L(69) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1431,10 +1431,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): mulpd %xmm0, %xmm1 movddup -15 * SIZE(AO), %xmm0 addpd %xmm1, %xmm8 @@ -1460,10 +1460,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1473,10 +1473,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): mulpd %xmm0, %xmm1 movddup -15 * SIZE(AO), %xmm0 addpd %xmm1, %xmm8 @@ -1487,10 +1487,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addpd %xmm9, %xmm8 mulpd %xmm7, %xmm8 @@ -1518,7 +1518,7 @@ #endif ALIGN_4 -.L69: +L(69): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -1528,9 +1528,9 @@ leaq (C, LDC, 2), C ALIGN_4 -.L70: +L(70): testq $1, N - jle .L999 + jle L(999) ALIGN_4 #if defined(TRMMKERNEL) && defined(LEFT) @@ -1544,10 +1544,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L80 + jle L(80) ALIGN_4 -.L71: +L(71): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1589,10 +1589,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -1620,12 +1620,12 @@ subq $1, %rax BRANCH - jg .L72 + jg L(72) addpd %xmm9, %xmm8 ALIGN_3 -.L75: +L(75): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1635,10 +1635,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addpd %xmm1, %xmm8 movddup -16 * SIZE(BO), %xmm1 mulpd %xmm0, %xmm1 @@ -1649,10 +1649,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addpd %xmm1, %xmm8 mulpd %xmm7, %xmm8 @@ -1682,13 +1682,13 @@ addq $2 * SIZE, CO1 decq I BRANCH - jg .L71 + jg L(71) ALIGN_4 -.L80: +L(80): testq $1, M BRANCH - jle .L999 + jle L(999) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1736,10 +1736,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): mulpd %xmm0, %xmm1 #ifndef TRMMKERNEL movaps -14 * SIZE(AO), %xmm0 @@ -1775,12 +1775,12 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) addpd %xmm9, %xmm8 ALIGN_3 -.L85: +L(85): movsd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1790,10 +1790,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): mulsd %xmm0, %xmm1 movsd -15 * SIZE(AO), %xmm0 addsd %xmm1, %xmm8 @@ -1804,10 +1804,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): haddpd %xmm8, %xmm8 mulsd %xmm7, %xmm8 @@ -1820,7 +1820,7 @@ movsd %xmm8, (CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/gemm_kernel_4x2_atom.S b/kernel/x86_64/gemm_kernel_4x2_atom.S index e5f2e9105b..56ef8f20c5 100644 --- a/kernel/x86_64/gemm_kernel_4x2_atom.S +++ b/kernel/x86_64/gemm_kernel_4x2_atom.S @@ -147,10 +147,10 @@ movq N, J sarq $1, J - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -168,10 +168,10 @@ movq M, I sarq $2, I - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -225,10 +225,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addsd %xmm2, %xmm13 movaps %xmm0, %xmm2 @@ -370,10 +370,10 @@ mulsd %xmm3, %xmm6 movsd 1 * SIZE(BO), %xmm3 - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else @@ -382,10 +382,10 @@ andq $3, %rax BRANCH BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): addsd %xmm2, %xmm13 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -424,10 +424,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): movsd ALPHA, %xmm5 addsd %xmm2, %xmm13 @@ -482,12 +482,12 @@ addq $4 * SIZE, CO2 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M - jle .L30 + jle L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -533,10 +533,10 @@ #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addsd %xmm2, %xmm9 movaps %xmm0, %xmm2 @@ -614,10 +614,10 @@ addq $8 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -628,10 +628,10 @@ andq $3, %rax BRANCH BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): addsd %xmm2, %xmm9 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -654,10 +654,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): addsd %xmm2, %xmm9 mulsd %xmm7, %xmm8 addsd %xmm6, %xmm11 @@ -696,9 +696,9 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -740,10 +740,10 @@ #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): addsd %xmm5, %xmm8 movsd 2 * SIZE(BO), %xmm5 mulsd %xmm0, %xmm1 @@ -785,10 +785,10 @@ addq $8 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -801,10 +801,10 @@ andq $3, %rax BRANCH BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulsd %xmm0, %xmm1 addq $2 * SIZE, BO mulsd %xmm0, %xmm3 @@ -818,10 +818,10 @@ addq $1 * SIZE, AO decq %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): mulsd %xmm7, %xmm8 mulsd %xmm7, %xmm9 @@ -847,19 +847,19 @@ #endif ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif movq BO, B decq J # j -- - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $1, N - je .L999 + je L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -873,10 +873,10 @@ movq M, I sarq $2, I - jle .L50 + jle L(50) ALIGN_4 -.L41: +L(41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -923,10 +923,10 @@ #endif sarq $2, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): addsd %xmm9, %xmm8 movsd 4 * SIZE(AO), %xmm9 mulsd %xmm4, %xmm0 @@ -1000,10 +1000,10 @@ mulsd %xmm5, %xmm15 movsd 1 * SIZE(BO), %xmm5 - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1020,10 +1020,10 @@ andq $3, %rax BRANCH BRANCH - je .L49 + je L(49) ALIGN_4 -.L46: +L(46): mulsd %xmm4, %xmm0 mulsd %xmm4, %xmm1 mulsd %xmm4, %xmm2 @@ -1043,10 +1043,10 @@ addq $1 * SIZE, BO decq %rax BRANCH - jg .L46 + jg L(46) ALIGN_4 -.L49: +L(49): mulsd %xmm7, %xmm8 mulsd %xmm7, %xmm10 mulsd %xmm7, %xmm12 @@ -1080,12 +1080,12 @@ addq $4 * SIZE, CO1 decq I # i -- - jg .L41 + jg L(41) ALIGN_4 -.L50: +L(50): testq $2, M - jle .L60 + jle L(60) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1126,10 +1126,10 @@ #endif sarq $2, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): addsd %xmm2, %xmm8 movsd 2 * SIZE(AO), %xmm2 mulsd %xmm4, %xmm0 @@ -1170,10 +1170,10 @@ mulsd %xmm5, %xmm3 movsd 1 * SIZE(BO), %xmm5 - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1187,10 +1187,10 @@ andq $3, %rax BRANCH BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): mulsd %xmm4, %xmm0 mulsd %xmm4, %xmm1 movsd 1 * SIZE(BO), %xmm4 @@ -1204,10 +1204,10 @@ addq $1 * SIZE, BO decq %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): mulsd %xmm7, %xmm8 mulsd %xmm7, %xmm10 @@ -1235,9 +1235,9 @@ addq $2 * SIZE, CO1 ALIGN_4 -.L60: +L(60): testq $1, M - je .L999 + je L(999) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1281,10 +1281,10 @@ #endif sarq $2, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): addsd %xmm5, %xmm8 movsd 2 * SIZE(BO), %xmm5 mulsd %xmm0, %xmm1 @@ -1309,13 +1309,13 @@ addq $4 * SIZE, BO decq %rax - jne .L62 + jne L(62) addsd %xmm5, %xmm8 addsd %xmm7, %xmm9 ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1326,10 +1326,10 @@ andq $3, %rax BRANCH BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): movsd 0 * SIZE(AO), %xmm0 movsd 0 * SIZE(BO), %xmm1 @@ -1341,10 +1341,10 @@ decq %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addsd %xmm9, %xmm8 mulsd %xmm7, %xmm8 @@ -1356,7 +1356,7 @@ movsd %xmm8, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/gemm_kernel_4x4_barcelona.S b/kernel/x86_64/gemm_kernel_4x4_barcelona.S index 9a29a800d8..4f634ddba9 100644 --- a/kernel/x86_64/gemm_kernel_4x4_barcelona.S +++ b/kernel/x86_64/gemm_kernel_4x4_barcelona.S @@ -462,10 +462,10 @@ #endif movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): movq C, CO1 # coffset1 = c leaq (C, LDC, 2), CO2 # coffset2 = c + ldc @@ -484,10 +484,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -543,10 +543,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -556,7 +556,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -566,7 +566,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -576,7 +576,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -586,7 +586,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -596,7 +596,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -606,7 +606,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -616,7 +616,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -625,10 +625,10 @@ KERNEL6(16 * 0) KERNEL7(16 * 0) KERNEL8(16 * 0) - jl .L12 + jl L(12) ALIGN_4 -.L15: +L(15): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -637,7 +637,7 @@ movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) ALIGN_4 KERNEL_SUB1(16 * 0) @@ -649,14 +649,14 @@ subq $-16 * SIZE, AO ALIGN_4 -.L16: +L(16): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L19 + je L(19) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -664,7 +664,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -691,10 +691,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L17 + jl L(17) ALIGN_4 -.L19: +L(19): prefetch -8 * SIZE(BB) subq $-16 * SIZE, BB @@ -783,18 +783,18 @@ addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $3, M - je .L39 + je L(39) testq $2, M - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -838,10 +838,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L26 + je L(26) ALIGN_4 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -897,10 +897,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L22 + jl L(22) ALIGN_4 -.L26: +L(26): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -909,7 +909,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L29 + je L(29) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -917,7 +917,7 @@ negq %rax ALIGN_4 -.L27: +L(27): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -933,10 +933,10 @@ movddup -11 * SIZE(BO, %rax, 4), %xmm5 addq $SIZE, %rax - jl .L27 + jl L(27) ALIGN_4 -.L29: +L(29): #ifndef TRMMKERNEL movupd (CO1), %xmm0 movupd (CO1, LDC), %xmm2 @@ -983,9 +983,9 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1030,10 +1030,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L36 + je L(36) ALIGN_4 -.L32: +L(32): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO, %rax, 4), %xmm0 addpd %xmm1, %xmm8 @@ -1061,10 +1061,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L32 + jl L(32) ALIGN_4 -.L36: +L(36): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1073,7 +1073,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L38 + je L(38) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -1081,7 +1081,7 @@ negq %rax ALIGN_4 -.L37: +L(37): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO, %rax, 4), %xmm0 addpd %xmm1, %xmm8 @@ -1090,10 +1090,10 @@ movddup -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L37 + jl L(37) ALIGN_4 -.L38: +L(38): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1131,7 +1131,7 @@ #endif ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif @@ -1139,18 +1139,18 @@ movq BO, B decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $3, N - je .L999 + je L(999) testq $2, N - je .L80 + je L(80) ALIGN_4 -.L41: +L(41): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1166,10 +1166,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1219,10 +1219,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L56 + je L(56) ALIGN_4 -.L52: +L(52): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -1274,10 +1274,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L52 + jl L(52) ALIGN_4 -.L56: +L(56): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1286,7 +1286,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L59 + je L(59) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -1294,7 +1294,7 @@ negq %rax ALIGN_4 -.L57: +L(57): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -1309,10 +1309,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L57 + jl L(57) ALIGN_4 -.L59: +L(59): #ifndef TRMMKERNEL movupd (CO1), %xmm0 movupd 2 * SIZE(CO1), %xmm1 @@ -1358,15 +1358,15 @@ addq $4 * SIZE, CO1 # coffset += 4 addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1409,10 +1409,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L66 + je L(66) ALIGN_4 -.L62: +L(62): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -1444,10 +1444,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L62 + jl L(62) ALIGN_4 -.L66: +L(66): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1456,7 +1456,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L69 + je L(69) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -1464,7 +1464,7 @@ negq %rax ALIGN_4 -.L67: +L(67): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -1474,10 +1474,10 @@ movddup -13 * SIZE(BO, %rax, 2), %xmm3 addq $SIZE, %rax - jl .L67 + jl L(67) ALIGN_4 -.L69: +L(69): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1516,12 +1516,12 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1564,10 +1564,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L76 + je L(76) ALIGN_4 -.L72: +L(72): mulpd -16 * SIZE(BO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -12 * SIZE(AO, %rax, 1), %xmm0 @@ -1586,10 +1586,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L72 + jl L(72) ALIGN_4 -.L76: +L(76): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1598,7 +1598,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L78 + je L(78) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -1606,16 +1606,16 @@ negq %rax ALIGN_4 -.L77: +L(77): mulpd -16 * SIZE(BO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L77 + jl L(77) ALIGN_4 -.L78: +L(78): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -1648,7 +1648,7 @@ #endif ALIGN_4 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -1658,12 +1658,12 @@ leaq (C, LDC, 2), C ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L81: +L(81): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1674,10 +1674,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1723,10 +1723,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L96 + je L(96) ALIGN_4 -.L92: +L(92): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -1754,10 +1754,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L92 + jl L(92) ALIGN_4 -.L96: +L(96): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1766,7 +1766,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L99 + je L(99) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -1774,7 +1774,7 @@ negq %rax ALIGN_4 -.L97: +L(97): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -1783,10 +1783,10 @@ movddup -15 * SIZE(BO, %rax, 1), %xmm1 addq $SIZE, %rax - jl .L97 + jl L(97) ALIGN_4 -.L99: +L(99): addpd %xmm9, %xmm8 addpd %xmm13, %xmm12 @@ -1823,15 +1823,15 @@ addq $4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1874,10 +1874,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L106 + je L(106) ALIGN_4 -.L102: +L(102): mulpd -16 * SIZE(AO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -12 * SIZE(BO, %rax, 1), %xmm0 @@ -1896,10 +1896,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L102 + jl L(102) ALIGN_4 -.L106: +L(106): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1908,7 +1908,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L109 + je L(109) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -1916,16 +1916,16 @@ negq %rax ALIGN_4 -.L107: +L(107): movddup -16 * SIZE(BO, %rax, 1), %xmm0 mulpd -16 * SIZE(AO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 addq $SIZE, %rax - jl .L107 + jl L(107) ALIGN_4 -.L109: +L(109): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -1960,12 +1960,12 @@ ALIGN_4 -.L110: +L(110): testq $1, M - je .L999 + je L(999) ALIGN_4 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2004,10 +2004,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L116 + je L(116) ALIGN_4 -.L112: +L(112): mulpd -16 * SIZE(BO, %rax, 1), %xmm0 addpd %xmm0, %xmm8 movapd -12 * SIZE(AO, %rax, 1), %xmm0 @@ -2018,10 +2018,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L112 + jl L(112) ALIGN_4 -.L116: +L(116): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2030,7 +2030,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L118 + je L(118) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -2038,16 +2038,16 @@ negq %rax ALIGN_4 -.L117: +L(117): mulsd -16 * SIZE(BO, %rax, 1), %xmm0 addsd %xmm0, %xmm8 movsd -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L117 + jl L(117) ALIGN_4 -.L118: +L(118): addpd %xmm9, %xmm8 haddpd %xmm8, %xmm8 @@ -2064,7 +2064,7 @@ movsd %xmm8, (CO1) ALIGN_4 -.L999: +L(999): movq (%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/gemm_kernel_4x4_core2.S b/kernel/x86_64/gemm_kernel_4x4_core2.S index 2f2ddc875b..2dd1092fe0 100644 --- a/kernel/x86_64/gemm_kernel_4x4_core2.S +++ b/kernel/x86_64/gemm_kernel_4x4_core2.S @@ -162,10 +162,10 @@ movq N, J sarq $2, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): /* Copying to Sub Buffer */ leaq 16 * SIZE + BUFFER, BO @@ -180,10 +180,10 @@ movq K, %rax sarq $2, %rax NOBRANCH - jle .L05 + jle L(05) ALIGN_3 -.L02: +L(02): prefetcht0 (PREFETCH_R + 0) * SIZE(B) prefetcht0 (PREFETCH_R + 8) * SIZE(B) @@ -242,18 +242,18 @@ subq $-16 * SIZE, B decq %rax BRANCH - jne .L02 + jne L(02) ALIGN_3 -.L05: +L(05): movq K, %rax andq $3, %rax BRANCH BRANCH - jle .L10 + jle L(10) ALIGN_3 -.L06: +L(06): movapd -14 * SIZE(B), %xmm1 movddup %xmm0, %xmm8 @@ -272,10 +272,10 @@ addq $8 * SIZE, BO decq %rax BRANCH - jne .L06 + jne L(06) ALIGN_4 -.L10: +L(10): leaq (PREFETCH_R + 0) * SIZE(B), BB movq C, CO1 # coffset1 = c @@ -285,10 +285,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -346,10 +346,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): PADDING; addpd %xmm2, %xmm10 movaps -15 * SIZE(BO), %xmm2 @@ -479,10 +479,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): prefetcht2 -8 * SIZE(BB) #ifndef TRMMKERNEL @@ -492,10 +492,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addpd %xmm2, %xmm10 movaps -15 * SIZE(BO), %xmm2 addpd %xmm3, %xmm14 @@ -530,10 +530,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): movddup ALPHA, %xmm7 addpd %xmm2, %xmm10 @@ -554,7 +554,7 @@ orq LDC, %rax testq $15, %rax NOBRANCH - jne .L18x + jne L(18x) #ifndef TRMMKERNEL addpd 0 * SIZE(CO1), %xmm8 @@ -595,11 +595,11 @@ addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- BRANCH - jg .L11 - jmp .L20 + jg L(11) + jmp L(20) ALIGN_4 -.L18x: +L(18x): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 @@ -664,13 +664,13 @@ addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -713,10 +713,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_4 -.L21: +L(21): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm2, %xmm8 movapd -16 * SIZE(BO), %xmm2 @@ -778,10 +778,10 @@ subq $-32 * SIZE, BO subq $1, %rax BRANCH - jg .L21 + jg L(21) ALIGN_4 -.L25: +L(25): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -791,10 +791,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): addpd %xmm2, %xmm8 movapd -16 * SIZE(BO), %xmm2 mulpd %xmm0, %xmm2 @@ -813,10 +813,10 @@ addq $8 * SIZE, BO subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): addpd %xmm2, %xmm8 addpd %xmm3, %xmm9 addpd %xmm4, %xmm10 @@ -873,10 +873,10 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $1, M BRANCH - jle .L39 + jle L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -920,10 +920,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_4 -.L31: +L(31): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addsd %xmm2, %xmm8 movsd -16 * SIZE(BO), %xmm2 @@ -985,10 +985,10 @@ subq $-32 * SIZE, BO subq $1, %rax BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L35: +L(35): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -998,10 +998,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): addsd %xmm2, %xmm8 movsd -16 * SIZE(BO), %xmm2 mulsd %xmm0, %xmm2 @@ -1020,10 +1020,10 @@ addq $8 * SIZE, BO subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): addsd %xmm2, %xmm8 addsd %xmm3, %xmm9 addsd %xmm4, %xmm10 @@ -1068,7 +1068,7 @@ #endif ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif @@ -1076,16 +1076,16 @@ leaq (C, LDC, 4), C subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $2, N BRANCH - jle .L80 + jle L(80) ALIGN_4 -.L41: +L(41): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -1096,12 +1096,12 @@ movq K, %rax sarq $3, %rax - jle .L43 + jle L(43) addq %rax, %rax ALIGN_4 -.L42: +L(42): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 movddup -14 * SIZE(B), %xmm10 @@ -1124,17 +1124,17 @@ addq $16 * SIZE, BO subq $1, %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): movq K, %rax andq $7, %rax BRANCH - jle .L45 + jle L(45) ALIGN_4 -.L44: +L(44): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 @@ -1144,20 +1144,20 @@ addq $2 * SIZE, B addq $4 * SIZE, BO subq $1, %rax - jne .L44 + jne L(44) ALIGN_4 -.L45: +L(45): movq C, CO1 leaq (C, LDC, 1), CO2 movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1204,10 +1204,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L55 + jle L(55) ALIGN_4 -.L51: +L(51): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm2, %xmm8 @@ -1275,10 +1275,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L51 + jg L(51) ALIGN_4 -.L55: +L(55): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1287,10 +1287,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): addpd %xmm2, %xmm8 movapd -16 * SIZE(BO), %xmm2 addpd %xmm3, %xmm12 @@ -1310,10 +1310,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): addpd %xmm2, %xmm8 addpd %xmm3, %xmm12 addpd %xmm4, %xmm9 @@ -1368,12 +1368,12 @@ addq $4 * SIZE, CO1 addq $4 * SIZE, CO2 subq $1, I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): testq $2, M - jle .L70 + jle L(70) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1417,10 +1417,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L65 + jle L(65) ALIGN_4 -.L61: +L(61): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm2, %xmm8 @@ -1456,10 +1456,10 @@ subq $ -8 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L61 + jg L(61) ALIGN_4 -.L65: +L(65): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1468,10 +1468,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): addpd %xmm2, %xmm8 movapd -16 * SIZE(BO), %xmm2 mulpd %xmm0, %xmm2 @@ -1483,10 +1483,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addpd %xmm2, %xmm8 addpd %xmm3, %xmm9 addpd %xmm4, %xmm10 @@ -1532,9 +1532,9 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L70: +L(70): testq $1, M - jle .L79 + jle L(79) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1577,10 +1577,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L75 + jle L(75) ALIGN_4 -.L71: +L(71): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addsd %xmm2, %xmm8 @@ -1616,10 +1616,10 @@ subq $ -4 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L71 + jg L(71) ALIGN_4 -.L75: +L(75): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1628,10 +1628,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): addsd %xmm2, %xmm8 movsd -16 * SIZE(BO), %xmm2 mulsd %xmm0, %xmm2 @@ -1643,10 +1643,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addsd %xmm2, %xmm8 addsd %xmm3, %xmm9 addsd %xmm4, %xmm10 @@ -1685,7 +1685,7 @@ movsd %xmm9, 0 * SIZE(CO2) ALIGN_4 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -1693,13 +1693,13 @@ leaq (C, LDC, 2), C ALIGN_4 -.L80: +L(80): testq $1, N BRANCH - jle .L999 + jle L(999) ALIGN_4 -.L81: +L(81): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -1710,12 +1710,12 @@ movq K, %rax sarq $4, %rax - jle .L83 + jle L(83) addq %rax, %rax ALIGN_4 -.L82: +L(82): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 movddup -14 * SIZE(B), %xmm10 @@ -1737,17 +1737,17 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO subq $1, %rax - jne .L82 + jne L(82) ALIGN_4 -.L83: +L(83): movq K, %rax andq $15, %rax BRANCH - jle .L85 + jle L(85) ALIGN_4 -.L84: +L(84): movddup -16 * SIZE(B), %xmm8 movapd %xmm8, 0 * SIZE(BO) @@ -1755,19 +1755,19 @@ addq $1 * SIZE, B addq $2 * SIZE, BO subq $1, %rax - jne .L84 + jne L(84) ALIGN_4 -.L85: +L(85): movq C, CO1 movq A, AO movq M, I sarq $2, I - jle .L100 + jle L(100) ALIGN_4 -.L90: +L(90): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1812,10 +1812,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L95 + jle L(95) ALIGN_4 -.L91: +L(91): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm0, %xmm8 @@ -1852,10 +1852,10 @@ subq $-16 * SIZE, AO subq $ -8 * SIZE, BO subq $1, %rax - jg .L91 + jg L(91) ALIGN_4 -.L95: +L(95): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1864,10 +1864,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): addpd %xmm0, %xmm8 movapd -16 * SIZE(AO), %xmm0 mulpd %xmm4, %xmm0 @@ -1879,10 +1879,10 @@ addq $4 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): addpd %xmm0, %xmm8 addpd %xmm1, %xmm12 addpd %xmm2, %xmm9 @@ -1926,12 +1926,12 @@ addq $4 * SIZE, CO1 subq $1, I - jg .L90 + jg L(90) ALIGN_4 -.L100: +L(100): testq $2, M - jle .L110 + jle L(110) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1976,10 +1976,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L105 + jle L(105) ALIGN_4 -.L101: +L(101): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm0, %xmm8 @@ -2002,10 +2002,10 @@ subq $-8 * SIZE, AO subq $-8 * SIZE, BO subq $1, %rax - jg .L101 + jg L(101) ALIGN_4 -.L105: +L(105): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2014,10 +2014,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L108 + je L(108) ALIGN_4 -.L106: +L(106): addpd %xmm0, %xmm8 movapd -16 * SIZE(AO), %xmm0 mulpd %xmm4, %xmm0 @@ -2026,10 +2026,10 @@ addq $2 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L106 + jg L(106) ALIGN_4 -.L108: +L(108): addpd %xmm0, %xmm8 addpd %xmm1, %xmm9 addpd %xmm2, %xmm10 @@ -2069,9 +2069,9 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L110: +L(110): testq $1, M - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2116,10 +2116,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L115 + jle L(115) ALIGN_4 -.L111: +L(111): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm0, %xmm8 @@ -2142,10 +2142,10 @@ subq $-4 * SIZE, AO subq $-8 * SIZE, BO subq $1, %rax - jg .L111 + jg L(111) ALIGN_4 -.L115: +L(115): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2154,10 +2154,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): addsd %xmm0, %xmm8 movsd -16 * SIZE(AO), %xmm0 mulsd %xmm4, %xmm0 @@ -2166,10 +2166,10 @@ addq $1 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addsd %xmm0, %xmm8 addsd %xmm1, %xmm9 addsd %xmm2, %xmm10 @@ -2190,7 +2190,7 @@ movsd %xmm8, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/gemm_kernel_4x4_penryn.S b/kernel/x86_64/gemm_kernel_4x4_penryn.S index 56611e5c58..beb36b5b11 100644 --- a/kernel/x86_64/gemm_kernel_4x4_penryn.S +++ b/kernel/x86_64/gemm_kernel_4x4_penryn.S @@ -187,10 +187,10 @@ movq N, J sarq $2, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -207,10 +207,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -267,10 +267,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm3, %xmm11 movaps -15 * SIZE(BO), %xmm3 @@ -398,10 +398,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): PREFETCHB -8 * SIZE(BB) #ifdef DUNNINGTON PREFETCHB 0 * SIZE(BB) @@ -415,10 +415,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addpd %xmm3, %xmm11 movaps -15 * SIZE(BO), %xmm3 addpd %xmm4, %xmm15 @@ -454,10 +454,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): movddup ALPHA, %xmm1 #ifndef DUNNINGTON @@ -499,7 +499,7 @@ orq LDC, %rax testq $15, %rax NOBRANCH - jne .L18x + jne L(18x) #ifndef TRMMKERNEL addpd 0 * SIZE(CO1), %xmm8 @@ -541,11 +541,11 @@ decq I # i -- BRANCH - jg .L11 - jmp .L20 + jg L(11) + jmp L(20) ALIGN_4 -.L18x: +L(18x): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 @@ -610,13 +610,13 @@ addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -664,10 +664,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_4 -.L22: +L(22): addpd %xmm3, %xmm11 movaps -15 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -733,10 +733,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -744,10 +744,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): addpd %xmm3, %xmm11 movaps -15 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -768,10 +768,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): addpd %xmm3, %xmm11 addpd %xmm5, %xmm10 @@ -833,10 +833,10 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $1, M BRANCH - jle .L39 + jle L(39) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -879,10 +879,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) shufps $0x44, %xmm0, %xmm0 @@ -929,10 +929,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -940,10 +940,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): shufps $0x44, %xmm0, %xmm0 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm3 @@ -959,10 +959,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): movddup ALPHA, %xmm3 addpd %xmm10, %xmm8 @@ -1004,7 +1004,7 @@ #endif ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif @@ -1015,13 +1015,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $2, N BRANCH - jle .L80 + jle L(80) movq C, CO1 leaq (C, LDC, 1), CO2 @@ -1039,10 +1039,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1088,10 +1088,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_4 -.L52: +L(52): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 @@ -1167,10 +1167,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_4 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1178,10 +1178,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -1204,10 +1204,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): movddup ALPHA, %xmm3 movaps %xmm8, %xmm0 @@ -1264,13 +1264,13 @@ addq $4 * SIZE, CO2 decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M BRANCH - jle .L70 + jle L(70) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1311,10 +1311,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x4e, %xmm2, %xmm7 @@ -1357,10 +1357,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1368,10 +1368,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm7 @@ -1386,10 +1386,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1433,10 +1433,10 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L70: +L(70): testq $1, M BRANCH - jle .L79 + jle L(79) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1476,10 +1476,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_4 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) shufps $0x44, %xmm0, %xmm0 @@ -1510,10 +1510,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -1521,10 +1521,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): shufps $0x44, %xmm0, %xmm0 mulpd %xmm0, %xmm2 movsd -15 * SIZE(AO), %xmm0 @@ -1536,10 +1536,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): movddup ALPHA, %xmm3 addpd %xmm9, %xmm8 @@ -1572,7 +1572,7 @@ #endif ALIGN_4 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -1581,10 +1581,10 @@ movq BO, B ALIGN_4 -.L80: +L(80): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1597,10 +1597,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1642,10 +1642,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_4 -.L92: +L(92): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm2, %xmm3 @@ -1702,10 +1702,10 @@ subq $ -4 * SIZE, BO subq $1, %rax BRANCH - jg .L92 + jg L(92) ALIGN_4 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -1713,10 +1713,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): pshufd $0x44, %xmm2, %xmm3 pshufd $0x44, %xmm2, %xmm4 movsd -16 * SIZE(BO), %xmm2 @@ -1734,10 +1734,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): movddup ALPHA, %xmm3 #ifndef TRMMKERNEL @@ -1776,13 +1776,13 @@ addq $4 * SIZE, CO1 decq I BRANCH - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M BRANCH - jle .L110 + jle L(110) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1821,10 +1821,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_4 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm2, %xmm3 @@ -1859,10 +1859,10 @@ subq $-4 * SIZE, BO subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_4 -.L105: +L(105): #ifndef TRMMKERNEL movq K, %rax #else @@ -1870,10 +1870,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L108 + je L(108) ALIGN_4 -.L106: +L(106): pshufd $0x44, %xmm2, %xmm3 movsd -16 * SIZE(BO), %xmm2 @@ -1886,10 +1886,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_4 -.L108: +L(108): addpd %xmm9, %xmm8 movddup ALPHA, %xmm3 @@ -1925,10 +1925,10 @@ addq $2 * SIZE, CO1 ALIGN_4 -.L110: +L(110): testq $1, M BRANCH - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1967,10 +1967,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_4 -.L112: +L(112): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulsd %xmm0, %xmm2 @@ -1997,10 +1997,10 @@ subq $-4 * SIZE, BO subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_4 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2008,10 +2008,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd %xmm0, %xmm2 addsd %xmm2, %xmm8 movsd -15 * SIZE(AO), %xmm0 @@ -2022,10 +2022,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): movddup ALPHA, %xmm3 addpd %xmm9, %xmm8 @@ -2043,7 +2043,7 @@ movlpd %xmm8, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/gemm_kernel_4x4_sse2.S b/kernel/x86_64/gemm_kernel_4x4_sse2.S index bc317da8e4..1d0af3eecb 100644 --- a/kernel/x86_64/gemm_kernel_4x4_sse2.S +++ b/kernel/x86_64/gemm_kernel_4x4_sse2.S @@ -402,10 +402,10 @@ #endif movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_3 -.L01: +L(01): /* Copying to Sub Buffer */ leaq 16 * SIZE + BUFFER, BO movq C, CO1 # coffset1 = c @@ -418,14 +418,14 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_3 #define RPREFETCHSIZE (8 * 7 + 4) #define WPREFETCHSIZE (8 * 8 + 4) -.L02: +L(02): PREFETCH (RPREFETCHSIZE + 0) * SIZE(B) movq 0 * SIZE(B), %mm0 @@ -498,17 +498,17 @@ subq $-16 * SIZE, B subq $1, %rax - jne .L02 + jne L(02) ALIGN_3 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_3 -.L04: +L(04): movq 0 * SIZE(B), %mm0 movq %mm0, -16 * SIZE(BO) movq %mm0, -15 * SIZE(BO) @@ -526,20 +526,20 @@ addq $4 * SIZE, B addq $8 * SIZE, BO subq $1, %rax - jne .L04 + jne L(04) ALIGN_3 -.L10: +L(10): movq A, AO # aoffset = a leaq (RPREFETCHSIZE + 0) * SIZE(B), BB movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_3 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -601,10 +601,10 @@ leaq (BO, %rax, 8), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -625,7 +625,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -647,7 +647,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -669,7 +669,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -691,7 +691,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -713,7 +713,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -735,7 +735,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -757,7 +757,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -779,17 +779,17 @@ addq $8 * SIZE, %rax BRANCH - jl .L12 + jl L(12) ALIGN_3 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_3 @@ -809,10 +809,10 @@ #else sarq $2, %rax NOBRANCH - jle .L16 + jle L(16) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -826,10 +826,10 @@ subq $-16 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) #endif -.L16: +L(16): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -838,7 +838,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L19 + je L(19) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -846,7 +846,7 @@ negq %rax ALIGN_3 -.L17: +L(17): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movapd -14 * SIZE(BO, %rax, 8), %xmm1 @@ -873,10 +873,10 @@ movapd -10 * SIZE(AO, %rax, 4), %xmm2 addq $SIZE, %rax - jl .L17 + jl L(17) ALIGN_3 -.L19: +L(19): PREFETCH 8 * SIZE(BB) subq $-12 * SIZE, BB @@ -963,18 +963,18 @@ addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_3 -.L20: +L(20): testq $3, M - je .L39 + je L(39) testq $2, M - je .L30 + je L(30) ALIGN_3 -.L21: +L(21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1016,10 +1016,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_3 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1129,10 +1129,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_3 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1141,10 +1141,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_3 -.L26: +L(26): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movapd 2 * SIZE(BO), %xmm1 @@ -1161,10 +1161,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_3 -.L29: +L(29): #ifndef TRMMKERNEL movlpd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 @@ -1214,12 +1214,12 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_3 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_3 -.L31: +L(31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1261,10 +1261,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_3 -.L32: +L(32): mulsd %xmm0, %xmm1 addsd %xmm1, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1373,10 +1373,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_3 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1385,10 +1385,10 @@ movsd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): mulsd %xmm0, %xmm1 addsd %xmm1, %xmm8 movsd 2 * SIZE(BO), %xmm1 @@ -1405,10 +1405,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movsd 0 * SIZE(CO2), %xmm2 @@ -1447,25 +1447,25 @@ #endif ALIGN_3 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C # c += 4 * ldc decq J # j -- - jg .L01 + jg L(01) ALIGN_3 -.L40: +L(40): testq $3, N - je .L999 + je L(999) testq $2, N - je .L80 + je L(80) ALIGN_4 -.L41: +L(41): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -1476,10 +1476,10 @@ movq K, %rax sarq $2, %rax - jle .L43 + jle L(43) ALIGN_3 -.L42: +L(42): PREFETCH 56 * SIZE(B) movq 0 * SIZE(B), %mm0 @@ -1512,17 +1512,17 @@ movq %mm7, -1 * SIZE(BO) decq %rax - jne .L42 + jne L(42) ALIGN_3 -.L43: +L(43): movq K, %rax andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_3 -.L44: +L(44): movq 0 * SIZE(B), %mm0 movq 1 * SIZE(B), %mm1 @@ -1534,20 +1534,20 @@ addq $2 * SIZE, B addq $4 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_3 -.L50: +L(50): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_3 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1594,10 +1594,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_3 -.L52: +L(52): mulpd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(BO), %xmm0 @@ -1709,10 +1709,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_3 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1721,10 +1721,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_3 -.L56: +L(56): movapd 0 * SIZE(BO), %xmm1 mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 @@ -1741,10 +1741,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L59: +L(59): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 @@ -1793,15 +1793,15 @@ addq $4 * SIZE, CO1 # coffset += 4 addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L51 + jg L(51) ALIGN_3 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_3 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1843,10 +1843,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_3 -.L62: +L(62): mulpd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(BO), %xmm0 @@ -1908,10 +1908,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_3 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1920,10 +1920,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_3 -.L66: +L(66): mulpd %xmm0, %xmm1 mulpd 2 * SIZE(BO), %xmm0 addpd %xmm1, %xmm8 @@ -1934,10 +1934,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_3 -.L69: +L(69): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 @@ -1978,12 +1978,12 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_3 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_3 -.L71: +L(71): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2025,10 +2025,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_3 -.L72: +L(72): mulsd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulsd 2 * SIZE(BO), %xmm0 @@ -2089,10 +2089,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_3 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2101,10 +2101,10 @@ movsd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): mulsd %xmm0, %xmm1 mulsd 2 * SIZE(BO), %xmm0 addsd %xmm1, %xmm8 @@ -2115,10 +2115,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_3 -.L78: +L(78): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movsd 0 * SIZE(CO2), %xmm2 @@ -2152,19 +2152,19 @@ #endif ALIGN_3 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif leaq (C, LDC, 2), C ALIGN_3 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L81: +L(81): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -2175,10 +2175,10 @@ movq K, %rax sarq $3, %rax - jle .L83 + jle L(83) ALIGN_3 -.L82: +L(82): PREFETCH 56 * SIZE(B) movq 0 * SIZE(B), %mm0 @@ -2211,17 +2211,17 @@ movq %mm7, -1 * SIZE(BO) decq %rax - jne .L82 + jne L(82) ALIGN_3 -.L83: +L(83): movq K, %rax andq $7, %rax BRANCH - jle .L90 + jle L(90) ALIGN_3 -.L84: +L(84): movq 0 * SIZE(B), %mm0 movq %mm0, 0 * SIZE(BO) @@ -2230,19 +2230,19 @@ addq $1 * SIZE, B addq $2 * SIZE, BO decq %rax - jne .L84 + jne L(84) ALIGN_3 -.L90: +L(90): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_3 -.L91: +L(91): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2286,10 +2286,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_3 -.L92: +L(92): mulpd %xmm1, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd -14 * SIZE(AO), %xmm1 @@ -2346,10 +2346,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_3 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2358,10 +2358,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_3 -.L96: +L(96): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO), %xmm1 addpd %xmm0, %xmm8 @@ -2372,10 +2372,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_3 -.L99: +L(99): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 @@ -2414,15 +2414,15 @@ addq $4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L91 + jg L(91) ALIGN_3 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_3 -.L101: +L(101): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2461,10 +2461,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L105 + je L(105) ALIGN_3 -.L102: +L(102): mulpd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -14 * SIZE(AO), %xmm0 @@ -2497,10 +2497,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_3 -.L105: +L(105): #ifndef TRMMKERNEL movq K, %rax #else @@ -2509,10 +2509,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_3 -.L106: +L(106): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movapd -14 * SIZE(AO), %xmm0 @@ -2521,10 +2521,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L106 + jg L(106) ALIGN_3 -.L109: +L(109): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -2556,12 +2556,12 @@ #endif ALIGN_3 -.L110: +L(110): testq $1, M - je .L999 + je L(999) ALIGN_3 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2600,10 +2600,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_3 -.L112: +L(112): mulsd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -15 * SIZE(AO), %xmm0 @@ -2635,10 +2635,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_3 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2647,10 +2647,10 @@ movsd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_3 -.L116: +L(116): mulsd %xmm0, %xmm1 movsd -15 * SIZE(AO), %xmm0 addsd %xmm1, %xmm8 @@ -2659,10 +2659,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_3 -.L118: +L(118): addsd %xmm10, %xmm8 addsd %xmm11, %xmm9 addsd %xmm9, %xmm8 @@ -2674,7 +2674,7 @@ movsd %xmm8, 0 * SIZE(CO1) ALIGN_3 -.L999: +L(999): movq %rbx, %rsp EMMS diff --git a/kernel/x86_64/gemm_kernel_4x4_sse3.S b/kernel/x86_64/gemm_kernel_4x4_sse3.S index ae153fe8b7..43b49601e8 100644 --- a/kernel/x86_64/gemm_kernel_4x4_sse3.S +++ b/kernel/x86_64/gemm_kernel_4x4_sse3.S @@ -390,10 +390,10 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -409,10 +409,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -471,9 +471,9 @@ andq $-8, %rax salq $4, %rax NOBRANCH - je .L15 + je L(15) -.L1X: +L(1X): KERNEL1 (16 * 0) KERNEL2 (16 * 0) KERNEL3 (16 * 0) @@ -492,7 +492,7 @@ KERNEL16(16 * 0) cmpq $128 * 1, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 1) KERNEL2 (16 * 1) KERNEL3 (16 * 1) @@ -511,7 +511,7 @@ KERNEL16(16 * 1) cmpq $128 * 2, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 2) KERNEL2 (16 * 2) KERNEL3 (16 * 2) @@ -530,7 +530,7 @@ KERNEL16(16 * 2) cmpq $128 * 3, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 3) KERNEL2 (16 * 3) KERNEL3 (16 * 3) @@ -549,7 +549,7 @@ KERNEL16(16 * 3) cmpq $128 * 4, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 4) KERNEL2 (16 * 4) KERNEL3 (16 * 4) @@ -568,7 +568,7 @@ KERNEL16(16 * 4) cmpq $128 * 5, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 5) KERNEL2 (16 * 5) KERNEL3 (16 * 5) @@ -587,7 +587,7 @@ KERNEL16(16 * 5) cmpq $128 * 6, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 6) KERNEL2 (16 * 6) KERNEL3 (16 * 6) @@ -606,7 +606,7 @@ KERNEL16(16 * 6) cmpq $128 * 7, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 7) KERNEL2 (16 * 7) KERNEL3 (16 * 7) @@ -628,18 +628,18 @@ addq $32 * 8 * SIZE, BO subq $128 * 8, %rax BRANCH - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 2), BO # * 64 #else sarq $3, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -866,11 +866,11 @@ addq $32 * SIZE, AO decq %rax BRANCH - jne .L12 + jne L(12) #endif ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else @@ -880,10 +880,10 @@ andq $7, %rax # if (k & 1) BRANCH BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 addpd %xmm9, %xmm0 @@ -915,10 +915,10 @@ addq $4 * SIZE, BO # boffset1 += 8 decq %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): mulpd %xmm15, %xmm0 mulpd %xmm15, %xmm4 mulpd %xmm15, %xmm1 @@ -926,10 +926,10 @@ testq $15, CO1 NOBRANCH - jne .L19x + jne L(19x) testq $15, LDC NOBRANCH - jne .L19x + jne L(19x) mulpd %xmm15, %xmm2 mulpd %xmm15, %xmm3 @@ -975,11 +975,11 @@ addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 - jmp .L20 + jg L(11) + jmp L(20) ALIGN_4 -.L19x: +L(19x): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhpd 1 * SIZE(CO1), %xmm8 @@ -1056,16 +1056,16 @@ addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1103,10 +1103,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1216,10 +1216,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1228,10 +1228,10 @@ movddup ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -1249,10 +1249,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhpd 1 * SIZE(CO1), %xmm8 @@ -1302,12 +1302,12 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_4 -.L31: +L(31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1345,10 +1345,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1410,10 +1410,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1422,10 +1422,10 @@ movddup ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -1437,10 +1437,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhpd 0 * SIZE(CO2), %xmm8 @@ -1474,7 +1474,7 @@ #endif ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif @@ -1482,12 +1482,12 @@ leaq (C, LDC, 4), C # c += 4 * ldc movq BO, B decq J # j -- - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $2, N - je .L80 + je L(80) ALIGN_4 #if defined(TRMMKERNEL) && defined(LEFT) @@ -1505,10 +1505,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1557,10 +1557,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1679,10 +1679,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1691,10 +1691,10 @@ movddup ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 addpd %xmm9, %xmm0 @@ -1713,10 +1713,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhpd 1 * SIZE(CO1), %xmm8 @@ -1765,15 +1765,15 @@ addq $4 * SIZE, CO1 # coffset += 4 addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1811,10 +1811,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1876,10 +1876,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1888,10 +1888,10 @@ movddup ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_4 -.L66: +L(66): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -1903,10 +1903,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L69: +L(69): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhpd 1 * SIZE(CO1), %xmm8 @@ -1946,12 +1946,12 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1989,10 +1989,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movddup 1 * SIZE(AO), %xmm8 @@ -2024,10 +2024,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2036,10 +2036,10 @@ movddup ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulpd %xmm8, %xmm9 movddup 1 * SIZE(AO), %xmm8 addpd %xmm9, %xmm0 @@ -2048,10 +2048,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhpd 0 * SIZE(CO2), %xmm8 @@ -2083,7 +2083,7 @@ #endif ALIGN_4 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -2091,9 +2091,9 @@ movq BO, B ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 #if defined(TRMMKERNEL) && defined(LEFT) @@ -2106,10 +2106,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2153,10 +2153,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(AO), %xmm9 @@ -2211,10 +2211,10 @@ addq $32 * SIZE, AO addq $8 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2223,10 +2223,10 @@ movddup ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_4 -.L96: +L(96): mulpd %xmm9, %xmm8 mulpd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -2237,10 +2237,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L99: +L(99): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhpd 1 * SIZE(CO1), %xmm8 @@ -2279,15 +2279,15 @@ addq $4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2325,10 +2325,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L105 + je L(105) ALIGN_4 -.L102: +L(102): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movddup 1 * SIZE(BO), %xmm9 @@ -2360,10 +2360,10 @@ addq $16 * SIZE, AO addq $ 8 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L105: +L(105): #ifndef TRMMKERNEL movq K, %rax #else @@ -2372,10 +2372,10 @@ movddup ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_4 -.L106: +L(106): mulpd %xmm9, %xmm8 movddup 1 * SIZE(BO), %xmm9 addpd %xmm8, %xmm0 @@ -2384,10 +2384,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L106 + jg L(106) ALIGN_4 -.L109: +L(109): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhpd 1 * SIZE(CO1), %xmm8 @@ -2421,12 +2421,12 @@ addq $2 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L110: +L(110): testq $1, M - je .L999 + je L(999) ALIGN_4 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2469,10 +2469,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulpd %xmm9, %xmm8 movapd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -2491,10 +2491,10 @@ addq $8 * SIZE, AO addq $8 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2503,10 +2503,10 @@ movddup ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd 0 * SIZE(BO), %xmm9 addsd %xmm9, %xmm0 movsd 1 * SIZE(AO), %xmm9 @@ -2514,10 +2514,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 #endif @@ -2532,7 +2532,7 @@ movsd %xmm0, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/gemm_kernel_4x8_nano.S b/kernel/x86_64/gemm_kernel_4x8_nano.S index e29520fa1b..a835d2fe08 100644 --- a/kernel/x86_64/gemm_kernel_4x8_nano.S +++ b/kernel/x86_64/gemm_kernel_4x8_nano.S @@ -160,9 +160,9 @@ movq N, J sarq $3, J - jle .L40 + jle L(40) -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -177,10 +177,10 @@ movq K, %rax sarq $1, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): PREFETCH (RPREFETCHSIZE + 0) * SIZE(B) pshufd $0x50, %xmm1, %xmm0 @@ -215,14 +215,14 @@ addq $32 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $1, %rax BRANCH - jle .L10 + jle L(10) pshufd $0x50, %xmm1, %xmm0 movaps %xmm0, -32 * SIZE(BO) @@ -238,7 +238,7 @@ subq $-16 * SIZE, BO ALIGN_4 -.L10: +L(10): movq C, CO1 leaq (C, LDC, 4), CO2 movq A, AO @@ -247,10 +247,10 @@ movq M, I sarq $2, I - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -307,10 +307,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L16 + jle L(16) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0)(AO) pshufd $0x4e, %xmm1, %xmm3 @@ -434,9 +434,9 @@ addq $ 64 * SIZE, BO decq %rax BRANCH - jg .L12 + jg L(12) -.L16: +L(16): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -445,10 +445,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L18 + je L(18) ALIGN_4 -.L17: +L(17): pshufd $0x4e, %xmm1, %xmm3 mulps %xmm0, %xmm1 addps %xmm1, %xmm8 @@ -481,10 +481,10 @@ addq $ 4 * SIZE, AO subq $-16 * SIZE, BO decq %rax - jg .L17 + jg L(17) ALIGN_4 -.L18: +L(18): leaq (LDC, LDC, 2), %rax mulps %xmm7, %xmm8 @@ -567,12 +567,12 @@ addq $4 * SIZE, CO1 addq $4 * SIZE, CO2 decq I - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -611,10 +611,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L26 + jle L(26) ALIGN_3 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0)(AO) mulps %xmm0, %xmm1 @@ -673,9 +673,9 @@ addq $64 * SIZE, BO decq %rax BRANCH - jg .L22 + jg L(22) -.L26: +L(26): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -684,10 +684,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L28 + je L(28) ALIGN_4 -.L27: +L(27): mulps %xmm0, %xmm1 addps %xmm1, %xmm8 movaps -28 * SIZE(BO), %xmm1 @@ -704,10 +704,10 @@ addq $ 2 * SIZE, AO subq $-16 * SIZE, BO decq %rax - jg .L27 + jg L(27) ALIGN_4 -.L28: +L(28): leaq (LDC, LDC, 2), %rax mulps %xmm7, %xmm8 @@ -761,9 +761,9 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -803,10 +803,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L36 + jle L(36) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0)(AO) shufps $0, %xmm0, %xmm0 @@ -869,9 +869,9 @@ addq $64 * SIZE, BO decq %rax BRANCH - jg .L32 + jg L(32) -.L36: +L(36): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -880,10 +880,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L38 + je L(38) ALIGN_4 -.L37: +L(37): shufps $0, %xmm0, %xmm0 mulps %xmm0, %xmm1 addps %xmm1, %xmm8 @@ -901,10 +901,10 @@ addq $ 1 * SIZE, AO subq $-16 * SIZE, BO decq %rax - jg .L37 + jg L(37) ALIGN_4 -.L38: +L(38): leaq (LDC, LDC, 2), %rax mulps %xmm7, %xmm8 @@ -956,18 +956,18 @@ #endif ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $8, KK #endif leaq (C, LDC, 8), C decq J - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $4, N - jle .L80 + jle L(80) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -983,10 +983,10 @@ movq K, %rax sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): PREFETCH (RPREFETCHSIZE + 0) * SIZE(B) pshufd $0x50, %xmm1, %xmm0 @@ -1021,17 +1021,17 @@ addq $32 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): movq K, %rax andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L45: +L(45): pshufd $0x50, %xmm1, %xmm0 movaps %xmm0, -32 * SIZE(BO) pshufd $0xfa, %xmm1, %xmm1 @@ -1041,20 +1041,20 @@ addq $ 4 * SIZE, B subq $-8 * SIZE, BO decq %rax - jne .L45 + jne L(45) ALIGN_4 -.L50: +L(50): movq C, CO1 leaq (C, LDC, 2), CO2 movq A, AO movq M, I sarq $2, I - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1097,10 +1097,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L56 + jle L(56) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0)(AO) pshufd $0x4e, %xmm1, %xmm3 @@ -1167,9 +1167,9 @@ subq $-32 * SIZE, BO decq %rax BRANCH - jg .L52 + jg L(52) -.L56: +L(56): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1178,10 +1178,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L58 + je L(58) ALIGN_4 -.L57: +L(57): pshufd $0x4e, %xmm1, %xmm3 mulps %xmm0, %xmm1 addps %xmm1, %xmm8 @@ -1200,10 +1200,10 @@ addq $ 4 * SIZE, AO subq $-8 * SIZE, BO decq %rax - jg .L57 + jg L(57) ALIGN_4 -.L58: +L(58): mulps %xmm7, %xmm8 mulps %xmm7, %xmm9 mulps %xmm7, %xmm10 @@ -1252,12 +1252,12 @@ addq $4 * SIZE, CO1 addq $4 * SIZE, CO2 decq I - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1296,10 +1296,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L66 + jle L(66) ALIGN_3 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0)(AO) mulps %xmm0, %xmm1 @@ -1334,9 +1334,9 @@ subq $-32 * SIZE, BO decq %rax BRANCH - jg .L62 + jg L(62) -.L66: +L(66): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1345,10 +1345,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L68 + je L(68) ALIGN_4 -.L67: +L(67): mulps %xmm0, %xmm1 mulps -28 * SIZE(BO), %xmm0 addps %xmm1, %xmm8 @@ -1359,10 +1359,10 @@ addq $ 2 * SIZE, AO subq $-8 * SIZE, BO decq %rax - jg .L67 + jg L(67) ALIGN_4 -.L68: +L(68): mulps %xmm7, %xmm8 mulps %xmm7, %xmm9 @@ -1400,9 +1400,9 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1439,10 +1439,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L76 + jle L(76) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0)(AO) shufps $0, %xmm0, %xmm0 @@ -1481,9 +1481,9 @@ subq $-32 * SIZE, BO decq %rax BRANCH - jg .L72 + jg L(72) -.L76: +L(76): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1492,10 +1492,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L78 + je L(78) ALIGN_4 -.L77: +L(77): shufps $0, %xmm0, %xmm0 mulps %xmm0, %xmm1 mulps -28 * SIZE(BO), %xmm0 @@ -1507,10 +1507,10 @@ addq $ 1 * SIZE, AO subq $-8 * SIZE, BO decq %rax - jg .L77 + jg L(77) ALIGN_4 -.L78: +L(78): mulps %xmm7, %xmm8 mulps %xmm7, %xmm9 @@ -1544,16 +1544,16 @@ #endif ALIGN_4 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C ALIGN_4 -.L80: +L(80): testq $2, N - jle .L120 + jle L(120) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1567,10 +1567,10 @@ movq K, %rax sarq $2, %rax - jle .L83 + jle L(83) ALIGN_4 -.L82: +L(82): pshufd $0x50, %xmm1, %xmm0 movaps %xmm0, -32 * SIZE(BO) pshufd $0xfa, %xmm1, %xmm1 @@ -1589,17 +1589,17 @@ subq $-16 * SIZE, BO decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L83: +L(83): movq K, %rax andq $3, %rax BRANCH - jle .L90 + jle L(90) ALIGN_4 -.L85: +L(85): pshufd $0x50, %xmm1, %xmm0 movaps %xmm0, -32 * SIZE(BO) movsd 2 * SIZE(B), %xmm1 @@ -1607,20 +1607,20 @@ addq $ 2 * SIZE, B subq $-4 * SIZE, BO decq %rax - jne .L85 + jne L(85) ALIGN_4 -.L90: +L(90): movq C, CO1 leaq (C, LDC), CO2 movq A, AO movq M, I sarq $2, I - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1659,10 +1659,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L96 + jle L(96) ALIGN_3 -.L92: +L(92): PREFETCH (PREFETCHSIZE + 0)(AO) pshufd $0x4e, %xmm1, %xmm3 @@ -1701,9 +1701,9 @@ subq $-16 * SIZE, BO decq %rax BRANCH - jg .L92 + jg L(92) -.L96: +L(96): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1712,10 +1712,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L98 + je L(98) ALIGN_4 -.L97: +L(97): pshufd $0x4e, %xmm1, %xmm3 mulps %xmm0, %xmm1 addps %xmm1, %xmm8 @@ -1727,10 +1727,10 @@ addq $ 4 * SIZE, AO subq $-4 * SIZE, BO decq %rax - jg .L97 + jg L(97) ALIGN_4 -.L98: +L(98): mulps %xmm7, %xmm8 mulps %xmm7, %xmm9 @@ -1765,12 +1765,12 @@ addq $4 * SIZE, CO1 addq $4 * SIZE, CO2 decq I - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1806,10 +1806,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L106 + jle L(106) ALIGN_3 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0)(AO) mulps %xmm0, %xmm1 @@ -1836,9 +1836,9 @@ subq $-16 * SIZE, BO decq %rax BRANCH - jg .L102 + jg L(102) -.L106: +L(106): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1847,10 +1847,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L108 + je L(108) ALIGN_4 -.L107: +L(107): mulps %xmm0, %xmm1 movddup -30 * SIZE(AO), %xmm0 addps %xmm1, %xmm8 @@ -1859,10 +1859,10 @@ addq $ 2 * SIZE, AO subq $-4 * SIZE, BO decq %rax - jg .L107 + jg L(107) ALIGN_4 -.L108: +L(108): mulps %xmm7, %xmm8 #ifndef TRMMKERNEL @@ -1892,9 +1892,9 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L110: +L(110): testq $1, M - je .L119 + je L(119) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1930,10 +1930,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L116 + jle L(116) ALIGN_3 -.L112: +L(112): PREFETCH (PREFETCHSIZE + 0)(AO) shufps $0, %xmm0, %xmm0 @@ -1964,9 +1964,9 @@ subq $-16 * SIZE, BO decq %rax BRANCH - jg .L112 + jg L(112) -.L116: +L(116): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1975,10 +1975,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L118 + je L(118) ALIGN_4 -.L117: +L(117): shufps $0, %xmm0, %xmm0 mulps %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 @@ -1988,10 +1988,10 @@ addq $ 1 * SIZE, AO subq $-4 * SIZE, BO decq %rax - jg .L117 + jg L(117) ALIGN_4 -.L118: +L(118): mulps %xmm7, %xmm8 movhlps %xmm8, %xmm9 @@ -2017,16 +2017,16 @@ #endif ALIGN_4 -.L119: +L(119): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif leaq (C, LDC, 2), C ALIGN_4 -.L120: +L(120): testq $1, N - jle .L999 + jle L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -2040,10 +2040,10 @@ movq K, %rax sarq $2, %rax - jle .L123 + jle L(123) ALIGN_4 -.L122: +L(122): pshufd $0x50, %xmm1, %xmm0 movaps %xmm0, -32 * SIZE(BO) pshufd $0xfa, %xmm1, %xmm1 @@ -2056,17 +2056,17 @@ subq $-8 * SIZE, BO decq %rax - jne .L122 + jne L(122) ALIGN_4 -.L123: +L(123): movq K, %rax andq $3, %rax BRANCH - jle .L130 + jle L(130) ALIGN_4 -.L125: +L(125): pshufd $0x50, %xmm1, %xmm0 movlps %xmm0, -32 * SIZE(BO) movss 1 * SIZE(B), %xmm1 @@ -2074,19 +2074,19 @@ addq $ 1 * SIZE, B subq $-2 * SIZE, BO decq %rax - jne .L125 + jne L(125) ALIGN_4 -.L130: +L(130): movq C, CO1 movq A, AO movq M, I sarq $2, I - jle .L140 + jle L(140) ALIGN_4 -.L131: +L(131): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2123,10 +2123,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L136 + jle L(136) ALIGN_3 -.L132: +L(132): PREFETCH (PREFETCHSIZE + 0)(AO) mulps %xmm0, %xmm1 @@ -2153,9 +2153,9 @@ subq $ -8 * SIZE, BO decq %rax BRANCH - jg .L132 + jg L(132) -.L136: +L(136): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2164,10 +2164,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L138 + je L(138) ALIGN_4 -.L137: +L(137): mulps %xmm0, %xmm1 movaps -28 * SIZE(AO), %xmm0 addps %xmm1, %xmm8 @@ -2176,10 +2176,10 @@ addq $ 4 * SIZE, AO subq $-2 * SIZE, BO decq %rax - jg .L137 + jg L(137) ALIGN_4 -.L138: +L(138): mulps %xmm7, %xmm8 #ifndef TRMMKERNEL @@ -2207,12 +2207,12 @@ addq $4 * SIZE, CO1 decq I - jg .L131 + jg L(131) ALIGN_4 -.L140: +L(140): testq $2, M - je .L150 + je L(150) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2248,10 +2248,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L146 + jle L(146) ALIGN_3 -.L142: +L(142): PREFETCH (PREFETCHSIZE + 0)(AO) mulps %xmm0, %xmm1 @@ -2278,9 +2278,9 @@ subq $-8 * SIZE, BO decq %rax BRANCH - jg .L142 + jg L(142) -.L146: +L(146): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2289,10 +2289,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L148 + je L(148) ALIGN_4 -.L147: +L(147): mulps %xmm0, %xmm1 movddup -30 * SIZE(AO), %xmm0 addps %xmm1, %xmm8 @@ -2301,10 +2301,10 @@ addq $ 2 * SIZE, AO subq $-2 * SIZE, BO decq %rax - jg .L147 + jg L(147) ALIGN_4 -.L148: +L(148): mulps %xmm7, %xmm8 #ifndef TRMMKERNEL @@ -2331,9 +2331,9 @@ addq $2 * SIZE, CO1 ALIGN_4 -.L150: +L(150): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2369,10 +2369,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L156 + jle L(156) ALIGN_3 -.L152: +L(152): PREFETCH (PREFETCHSIZE + 0)(AO) mulss %xmm0, %xmm1 @@ -2399,9 +2399,9 @@ subq $-8 * SIZE, BO decq %rax BRANCH - jg .L152 + jg L(152) -.L156: +L(156): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2410,10 +2410,10 @@ movq KKK, %rax #endif andq $3, %rax - je .L158 + je L(158) ALIGN_4 -.L157: +L(157): mulss %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addss %xmm1, %xmm8 @@ -2422,10 +2422,10 @@ addq $1 * SIZE, AO addq $2 * SIZE, BO decq %rax - jg .L157 + jg L(157) ALIGN_4 -.L158: +L(158): mulss %xmm7, %xmm8 #ifndef TRMMKERNEL @@ -2448,7 +2448,7 @@ #endif ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/gemm_kernel_4x8_nehalem.S b/kernel/x86_64/gemm_kernel_4x8_nehalem.S index 549ea13b3f..75cf5c3f97 100644 --- a/kernel/x86_64/gemm_kernel_4x8_nehalem.S +++ b/kernel/x86_64/gemm_kernel_4x8_nehalem.S @@ -164,10 +164,10 @@ movq N, J sarq $3, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -184,10 +184,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): prefetcht2 -32 * SIZE(BB) subq $-16 * SIZE, BB @@ -254,10 +254,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm12 @@ -380,10 +380,10 @@ subq $-16 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -393,10 +393,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addps %xmm1, %xmm12 movaps -32 * SIZE(BO), %xmm1 addps %xmm2, %xmm13 @@ -432,10 +432,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): addps %xmm1, %xmm12 addps %xmm2, %xmm13 addps %xmm3, %xmm14 @@ -527,13 +527,13 @@ addq $4 * SIZE, CO2 decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -578,10 +578,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -659,10 +659,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -672,10 +672,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -699,10 +699,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): addps %xmm1, %xmm8 addps %xmm2, %xmm9 addps %xmm3, %xmm10 @@ -759,10 +759,10 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L30: +L(30): testq $1, M BRANCH - jle .L39 + jle L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -801,10 +801,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movaps -32 * SIZE(BO), %xmm2 @@ -844,10 +844,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -857,10 +857,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -875,10 +875,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): addps %xmm2, %xmm8 addps %xmm3, %xmm12 @@ -934,7 +934,7 @@ ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addq $8, KK #endif @@ -945,12 +945,12 @@ subq $1, J BRANCH - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $4, N - jle .L70 + jle L(70) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -964,10 +964,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L50 + jle L(50) ALIGN_4 -.L41: +L(41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1014,10 +1014,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -1084,10 +1084,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1097,10 +1097,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): addps %xmm1, %xmm8 movaps -32 * SIZE(BO), %xmm1 addps %xmm2, %xmm9 @@ -1121,10 +1121,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): addps %xmm1, %xmm8 addps %xmm2, %xmm9 addps %xmm3, %xmm10 @@ -1192,13 +1192,13 @@ addq $4 * SIZE, CO2 decq I BRANCH - jg .L41 + jg L(41) ALIGN_4 -.L50: +L(50): testq $2, M BRANCH - jle .L60 + jle L(60) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1238,10 +1238,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1283,10 +1283,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1296,10 +1296,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1314,10 +1314,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): addps %xmm1, %xmm8 addps %xmm2, %xmm9 @@ -1356,10 +1356,10 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L69 + jle L(69) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1397,10 +1397,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movaps -32 * SIZE(BO), %xmm2 @@ -1428,11 +1428,11 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) addps %xmm9, %xmm8 ALIGN_3 -.L65: +L(65): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1442,10 +1442,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -1457,10 +1457,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): addps %xmm2, %xmm8 mulps %xmm7, %xmm8 @@ -1496,7 +1496,7 @@ ALIGN_4 -.L69: +L(69): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif @@ -1506,9 +1506,9 @@ leaq (C, LDC, 4), C ALIGN_4 -.L70: +L(70): testq $2, N - jle .L100 + jle L(100) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1522,10 +1522,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L80 + jle L(80) ALIGN_4 -.L71: +L(71): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1567,10 +1567,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -1613,10 +1613,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_3 -.L75: +L(75): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1626,10 +1626,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addps %xmm1, %xmm8 pshufd $0x00, %xmm3, %xmm1 mulps %xmm0, %xmm1 @@ -1644,10 +1644,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_3 -.L78: +L(78): addps %xmm1, %xmm8 addps %xmm2, %xmm9 @@ -1686,13 +1686,13 @@ addq $4 * SIZE, CO2 decq I BRANCH - jg .L71 + jg L(71) ALIGN_4 -.L80: +L(80): testq $2, M BRANCH - jle .L90 + jle L(90) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1732,10 +1732,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): addps %xmm1, %xmm8 movsd -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -1765,10 +1765,10 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) ALIGN_3 -.L85: +L(85): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1778,10 +1778,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): addps %xmm1, %xmm8 movsd -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -1793,10 +1793,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_3 -.L88: +L(88): addps %xmm1, %xmm8 mulps %xmm7, %xmm8 @@ -1827,10 +1827,10 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L90: +L(90): testq $1, M BRANCH - jle .L99 + jle L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1868,10 +1868,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_3 -.L92: +L(92): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movsd -32 * SIZE(BO), %xmm2 @@ -1899,11 +1899,11 @@ subq $1, %rax BRANCH - jg .L92 + jg L(92) addps %xmm9, %xmm8 ALIGN_3 -.L95: +L(95): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1913,10 +1913,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_3 -.L96: +L(96): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -1928,10 +1928,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_3 -.L98: +L(98): addps %xmm2, %xmm8 mulps %xmm7, %xmm8 @@ -1961,7 +1961,7 @@ ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -1971,9 +1971,9 @@ leaq (C, LDC, 2), C ALIGN_4 -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1986,10 +1986,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L110 + jle L(110) ALIGN_4 -.L101: +L(101): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2027,10 +2027,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_3 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -2061,10 +2061,10 @@ subq $ -4 * SIZE, BO subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_3 -.L105: +L(105): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2074,10 +2074,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L108 + je L(108) ALIGN_3 -.L106: +L(106): addps %xmm1, %xmm8 pshufd $0x00, %xmm3, %xmm1 movss -31 * SIZE(BO), %xmm3 @@ -2089,10 +2089,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_3 -.L108: +L(108): addps %xmm1, %xmm8 mulps %xmm7, %xmm8 @@ -2123,13 +2123,13 @@ addq $4 * SIZE, CO1 decq I BRANCH - jg .L101 + jg L(101) ALIGN_4 -.L110: +L(110): testq $2, M BRANCH - jle .L120 + jle L(120) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2166,10 +2166,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_3 -.L112: +L(112): addps %xmm1, %xmm8 movss -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -2199,10 +2199,10 @@ subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_3 -.L115: +L(115): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2212,10 +2212,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_3 -.L116: +L(116): addps %xmm1, %xmm8 movss -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -2227,10 +2227,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_3 -.L118: +L(118): addps %xmm1, %xmm8 mulps %xmm7, %xmm8 @@ -2257,10 +2257,10 @@ addq $2 * SIZE, CO1 ALIGN_4 -.L120: +L(120): testq $1, M BRANCH - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2297,10 +2297,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L125 + jle L(125) ALIGN_3 -.L122: +L(122): addss %xmm2, %xmm8 movss -32 * SIZE(BO), %xmm2 mulss %xmm0, %xmm2 @@ -2326,10 +2326,10 @@ subq $1, %rax BRANCH - jg .L122 + jg L(122) ALIGN_3 -.L125: +L(125): movddup ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2339,10 +2339,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_3 -.L126: +L(126): addss %xmm2, %xmm8 movss -32 * SIZE(BO), %xmm2 mulss %xmm0, %xmm2 @@ -2353,10 +2353,10 @@ subq $1, %rax BRANCH - jg .L126 + jg L(126) ALIGN_3 -.L128: +L(128): addps %xmm2, %xmm8 mulps %xmm7, %xmm8 @@ -2368,7 +2368,7 @@ movss %xmm8, (CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/gemm_kernel_8x4_barcelona.S b/kernel/x86_64/gemm_kernel_8x4_barcelona.S index 1849565914..e49aa189c1 100644 --- a/kernel/x86_64/gemm_kernel_8x4_barcelona.S +++ b/kernel/x86_64/gemm_kernel_8x4_barcelona.S @@ -473,9 +473,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L50 + jle L(50) -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -486,10 +486,10 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): prefetch (RPREFETCHSIZE + 0) * SIZE(B) movaps 0 * SIZE(B), %xmm3 @@ -547,17 +547,17 @@ addq $64 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movaps 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -573,10 +573,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): movq C, CO1 leaq (C, LDC, 1), CO2 movq A, AO @@ -585,10 +585,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -646,10 +646,10 @@ leaq (BO, %rax, 8), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_3 -.L12: +L(12): KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -659,7 +659,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -669,7 +669,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -679,7 +679,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -689,7 +689,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -699,7 +699,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -709,7 +709,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -719,7 +719,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -729,10 +729,10 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) BRANCH - jl .L12 + jl L(12) ALIGN_4 -.L15: +L(15): prefetch 16 * SIZE(BB) subq $-32 * SIZE, BB @@ -744,7 +744,7 @@ movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_3 @@ -757,14 +757,14 @@ addq $64 * SIZE, BO ALIGN_3 -.L16: +L(16): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L18 + je L(18) leaq (, %rax, 8), %rax leaq (AO, %rax, 4), AO @@ -772,7 +772,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulps %xmm1, %xmm0 mulps -28 * SIZE(AO, %rax, 4), %xmm1 addps %xmm0, %xmm8 @@ -799,10 +799,10 @@ movaps %xmm0, %xmm2 addq $SIZE * 2, %rax - jl .L17 + jl L(17) ALIGN_4 -.L18: +L(18): #ifndef TRMMKERNEL movups 0 * SIZE(CO1), %xmm0 movups 4 * SIZE(CO1), %xmm1 @@ -875,12 +875,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -924,10 +924,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) @@ -1041,10 +1041,10 @@ addq $ 32 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1053,10 +1053,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1073,10 +1073,10 @@ addq $ 4 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): mulps %xmm15, %xmm0 mulps %xmm15, %xmm1 mulps %xmm15, %xmm2 @@ -1126,9 +1126,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $2, M - je .L40 + je L(40) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1172,10 +1172,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) @@ -1294,10 +1294,10 @@ addq $ 16 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1306,10 +1306,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movsd 4 * SIZE(BO), %xmm9 @@ -1327,10 +1327,10 @@ addq $ 2 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): mulps %xmm15, %xmm0 mulps %xmm15, %xmm1 mulps %xmm15, %xmm2 @@ -1370,9 +1370,9 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L40: +L(40): testq $1, M - je .L49 + je L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1417,10 +1417,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): mulss %xmm8, %xmm9 addss %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) @@ -1539,10 +1539,10 @@ addq $ 8 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1551,10 +1551,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -1572,10 +1572,10 @@ addq $ 1 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): mulss %xmm15, %xmm0 mulss %xmm15, %xmm1 mulss %xmm15, %xmm2 @@ -1613,19 +1613,19 @@ #endif ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C # c += 4 * ldc decq J # j -- - jg .L01 + jg L(01) -.L50: +L(50): testq $2, N - je .L100 + je L(100) -.L51: +L(51): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1636,10 +1636,10 @@ movq K, %rax sarq $2, %rax - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): prefetch (RPREFETCHSIZE + 0) * SIZE(B) movaps 0 * SIZE(B), %xmm3 @@ -1672,17 +1672,17 @@ addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): movq K, %rax andq $3, %rax BRANCH - jle .L60 + jle L(60) ALIGN_4 -.L54: +L(54): movsd 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -1699,20 +1699,20 @@ addq $ 2 * SIZE, B addq $ 8 * SIZE, BO decq %rax - jne .L54 + jne L(54) ALIGN_4 -.L60: +L(60): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L70 + jle L(70) ALIGN_4 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1760,10 +1760,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1883,10 +1883,10 @@ addq $64 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1895,10 +1895,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -1915,10 +1915,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -1969,12 +1969,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L61 + jg L(61) ALIGN_4 -.L70: +L(70): testq $4, M - je .L80 + je L(80) #if !defined(TRMMKERNEL) || \ @@ -2019,10 +2019,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2086,10 +2086,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2098,10 +2098,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -2112,10 +2112,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2156,9 +2156,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L80: +L(80): testq $2, M - je .L90 + je L(90) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2202,10 +2202,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L85 + je L(85) ALIGN_4 -.L82: +L(82): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) @@ -2276,10 +2276,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L85: +L(85): #ifndef TRMMKERNEL movq K, %rax #else @@ -2288,10 +2288,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movsd 4 * SIZE(BO), %xmm9 @@ -2303,10 +2303,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movsd 0 * SIZE(CO2), %xmm10 @@ -2343,9 +2343,9 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L90: +L(90): testq $1, M - je .L99 + je L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2389,10 +2389,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) @@ -2463,10 +2463,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2475,10 +2475,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -2490,10 +2490,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): #ifndef TRMMKERNEL movss 0 * SIZE(CO1), %xmm8 movss 0 * SIZE(CO2), %xmm10 @@ -2526,7 +2526,7 @@ #endif ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -2534,11 +2534,11 @@ ALIGN_4 -.L100: +L(100): testq $1, N - je .L999 + je L(999) -.L101: +L(101): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -2549,11 +2549,11 @@ movq K, %rax sarq $3, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): prefetch (RPREFETCHSIZE + 0) * SIZE(B) movups 0 * SIZE(B), %xmm3 @@ -2586,17 +2586,17 @@ addq $32 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): movq K, %rax andq $7, %rax BRANCH - jle .L110 + jle L(110) ALIGN_4 -.L104: +L(104): movss 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -2606,19 +2606,19 @@ addq $ 1 * SIZE, B addq $ 4 * SIZE, BO decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L110: +L(110): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L120 + jle L(120) ALIGN_4 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2665,10 +2665,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulps %xmm9, %xmm8 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2741,10 +2741,10 @@ addq $64 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2753,10 +2753,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulps %xmm9, %xmm8 mulps -28 * SIZE(AO), %xmm9 addps %xmm8, %xmm0 @@ -2767,10 +2767,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2805,12 +2805,12 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L111 + jg L(111) ALIGN_4 -.L120: +L(120): testq $4, M - je .L130 + je L(130) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2852,10 +2852,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L125 + je L(125) ALIGN_4 -.L122: +L(122): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2893,10 +2893,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L122 + jne L(122) ALIGN_4 -.L125: +L(125): #ifndef TRMMKERNEL movq K, %rax #else @@ -2905,10 +2905,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): mulps %xmm8, %xmm9 movaps -28 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -2917,10 +2917,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2954,9 +2954,9 @@ addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L130: +L(130): testq $2, M - je .L140 + je L(140) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2998,10 +2998,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L135 + je L(135) ALIGN_4 -.L132: +L(132): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3047,10 +3047,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L132 + jne L(132) ALIGN_4 -.L135: +L(135): #ifndef TRMMKERNEL movq K, %rax #else @@ -3059,10 +3059,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): mulps %xmm8, %xmm9 movsd -30 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -3071,10 +3071,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): addps %xmm1, %xmm0 mulps %xmm15, %xmm0 @@ -3101,9 +3101,9 @@ addq $2 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L140: +L(140): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3145,10 +3145,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L145 + je L(145) ALIGN_4 -.L142: +L(142): mulss %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3182,10 +3182,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L142 + jne L(142) ALIGN_4 -.L145: +L(145): #ifndef TRMMKERNEL movq K, %rax #else @@ -3194,10 +3194,10 @@ movss ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): mulss %xmm8, %xmm9 movss -31 * SIZE(AO), %xmm8 addss %xmm9, %xmm0 @@ -3206,10 +3206,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): addss %xmm1, %xmm0 addss %xmm3, %xmm2 addss %xmm2, %xmm0 @@ -3223,7 +3223,7 @@ movss %xmm0, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/gemm_kernel_8x4_core2.S b/kernel/x86_64/gemm_kernel_8x4_core2.S index c31dc90dc1..cc2f89b2ac 100644 --- a/kernel/x86_64/gemm_kernel_8x4_core2.S +++ b/kernel/x86_64/gemm_kernel_8x4_core2.S @@ -161,10 +161,10 @@ movq N, J sarq $2, J - jle .L50 + jle L(50) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -177,10 +177,10 @@ movq K, %rax sarq $2, %rax - jle .L05 + jle L(05) ALIGN_4 -.L02: +L(02): prefetcht0 (PREFETCH_R + 0) * SIZE(B) movaps -28 * SIZE(B), %xmm7 movaps -24 * SIZE(B), %xmm11 @@ -233,17 +233,17 @@ subq $-16 * SIZE, B subq $-64 * SIZE, BO subq $1, %rax - jne .L02 + jne L(02) ALIGN_4 -.L05: +L(05): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L06: +L(06): movaps -32 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -259,10 +259,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO subq $1, %rax - jne .L06 + jne L(06) ALIGN_4 -.L10: +L(10): leaq (PREFETCH_R + 0) * SIZE(B), BB movq C, CO1 # coffset1 = c @@ -271,10 +271,10 @@ movq M, I sarq $3, I - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -330,10 +330,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): addps %xmm2, %xmm10 movaps -32 * SIZE(BO), %xmm2 addps %xmm3, %xmm14 @@ -463,10 +463,10 @@ subq $-64 * SIZE, BO subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): prefetcht2 -16 * SIZE(BB) #ifndef TRMMKERNEL @@ -476,10 +476,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): addps %xmm2, %xmm10 movaps -32 * SIZE(BO), %xmm2 addps %xmm3, %xmm14 @@ -514,10 +514,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): movaps ALPHA, %xmm7 addps %xmm2, %xmm10 @@ -602,12 +602,12 @@ addq $8 * SIZE, CO1 addq $8 * SIZE, CO2 subq $1, I - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M - jle .L30 + jle L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -643,10 +643,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L25 + jle L(25) ALIGN_4 -.L21: +L(21): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -716,10 +716,10 @@ subq $-16 * SIZE, AO subq $-64 * SIZE, BO subq $1, %rax - jg .L21 + jg L(21) ALIGN_4 -.L25: +L(25): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -728,10 +728,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): movaps -32 * SIZE(AO), %xmm0 movaps -32 * SIZE(BO), %xmm2 movaps -28 * SIZE(BO), %xmm3 @@ -751,10 +751,10 @@ addq $ 4 * SIZE, AO addq $16 * SIZE, BO subq $1, %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -807,9 +807,9 @@ subq $1, I ALIGN_4 -.L30: +L(30): testq $2, M - jle .L40 + jle L(40) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -850,10 +850,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L35 + jle L(35) ALIGN_4 -.L31: +L(31): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -32 * SIZE(AO), %xmm0 @@ -923,10 +923,10 @@ subq $ -8 * SIZE, AO subq $-64 * SIZE, BO subq $1, %rax - jg .L31 + jg L(31) ALIGN_4 -.L35: +L(35): movsd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -935,10 +935,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): movsd -32 * SIZE(AO), %xmm0 movsd -32 * SIZE(BO), %xmm2 movsd -28 * SIZE(BO), %xmm3 @@ -958,10 +958,10 @@ addq $ 2 * SIZE, AO addq $16 * SIZE, BO subq $1, %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movsd 0 * SIZE(CO2), %xmm2 @@ -1003,9 +1003,9 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L40: +L(40): testq $1, M - jle .L49 + jle L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1047,10 +1047,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L45 + jle L(45) ALIGN_4 -.L41: +L(41): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movss -32 * SIZE(AO), %xmm0 @@ -1120,10 +1120,10 @@ subq $ -4 * SIZE, AO subq $-64 * SIZE, BO subq $1, %rax - jg .L41 + jg L(41) ALIGN_4 -.L45: +L(45): movss ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1132,10 +1132,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): movss -32 * SIZE(AO), %xmm0 movss -32 * SIZE(BO), %xmm2 movss -28 * SIZE(BO), %xmm3 @@ -1155,10 +1155,10 @@ addq $ 1 * SIZE, AO addq $16 * SIZE, BO subq $1, %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): #ifndef TRMMKERNEL movss 0 * SIZE(CO1), %xmm0 movss 0 * SIZE(CO2), %xmm2 @@ -1198,22 +1198,22 @@ #endif ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C subq $1, J - jg .L01 + jg L(01) ALIGN_4 -.L50: +L(50): testq $2, N - jle .L100 + jle L(100) ALIGN_4 -.L51: +L(51): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -1224,12 +1224,12 @@ movq K, %rax sarq $3, %rax - jle .L53 + jle L(53) addq %rax, %rax ALIGN_4 -.L52: +L(52): movaps -32 * SIZE(B), %xmm3 movaps -28 * SIZE(B), %xmm7 @@ -1259,17 +1259,17 @@ addq $32 * SIZE, BO subq $1, %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): movq K, %rax andq $7, %rax BRANCH - jle .L55 + jle L(55) ALIGN_4 -.L54: +L(54): movss -32 * SIZE(B), %xmm8 movss -31 * SIZE(B), %xmm9 @@ -1282,20 +1282,20 @@ addq $2 * SIZE, B addq $8 * SIZE, BO subq $1, %rax - jne .L54 + jne L(54) ALIGN_4 -.L55: +L(55): movq C, CO1 leaq (C, LDC, 1), CO2 movq A, AO # aoffset = a movq M, I sarq $3, I - jle .L70 + jle L(70) ALIGN_4 -.L60: +L(60): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1333,10 +1333,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L65 + jle L(65) ALIGN_4 -.L61: +L(61): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -1414,10 +1414,10 @@ subq $-32 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jg .L61 + jg L(61) ALIGN_4 -.L65: +L(65): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1426,10 +1426,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): movaps -32 * SIZE(AO), %xmm0 movaps -28 * SIZE(AO), %xmm1 @@ -1451,10 +1451,10 @@ addq $8 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -1503,12 +1503,12 @@ addq $8 * SIZE, CO1 addq $8 * SIZE, CO2 subq $1, I - jg .L60 + jg L(60) ALIGN_4 -.L70: +L(70): testq $4, M - jle .L80 + jle L(80) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1544,10 +1544,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L75 + jle L(75) ALIGN_4 -.L71: +L(71): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -1587,10 +1587,10 @@ subq $-16 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jg .L71 + jg L(71) ALIGN_4 -.L75: +L(75): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1599,10 +1599,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): movaps -32 * SIZE(AO), %xmm0 movaps -32 * SIZE(BO), %xmm2 movaps -28 * SIZE(BO), %xmm3 @@ -1616,10 +1616,10 @@ addq $4 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm10, %xmm8 addps %xmm11, %xmm9 @@ -1659,9 +1659,9 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L80: +L(80): testq $2, M - jle .L90 + jle L(90) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1697,10 +1697,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L85 + jle L(85) ALIGN_4 -.L81: +L(81): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -32 * SIZE(AO), %xmm0 @@ -1740,10 +1740,10 @@ subq $ -8 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jg .L81 + jg L(81) ALIGN_4 -.L85: +L(85): movsd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1752,10 +1752,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): movsd -32 * SIZE(AO), %xmm0 movsd -32 * SIZE(BO), %xmm2 movsd -28 * SIZE(BO), %xmm3 @@ -1769,10 +1769,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): addps %xmm10, %xmm8 addps %xmm11, %xmm9 @@ -1809,9 +1809,9 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L90: +L(90): testq $1, M - jle .L99 + jle L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1847,10 +1847,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L95 + jle L(95) ALIGN_4 -.L91: +L(91): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1891,10 +1891,10 @@ subq $ -4 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jg .L91 + jg L(91) ALIGN_4 -.L95: +L(95): movss ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1903,10 +1903,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): movss -32 * SIZE(AO), %xmm0 movss -32 * SIZE(BO), %xmm2 movss -28 * SIZE(BO), %xmm3 @@ -1920,10 +1920,10 @@ addq $1 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): addss %xmm10, %xmm8 addss %xmm11, %xmm9 @@ -1957,7 +1957,7 @@ #endif ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -1967,12 +1967,12 @@ -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) ALIGN_4 -.L101: +L(101): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -1983,12 +1983,12 @@ movq K, %rax sarq $4, %rax - jle .L103 + jle L(103) addq %rax, %rax ALIGN_4 -.L102: +L(102): movss -32 * SIZE(B), %xmm0 movss -31 * SIZE(B), %xmm1 movss -30 * SIZE(B), %xmm2 @@ -2019,17 +2019,17 @@ addq $ 8 * SIZE, B subq $-32 * SIZE, BO subq $1, %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): movq K, %rax andq $15, %rax BRANCH - jle .L105 + jle L(105) ALIGN_4 -.L104: +L(104): movss -32 * SIZE(B), %xmm8 shufps $0, %xmm8, %xmm8 @@ -2039,19 +2039,19 @@ addq $1 * SIZE, B addq $4 * SIZE, BO subq $1, %rax - jne .L104 + jne L(104) ALIGN_4 -.L105: +L(105): movq C, CO1 movq A, AO movq M, I sarq $3, I - jle .L120 + jle L(120) ALIGN_4 -.L110: +L(110): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2088,10 +2088,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L115 + jle L(115) ALIGN_4 -.L111: +L(111): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -2137,10 +2137,10 @@ subq $-32 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L111 + jg L(111) ALIGN_4 -.L115: +L(115): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2149,10 +2149,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): movaps -32 * SIZE(AO), %xmm0 movaps -28 * SIZE(AO), %xmm1 @@ -2167,10 +2167,10 @@ addq $8 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addps %xmm9, %xmm8 addps %xmm13, %xmm12 @@ -2209,12 +2209,12 @@ addq $8 * SIZE, CO1 subq $1, I - jg .L110 + jg L(110) ALIGN_4 -.L120: +L(120): testq $4, M - jle .L130 + jle L(130) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2250,10 +2250,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L125 + jle L(125) ALIGN_4 -.L121: +L(121): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -2281,10 +2281,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L121 + jg L(121) ALIGN_4 -.L125: +L(125): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2293,10 +2293,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): movaps -32 * SIZE(AO), %xmm0 movaps -32 * SIZE(BO), %xmm2 @@ -2306,10 +2306,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): addps %xmm10, %xmm8 addps %xmm11, %xmm9 @@ -2345,9 +2345,9 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L130: +L(130): testq $2, M - jle .L140 + jle L(140) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2383,10 +2383,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L135 + jle L(135) ALIGN_4 -.L131: +L(131): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -32 * SIZE(AO), %xmm0 @@ -2412,10 +2412,10 @@ subq $ -8 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L131 + jg L(131) ALIGN_4 -.L135: +L(135): movsd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2424,10 +2424,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): movsd -32 * SIZE(AO), %xmm0 movsd -32 * SIZE(BO), %xmm2 @@ -2437,10 +2437,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 #endif @@ -2472,9 +2472,9 @@ addq $2 * SIZE, CO1 ALIGN_4 -.L140: +L(140): testq $1, M - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2510,10 +2510,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L145 + jle L(145) ALIGN_4 -.L141: +L(141): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movss -32 * SIZE(AO), %xmm0 @@ -2539,10 +2539,10 @@ subq $ -4 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L141 + jg L(141) ALIGN_4 -.L145: +L(145): movss ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2551,10 +2551,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): movss -32 * SIZE(AO), %xmm0 movss -32 * SIZE(BO), %xmm2 @@ -2564,10 +2564,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): #ifndef TRMMKERNEL movss 0 * SIZE(CO1), %xmm0 #endif @@ -2584,7 +2584,7 @@ movss %xmm8, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/gemm_kernel_8x4_penryn.S b/kernel/x86_64/gemm_kernel_8x4_penryn.S index b381de979a..9a2cbeaab5 100644 --- a/kernel/x86_64/gemm_kernel_8x4_penryn.S +++ b/kernel/x86_64/gemm_kernel_8x4_penryn.S @@ -164,10 +164,10 @@ movq N, J sarq $2, J NOBRANCH - jle .L50 + jle L(50) ALIGN_4 -.L10: +L(10): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -184,10 +184,10 @@ movq M, I sarq $3, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -247,10 +247,10 @@ #endif sarq $3, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH -32 * SIZE(PREA) addps %xmm6, %xmm10 addps %xmm3, %xmm14 @@ -501,10 +501,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): prefetcht0 -16 * SIZE(BB) #ifndef TRMMKERNEL @@ -514,10 +514,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addps %xmm6, %xmm10 addps %xmm3, %xmm14 movaps %xmm2, %xmm3 @@ -553,10 +553,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): addps %xmm6, %xmm10 addps %xmm3, %xmm14 addps %xmm4, %xmm11 @@ -664,13 +664,13 @@ addq $8 * SIZE, CO2 decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M BRANCH - jle .L30 + jle L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -716,10 +716,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): addps %xmm6, %xmm10 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x39, %xmm2, %xmm7 @@ -786,10 +786,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -797,10 +797,10 @@ #endif andq $3, %rax BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): addps %xmm6, %xmm10 pshufd $0x39, %xmm2, %xmm7 mulps %xmm0, %xmm2 @@ -821,10 +821,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): addps %xmm6, %xmm10 addps %xmm4, %xmm11 @@ -892,10 +892,10 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L30: +L(30): testq $2, M BRANCH - jle .L40 + jle L(40) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -940,10 +940,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm0, %xmm1 @@ -989,10 +989,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1000,10 +1000,10 @@ #endif andq $3, %rax BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): pshufd $0x44, %xmm0, %xmm1 movsd -30 * SIZE(AO), %xmm0 addps %xmm3, %xmm8 @@ -1019,10 +1019,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): movddup ALPHA, %xmm2 addps %xmm10, %xmm8 @@ -1066,10 +1066,10 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L40: +L(40): testq $1, M BRANCH - jle .L49 + jle L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1107,10 +1107,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x00, %xmm0, %xmm1 @@ -1142,10 +1142,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1153,10 +1153,10 @@ #endif andq $3, %rax BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 mulps %xmm1, %xmm2 @@ -1168,10 +1168,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): movddup ALPHA, %xmm2 addps %xmm9, %xmm8 mulps %xmm2, %xmm8 @@ -1207,7 +1207,7 @@ #endif ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif @@ -1218,12 +1218,12 @@ subq $1, J BRANCH - jg .L10 + jg L(10) ALIGN_4 -.L50: +L(50): testq $2, N - jle .L90 + jle L(90) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1241,10 +1241,10 @@ movq M, I sarq $3, I NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1296,10 +1296,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm3, %xmm8 @@ -1375,10 +1375,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1386,10 +1386,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addps %xmm3, %xmm8 pshufd $0x00, %xmm2, %xmm3 mulps %xmm0, %xmm3 @@ -1412,10 +1412,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): movddup ALPHA, %xmm7 addps %xmm3, %xmm8 @@ -1470,13 +1470,13 @@ addq $8 * SIZE, CO2 decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $4, M BRANCH - jle .L70 + jle L(70) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1519,10 +1519,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm3, %xmm8 @@ -1564,10 +1564,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1575,10 +1575,10 @@ #endif andq $3, %rax BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): addps %xmm3, %xmm8 pshufd $0x00, %xmm2, %xmm3 mulps %xmm0, %xmm3 @@ -1593,10 +1593,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): movddup ALPHA, %xmm7 addps %xmm10, %xmm8 @@ -1640,10 +1640,10 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L70: +L(70): testq $2, M BRANCH - jle .L80 + jle L(80) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1682,10 +1682,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm3, %xmm8 @@ -1717,10 +1717,10 @@ subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_3 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -1728,10 +1728,10 @@ #endif andq $3, %rax BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addps %xmm3, %xmm8 pshufd $0x44, %xmm0, %xmm1 movsd -30 * SIZE(AO), %xmm0 @@ -1744,10 +1744,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_3 -.L78: +L(78): movddup ALPHA, %xmm2 addps %xmm9, %xmm8 @@ -1782,10 +1782,10 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L80: +L(80): testq $1, M BRANCH - jle .L89 + jle L(89) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1823,10 +1823,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x00, %xmm0, %xmm1 @@ -1858,10 +1858,10 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) ALIGN_3 -.L85: +L(85): #ifndef TRMMKERNEL movq K, %rax #else @@ -1869,10 +1869,10 @@ #endif andq $3, %rax BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 mulps %xmm1, %xmm2 @@ -1884,10 +1884,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_3 -.L88: +L(88): movddup ALPHA, %xmm2 addps %xmm9, %xmm8 mulps %xmm2, %xmm8 @@ -1917,7 +1917,7 @@ #endif ALIGN_4 -.L89: +L(89): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -1927,9 +1927,9 @@ leaq (C, LDC, 2), C ALIGN_4 -.L90: +L(90): testq $1, N - jle .L999 + jle L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1942,10 +1942,10 @@ movq M, I sarq $3, I NOBRANCH - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1986,10 +1986,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_3 -.L92: +L(92): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x00, %xmm2, %xmm3 @@ -2033,10 +2033,10 @@ subq $1, %rax BRANCH - jg .L92 + jg L(92) ALIGN_3 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2044,10 +2044,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_3 -.L96: +L(96): pshufd $0x00, %xmm2, %xmm3 movss -31 * SIZE(BO), %xmm2 mulps %xmm3, %xmm0 @@ -2062,10 +2062,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_3 -.L98: +L(98): movddup ALPHA, %xmm7 addps %xmm10, %xmm8 @@ -2105,13 +2105,13 @@ addq $8 * SIZE, CO1 decq I BRANCH - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $4, M BRANCH - jle .L110 + jle L(110) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2149,10 +2149,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_3 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x00, %xmm2, %xmm3 @@ -2182,10 +2182,10 @@ subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_3 -.L105: +L(105): #ifndef TRMMKERNEL movq K, %rax #else @@ -2193,10 +2193,10 @@ #endif andq $3, %rax BRANCH - je .L108 + je L(108) ALIGN_3 -.L106: +L(106): pshufd $0x00, %xmm2, %xmm3 movss -31 * SIZE(BO), %xmm2 mulps %xmm0, %xmm3 @@ -2208,10 +2208,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_3 -.L108: +L(108): movddup ALPHA, %xmm7 addps %xmm9, %xmm8 @@ -2243,10 +2243,10 @@ addq $4 * SIZE, CO1 ALIGN_4 -.L110: +L(110): testq $2, M BRANCH - jle .L120 + jle L(120) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2285,10 +2285,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_3 -.L112: +L(112): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x00, %xmm2, %xmm3 @@ -2318,10 +2318,10 @@ subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_3 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2329,10 +2329,10 @@ #endif andq $3, %rax BRANCH - je .L118 + je L(118) ALIGN_3 -.L116: +L(116): pshufd $0x00, %xmm2, %xmm3 movss -31 * SIZE(BO), %xmm2 mulps %xmm0, %xmm3 @@ -2344,10 +2344,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_3 -.L118: +L(118): movddup ALPHA, %xmm2 mulps %xmm2, %xmm8 @@ -2376,10 +2376,10 @@ addq $2 * SIZE, CO1 ALIGN_4 -.L120: +L(120): testq $1, M BRANCH - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2417,10 +2417,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L125 + jle L(125) ALIGN_3 -.L122: +L(122): mulss %xmm0, %xmm2 movss -31 * SIZE(AO), %xmm0 addss %xmm2, %xmm8 @@ -2446,10 +2446,10 @@ subq $1, %rax BRANCH - jg .L122 + jg L(122) ALIGN_3 -.L125: +L(125): #ifndef TRMMKERNEL movq K, %rax #else @@ -2457,10 +2457,10 @@ #endif andq $3, %rax BRANCH - je .L128 + je L(128) ALIGN_3 -.L126: +L(126): mulss %xmm0, %xmm2 movss -31 * SIZE(AO), %xmm0 addss %xmm2, %xmm8 @@ -2471,10 +2471,10 @@ subq $1, %rax BRANCH - jg .L126 + jg L(126) ALIGN_3 -.L128: +L(128): movss ALPHA, %xmm2 addss %xmm9, %xmm8 mulss %xmm2, %xmm8 @@ -2486,7 +2486,7 @@ movss %xmm8, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/gemm_kernel_8x4_sse.S b/kernel/x86_64/gemm_kernel_8x4_sse.S index 1602c13c50..a93f597b63 100644 --- a/kernel/x86_64/gemm_kernel_8x4_sse.S +++ b/kernel/x86_64/gemm_kernel_8x4_sse.S @@ -408,9 +408,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L50 + jle L(50) -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -423,12 +423,12 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): PREFETCH (RPREFETCHSIZE + 0) * SIZE(B) movd 1 * SIZE(B), %mm1 @@ -475,17 +475,17 @@ addq $32 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movd 0 * SIZE(B), %mm0 movd 1 * SIZE(B), %mm1 movd 2 * SIZE(B), %mm2 @@ -508,10 +508,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a @@ -520,10 +520,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -583,10 +583,10 @@ leaq (BO, %rax, 8), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -607,7 +607,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -628,7 +628,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -649,7 +649,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -670,7 +670,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -691,7 +691,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -712,7 +712,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -733,7 +733,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -754,10 +754,10 @@ addq $16 * SIZE, %rax BRANCH - jl .L12 + jl L(12) ALIGN_3 -.L15: +L(15): PREFETCH -16 * SIZE(BB) subq $-16 * SIZE, BB @@ -767,7 +767,7 @@ movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_3 @@ -786,10 +786,10 @@ #else sarq $2, %rax NOBRANCH - jle .L16 + jle L(16) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -803,10 +803,10 @@ subq $-32 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) #endif -.L16: +L(16): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -815,7 +815,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L18 + je L(18) leaq (, %rax, 8), %rax leaq (AO, %rax, 4), AO @@ -823,7 +823,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulps %xmm0, %xmm1 addps %xmm1, %xmm8 movaps -28 * SIZE(BO, %rax, 8), %xmm1 @@ -850,10 +850,10 @@ movaps -20 * SIZE(AO, %rax, 4), %xmm2 addq $SIZE * 2, %rax - jl .L17 + jl L(17) ALIGN_4 -.L18: +L(18): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -936,12 +936,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -985,10 +985,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) @@ -1102,10 +1102,10 @@ addq $ 32 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1114,10 +1114,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1134,10 +1134,10 @@ addq $ 4 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): mulps %xmm15, %xmm0 mulps %xmm15, %xmm1 mulps %xmm15, %xmm2 @@ -1187,9 +1187,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $2, M - je .L40 + je L(40) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1233,10 +1233,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) @@ -1355,10 +1355,10 @@ addq $ 16 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1367,10 +1367,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1388,10 +1388,10 @@ addq $ 2 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): mulps %xmm15, %xmm0 mulps %xmm15, %xmm1 mulps %xmm15, %xmm2 @@ -1443,9 +1443,9 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L40: +L(40): testq $1, M - je .L49 + je L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1490,10 +1490,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): mulss %xmm8, %xmm9 addss %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) @@ -1612,10 +1612,10 @@ addq $ 8 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1624,10 +1624,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -1645,10 +1645,10 @@ addq $ 1 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): mulss %xmm15, %xmm0 mulss %xmm15, %xmm1 mulss %xmm15, %xmm2 @@ -1686,19 +1686,19 @@ #endif ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C # c += 4 * ldc decq J # j -- - jg .L01 + jg L(01) -.L50: +L(50): testq $2, N - je .L100 + je L(100) -.L51: +L(51): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1709,10 +1709,10 @@ movq K, %rax sarq $2, %rax - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): #if defined(PENTIUM4) || defined(GENERIC) movss 0 * SIZE(B), %xmm0 movss 1 * SIZE(B), %xmm1 @@ -1790,17 +1790,17 @@ #endif decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): movq K, %rax andq $3, %rax BRANCH - jle .L60 + jle L(60) ALIGN_4 -.L54: +L(54): #if defined(PENTIUM4) || defined(GENERIC) movss 0 * SIZE(B), %xmm0 movss 1 * SIZE(B), %xmm1 @@ -1828,20 +1828,20 @@ addq $ 2 * SIZE, B addq $ 8 * SIZE, BO decq %rax - jne .L54 + jne L(54) ALIGN_4 -.L60: +L(60): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L70 + jle L(70) ALIGN_4 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1889,10 +1889,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2012,10 +2012,10 @@ addq $64 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -2024,10 +2024,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -2044,10 +2044,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2098,12 +2098,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L61 + jg L(61) ALIGN_4 -.L70: +L(70): testq $4, M - je .L80 + je L(80) #if !defined(TRMMKERNEL) || \ @@ -2148,10 +2148,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2215,10 +2215,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2227,10 +2227,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -2241,10 +2241,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2285,9 +2285,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L80: +L(80): testq $2, M - je .L90 + je L(90) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2331,10 +2331,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L85 + je L(85) ALIGN_4 -.L82: +L(82): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) @@ -2405,10 +2405,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L85: +L(85): #ifndef TRMMKERNEL movq K, %rax #else @@ -2417,10 +2417,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -2432,10 +2432,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): #ifndef TRMMKERNEL #ifdef movsd xorps %xmm8, %xmm8 @@ -2478,9 +2478,9 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L90: +L(90): testq $1, M - je .L99 + je L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2524,10 +2524,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) @@ -2598,10 +2598,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2610,10 +2610,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -2625,10 +2625,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): #ifndef TRMMKERNEL movss 0 * SIZE(CO1), %xmm8 movss 0 * SIZE(CO2), %xmm10 @@ -2661,7 +2661,7 @@ #endif ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -2669,11 +2669,11 @@ ALIGN_4 -.L100: +L(100): testq $1, N - je .L999 + je L(999) -.L101: +L(101): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -2684,11 +2684,11 @@ movq K, %rax sarq $3, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): #if defined(PENTIUM4) || defined(GENERIC) movss 0 * SIZE(B), %xmm0 movss 1 * SIZE(B), %xmm1 @@ -2766,17 +2766,17 @@ #endif decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): movq K, %rax andq $7, %rax BRANCH - jle .L110 + jle L(110) ALIGN_4 -.L104: +L(104): #if defined(PENTIUM4) || defined(GENERIC) movss 0 * SIZE(B), %xmm0 shufps $0, %xmm0, %xmm0 @@ -2793,19 +2793,19 @@ addq $ 1 * SIZE, B addq $ 4 * SIZE, BO decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L110: +L(110): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L120 + jle L(120) ALIGN_4 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2852,10 +2852,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulps %xmm9, %xmm8 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2928,10 +2928,10 @@ addq $64 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2940,10 +2940,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulps %xmm9, %xmm8 mulps -28 * SIZE(AO), %xmm9 addps %xmm8, %xmm0 @@ -2954,10 +2954,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2992,12 +2992,12 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L111 + jg L(111) ALIGN_4 -.L120: +L(120): testq $4, M - je .L130 + je L(130) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3039,10 +3039,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L125 + je L(125) ALIGN_4 -.L122: +L(122): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3080,10 +3080,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L122 + jne L(122) ALIGN_4 -.L125: +L(125): #ifndef TRMMKERNEL movq K, %rax #else @@ -3092,10 +3092,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): mulps %xmm8, %xmm9 movaps -28 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -3104,10 +3104,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -3141,9 +3141,9 @@ addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L130: +L(130): testq $2, M - je .L140 + je L(140) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3185,10 +3185,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L135 + je L(135) ALIGN_4 -.L132: +L(132): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3234,10 +3234,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L132 + jne L(132) ALIGN_4 -.L135: +L(135): #ifndef TRMMKERNEL movq K, %rax #else @@ -3246,10 +3246,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): mulps %xmm8, %xmm9 movsd -30 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -3258,10 +3258,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): addps %xmm1, %xmm0 mulps %xmm15, %xmm0 @@ -3291,9 +3291,9 @@ addq $2 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L140: +L(140): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3335,10 +3335,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L145 + je L(145) ALIGN_4 -.L142: +L(142): mulss %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3372,10 +3372,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L142 + jne L(142) ALIGN_4 -.L145: +L(145): #ifndef TRMMKERNEL movq K, %rax #else @@ -3384,10 +3384,10 @@ movss ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): mulss %xmm8, %xmm9 movss -31 * SIZE(AO), %xmm8 addss %xmm9, %xmm0 @@ -3396,10 +3396,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): addss %xmm1, %xmm0 addss %xmm3, %xmm2 addss %xmm2, %xmm0 @@ -3413,7 +3413,7 @@ movss %xmm0, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp EMMS diff --git a/kernel/x86_64/gemm_kernel_8x4_sse3.S b/kernel/x86_64/gemm_kernel_8x4_sse3.S index c853e46d18..324836e075 100644 --- a/kernel/x86_64/gemm_kernel_8x4_sse3.S +++ b/kernel/x86_64/gemm_kernel_8x4_sse3.S @@ -394,9 +394,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L50 + jle L(50) -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -407,10 +407,10 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): movddup 0 * SIZE(B), %xmm0 movddup 2 * SIZE(B), %xmm1 movddup 4 * SIZE(B), %xmm2 @@ -436,17 +436,17 @@ addq $32 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movddup 0 * SIZE(B), %xmm0 movddup 2 * SIZE(B), %xmm1 @@ -456,10 +456,10 @@ addq $4 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a @@ -468,10 +468,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): prefetcht0 0 * SIZE(BB) prefetcht0 8 * SIZE(BB) subq $-16 * SIZE, BB @@ -532,9 +532,9 @@ #if 1 andq $-8, %rax salq $4, %rax - je .L15 + je L(15) -.L1X: +L(1X): KERNEL1 (64 * 0) KERNEL2 (64 * 0) KERNEL3 (64 * 0) @@ -553,7 +553,7 @@ KERNEL16(64 * 0) cmpq $128 * 1, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 1) KERNEL2 (64 * 1) KERNEL3 (64 * 1) @@ -572,7 +572,7 @@ KERNEL16(64 * 1) cmpq $128 * 2, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 2) KERNEL2 (64 * 2) KERNEL3 (64 * 2) @@ -591,7 +591,7 @@ KERNEL16(64 * 2) cmpq $128 * 3, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 3) KERNEL2 (64 * 3) KERNEL3 (64 * 3) @@ -610,7 +610,7 @@ KERNEL16(64 * 3) cmpq $128 * 4, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 4) KERNEL2 (64 * 4) KERNEL3 (64 * 4) @@ -629,7 +629,7 @@ KERNEL16(64 * 4) cmpq $128 * 5, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 5) KERNEL2 (64 * 5) KERNEL3 (64 * 5) @@ -648,7 +648,7 @@ KERNEL16(64 * 5) cmpq $128 * 6, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 6) KERNEL2 (64 * 6) KERNEL3 (64 * 6) @@ -667,7 +667,7 @@ KERNEL16(64 * 6) cmpq $128 * 7, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 7) KERNEL2 (64 * 7) KERNEL3 (64 * 7) @@ -688,17 +688,17 @@ addq $64 * 8 * SIZE, AO addq $64 * 8 * SIZE, BO subq $128 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO leaq (BO, %rax, 2), BO #else sarq $3, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): KERNEL1 (64 * 0) KERNEL2 (64 * 0) KERNEL3 (64 * 0) @@ -719,11 +719,11 @@ addq $64 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L12 + jne L(12) #endif ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else @@ -732,10 +732,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -766,10 +766,10 @@ addq $8 * SIZE, AO addq $8 * SIZE, BO decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 mulps %xmm15, %xmm0 @@ -859,12 +859,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -908,10 +908,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm9, %xmm0 @@ -1021,10 +1021,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1033,10 +1033,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -1054,10 +1054,10 @@ addq $4 * SIZE, AO addq $8 * SIZE, BO decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -1112,9 +1112,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $2, M - je .L40 + je L(40) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1155,10 +1155,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): shufps $0x50, %xmm9, %xmm9 mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1236,10 +1236,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1248,10 +1248,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): shufps $0x50, %xmm9, %xmm9 mulps %xmm8, %xmm9 addps %xmm9, %xmm0 @@ -1265,10 +1265,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhps 0 * SIZE(CO2), %xmm8 @@ -1309,9 +1309,9 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L40: +L(40): testq $1, M - je .L49 + je L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1350,10 +1350,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): shufps $0, %xmm8, %xmm8 movhps 4 * SIZE(BO), %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1407,10 +1407,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1419,10 +1419,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): shufps $0, %xmm8, %xmm8 movhps 4 * SIZE(BO), %xmm9 mulps %xmm8, %xmm9 @@ -1433,10 +1433,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movss 0 * SIZE(CO1), %xmm8 movss 0 * SIZE(CO2), %xmm9 @@ -1485,19 +1485,19 @@ #endif ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C # c += 4 * ldc decq J # j -- - jg .L01 + jg L(01) -.L50: +L(50): testq $2, N - je .L100 + je L(100) -.L51: +L(51): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1508,10 +1508,10 @@ movq K, %rax sarq $3, %rax - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): movddup 0 * SIZE(B), %xmm0 movddup 2 * SIZE(B), %xmm1 movddup 4 * SIZE(B), %xmm2 @@ -1537,37 +1537,37 @@ addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): movq K, %rax andq $7, %rax BRANCH - jle .L60 + jle L(60) ALIGN_4 -.L54: +L(54): movddup 0 * SIZE(B), %xmm0 movaps %xmm0, 0 * SIZE(BO) addq $ 2 * SIZE, B addq $ 4 * SIZE, BO decq %rax - jne .L54 + jne L(54) ALIGN_4 -.L60: +L(60): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L70 + jle L(70) ALIGN_4 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1612,10 +1612,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm9, %xmm0 @@ -1734,10 +1734,10 @@ addq $64 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1746,10 +1746,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -1768,10 +1768,10 @@ addq $8 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -1822,12 +1822,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L61 + jg L(61) ALIGN_4 -.L70: +L(70): testq $4, M - je .L80 + je L(80) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1868,10 +1868,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm9, %xmm0 @@ -1933,10 +1933,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -1945,10 +1945,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -1960,10 +1960,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2004,9 +2004,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L80: +L(80): testq $2, M - je .L90 + je L(90) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2045,10 +2045,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L85 + je L(85) ALIGN_4 -.L82: +L(82): shufps $0x50, %xmm9, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulps %xmm8, %xmm9 @@ -2094,10 +2094,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L85: +L(85): #ifndef TRMMKERNEL movq K, %rax #else @@ -2106,10 +2106,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): shufps $0x50, %xmm9, %xmm9 mulps %xmm8, %xmm9 movddup 2 * SIZE(AO), %xmm8 @@ -2119,10 +2119,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhps 0 * SIZE(CO2), %xmm8 @@ -2154,9 +2154,9 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L90: +L(90): testq $1, M - je .L99 + je L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2195,10 +2195,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): shufps $0, %xmm8, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulps %xmm8, %xmm9 @@ -2244,10 +2244,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2256,10 +2256,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): shufps $0, %xmm8, %xmm8 mulps %xmm8, %xmm9 movss 1 * SIZE(AO), %xmm8 @@ -2269,10 +2269,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movss 0 * SIZE(CO1), %xmm8 movss 0 * SIZE(CO2), %xmm9 @@ -2308,18 +2308,18 @@ #endif ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif leaq (C, LDC, 2), C # c += 4 * ldc ALIGN_4 -.L100: +L(100): testq $1, N - je .L999 + je L(999) -.L101: +L(101): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -2330,11 +2330,11 @@ movq K, %rax sarq $3, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movss 0 * SIZE(B), %xmm0 movss 1 * SIZE(B), %xmm1 movss 2 * SIZE(B), %xmm2 @@ -2365,17 +2365,17 @@ addq $16 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): movq K, %rax andq $7, %rax BRANCH - jle .L110 + jle L(110) ALIGN_4 -.L104: +L(104): movss 0 * SIZE(B), %xmm0 movss %xmm0, 0 * SIZE(BO) movss %xmm0, 1 * SIZE(BO) @@ -2383,19 +2383,19 @@ addq $ 1 * SIZE, B addq $ 2 * SIZE, BO decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L110: +L(110): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L120 + jle L(120) ALIGN_4 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2439,10 +2439,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulps %xmm8, %xmm9 movaps 4 * SIZE(AO), %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2513,10 +2513,10 @@ addq $64 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2525,10 +2525,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulps %xmm8, %xmm9 movaps 4 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -2541,10 +2541,10 @@ addq $8 * SIZE, AO addq $2 * SIZE, BO decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2582,12 +2582,12 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L111 + jg L(111) ALIGN_4 -.L120: +L(120): testq $4, M - je .L130 + je L(130) #if !defined(TRMMKERNEL) || \ @@ -2633,10 +2633,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L125 + je L(125) ALIGN_4 -.L122: +L(122): mulps %xmm8, %xmm9 movaps 4 * SIZE(AO), %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2674,10 +2674,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L122 + jne L(122) ALIGN_4 -.L125: +L(125): #ifndef TRMMKERNEL movq K, %rax #else @@ -2686,10 +2686,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): mulps %xmm8, %xmm9 movaps 4 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -2698,10 +2698,10 @@ addq $4 * SIZE, AO addq $2 * SIZE, BO decq %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2733,9 +2733,9 @@ addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L130: +L(130): testq $2, M - je .L140 + je L(140) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2776,10 +2776,10 @@ movq %rax, KKK #endif sarq $4, %rax - je .L135 + je L(135) ALIGN_4 -.L132: +L(132): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps 4 * SIZE(AO), %xmm8 @@ -2811,10 +2811,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L132 + jne L(132) ALIGN_4 -.L135: +L(135): #ifndef TRMMKERNEL movq K, %rax #else @@ -2823,10 +2823,10 @@ movaps ALPHA, %xmm15 andq $15, %rax # if (k & 1) BRANCH - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): movsd 0 * SIZE(AO), %xmm8 movsd 0 * SIZE(BO), %xmm9 @@ -2836,10 +2836,10 @@ addq $2 * SIZE, AO addq $2 * SIZE, BO decq %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movsd 0 * SIZE(CO1), %xmm8 #endif @@ -2872,9 +2872,9 @@ addq $2 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L140: +L(140): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2915,10 +2915,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L145 + je L(145) ALIGN_4 -.L142: +L(142): mulss %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movss 1 * SIZE(AO), %xmm8 @@ -2950,10 +2950,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L142 + jne L(142) ALIGN_4 -.L145: +L(145): #ifndef TRMMKERNEL movq K, %rax #else @@ -2962,10 +2962,10 @@ movss ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): movss 0 * SIZE(AO), %xmm8 movss 0 * SIZE(BO), %xmm9 mulps %xmm8, %xmm9 @@ -2974,10 +2974,10 @@ addq $1 * SIZE, AO addq $2 * SIZE, BO decq %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): #if! defined(TRMMKERNEL) && !defined(BETAZERO) movss 0 * SIZE(CO1), %xmm8 #endif @@ -2992,7 +2992,7 @@ movss %xmm0, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/gemm_ncopy_2.S b/kernel/x86_64/gemm_ncopy_2.S index b069f9cf7f..d132a9b65d 100644 --- a/kernel/x86_64/gemm_ncopy_2.S +++ b/kernel/x86_64/gemm_ncopy_2.S @@ -115,20 +115,20 @@ movq N, J sarq $1, J - jle .L20 + jle L(20) ALIGN_4 -.L12: +L(12): movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), A movq M, I sarq $2, I - jle .L14 + jle L(14) ALIGN_4 -.L13: +L(13): #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 movss 0 * SIZE(AO2), %xmm1 @@ -174,16 +174,16 @@ addq $4 * SIZE, AO2 subq $-8 * SIZE, B decq I - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): movq M, I andq $3, I - jle .L16 + jle L(16) ALIGN_4 -.L15: +L(15): #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 movss 0 * SIZE(AO2), %xmm1 @@ -200,26 +200,26 @@ addq $SIZE, AO2 addq $2 * SIZE, B decq I - jg .L15 + jg L(15) ALIGN_4 -.L16: +L(16): decq J - jg .L12 + jg L(12) ALIGN_4 -.L20: +L(20): testq $1, N - jle .L999 + jle L(999) movq A, AO1 movq M, I sarq $2, I - jle .L34 + jle L(34) ALIGN_4 -.L33: +L(33): #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 movss 1 * SIZE(AO1), %xmm1 @@ -244,16 +244,16 @@ addq $4 * SIZE, AO1 subq $-4 * SIZE, B decq I - jg .L33 + jg L(33) ALIGN_4 -.L34: +L(34): movq M, I andq $3, I - jle .L999 + jle L(999) ALIGN_4 -.L35: +L(35): #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 movss %xmm0, 0 * SIZE(B) @@ -265,11 +265,11 @@ addq $SIZE, AO1 addq $1 * SIZE, B decq I - jg .L35 + jg L(35) ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 movups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/gemm_ncopy_2_bulldozer.S b/kernel/x86_64/gemm_ncopy_2_bulldozer.S index 1911d3c748..477e76517c 100644 --- a/kernel/x86_64/gemm_ncopy_2_bulldozer.S +++ b/kernel/x86_64/gemm_ncopy_2_bulldozer.S @@ -102,20 +102,20 @@ movq N, J sarq $1, J - jle .L20 + jle L(20) ALIGN_4 -.L01: +L(01): movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), A movq M, I sarq $3, I - jle .L08 + jle L(08) ALIGN_4 -.L03: +L(03): #ifndef DOUBLE vmovss 0 * SIZE(AO1), %xmm0 @@ -194,18 +194,18 @@ addq $8 * SIZE, AO2 subq $-16 * SIZE, B decq I - jg .L03 + jg L(03) ALIGN_4 -.L08: +L(08): testq $4 , M - je .L14 + je L(14) ALIGN_4 -.L13: +L(13): #ifndef DOUBLE vmovss 0 * SIZE(AO1), %xmm0 vmovss 0 * SIZE(AO2), %xmm1 @@ -248,13 +248,13 @@ subq $-8 * SIZE, B ALIGN_4 -.L14: +L(14): movq M, I andq $3, I - jle .L16 + jle L(16) ALIGN_4 -.L15: +L(15): #ifndef DOUBLE vmovss 0 * SIZE(AO1), %xmm0 vmovss 0 * SIZE(AO2), %xmm1 @@ -272,26 +272,26 @@ addq $SIZE, AO2 addq $2 * SIZE, B decq I - jg .L15 + jg L(15) ALIGN_4 -.L16: +L(16): decq J - jg .L01 + jg L(01) ALIGN_4 -.L20: +L(20): testq $1, N - jle .L999 + jle L(999) movq A, AO1 movq M, I sarq $2, I - jle .L34 + jle L(34) ALIGN_4 -.L33: +L(33): #ifndef DOUBLE vmovups 0 * SIZE(AO1), %xmm0 @@ -307,16 +307,16 @@ addq $4 * SIZE, AO1 subq $-4 * SIZE, B decq I - jg .L33 + jg L(33) ALIGN_4 -.L34: +L(34): movq M, I andq $3, I - jle .L999 + jle L(999) ALIGN_4 -.L35: +L(35): #ifndef DOUBLE vmovss 0 * SIZE(AO1), %xmm0 vmovss %xmm0, 0 * SIZE(B) @@ -328,11 +328,11 @@ addq $SIZE, AO1 addq $1 * SIZE, B decq I - jg .L35 + jg L(35) ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI vmovups 0(%rsp), %xmm6 vmovups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/gemm_ncopy_4.S b/kernel/x86_64/gemm_ncopy_4.S index d30e9d362f..f9b2ef9074 100644 --- a/kernel/x86_64/gemm_ncopy_4.S +++ b/kernel/x86_64/gemm_ncopy_4.S @@ -143,10 +143,10 @@ movq N, J sarq $2, J - jle .L20 + jle L(20) ALIGN_4 -.L12: +L(12): movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), AO3 @@ -155,10 +155,10 @@ movq M, I sarq $2, I - jle .L14 + jle L(14) ALIGN_4 -.L13: +L(13): #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 movss 0 * SIZE(AO2), %xmm1 @@ -262,16 +262,16 @@ subq $-16 * SIZE, B decq I - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): movq M, I andq $3, I - jle .L16 + jle L(16) ALIGN_4 -.L15: +L(15): #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 movss 0 * SIZE(AO2), %xmm1 @@ -298,17 +298,17 @@ addq $SIZE, AO4 addq $4 * SIZE, B decq I - jg .L15 + jg L(15) ALIGN_4 -.L16: +L(16): decq J - jg .L12 + jg L(12) ALIGN_4 -.L20: +L(20): testq $2, N - jle .L30 + jle L(30) movq A, AO1 leaq (A, LDA), AO2 @@ -316,10 +316,10 @@ movq M, I sarq $2, I - jle .L24 + jle L(24) ALIGN_4 -.L23: +L(23): #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 movss 0 * SIZE(AO2), %xmm1 @@ -369,16 +369,16 @@ addq $4 * SIZE, AO2 subq $-8 * SIZE, B decq I - jg .L23 + jg L(23) ALIGN_4 -.L24: +L(24): movq M, I andq $3, I - jle .L30 + jle L(30) ALIGN_4 -.L25: +L(25): #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 movss 0 * SIZE(AO2), %xmm1 @@ -396,21 +396,21 @@ addq $SIZE, AO2 addq $2 * SIZE, B decq I - jg .L25 + jg L(25) ALIGN_4 -.L30: +L(30): testq $1, N - jle .L999 + jle L(999) movq A, AO1 movq M, I sarq $2, I - jle .L34 + jle L(34) ALIGN_4 -.L33: +L(33): #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 movss 1 * SIZE(AO1), %xmm1 @@ -435,16 +435,16 @@ addq $4 * SIZE, AO1 subq $-4 * SIZE, B decq I - jg .L33 + jg L(33) ALIGN_4 -.L34: +L(34): movq M, I andq $3, I - jle .L999 + jle L(999) ALIGN_4 -.L35: +L(35): #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 movss %xmm0, 0 * SIZE(B) @@ -456,11 +456,11 @@ addq $SIZE, AO1 addq $1 * SIZE, B decq I - jg .L35 + jg L(35) ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 movups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/gemm_ncopy_4_opteron.S b/kernel/x86_64/gemm_ncopy_4_opteron.S index ea39f89367..21b8b233b9 100644 --- a/kernel/x86_64/gemm_ncopy_4_opteron.S +++ b/kernel/x86_64/gemm_ncopy_4_opteron.S @@ -118,10 +118,10 @@ movq N, J sarq $2, J - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if 0 movq A, AO1 leaq (A, LDA, 1), AO2 @@ -130,10 +130,10 @@ movq M, I sarq $4, I - jle .L13 + jle L(13) ALIGN_4 -.L12: +L(12): MOVQ 0 * SIZE(AO1), %mm0 addq $8 * SIZE, AO1 MOVQ 0 * SIZE(AO2), %mm1 @@ -144,10 +144,10 @@ addq $8 * SIZE, AO4 decq I - jg .L12 + jg L(12) ALIGN_4 -.L13: +L(13): #endif movq A, AO1 @@ -158,10 +158,10 @@ movq M, I sarq $2, I - jle .L15 + jle L(15) ALIGN_4 -.L14: +L(14): RPREFETCH (RPREFETCHSIZE) * SIZE(AO1) MOVQ 0 * SIZE(AO1), %mm0 @@ -217,16 +217,16 @@ subq $-16 * SIZE, B decq I - jg .L14 + jg L(14) ALIGN_4 -.L15: +L(15): movq M, I andq $3, I - jle .L17 + jle L(17) ALIGN_4 -.L16: +L(16): MOVQ 0 * SIZE(AO1), %mm0 MOVQ 0 * SIZE(AO2), %mm1 MOVQ 0 * SIZE(AO3), %mm2 @@ -243,17 +243,17 @@ addq $SIZE, AO4 addq $4 * SIZE, B decq I - jg .L16 + jg L(16) ALIGN_4 -.L17: +L(17): decq J - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, N - jle .L30 + jle L(30) movq A, AO1 leaq (A, LDA), AO2 @@ -261,10 +261,10 @@ movq M, I sarq $2, I - jle .L24 + jle L(24) ALIGN_4 -.L23: +L(23): prefetch (RPREFETCHSIZE) * SIZE(AO1) MOVQ 0 * SIZE(AO1), %mm0 prefetch (RPREFETCHSIZE) * SIZE(AO2) @@ -291,16 +291,16 @@ addq $4 * SIZE, AO2 subq $-8 * SIZE, B decq I - jg .L23 + jg L(23) ALIGN_4 -.L24: +L(24): movq M, I andq $3, I - jle .L30 + jle L(30) ALIGN_4 -.L25: +L(25): MOVQ 0 * SIZE(AO1), %mm0 MOVQ 0 * SIZE(AO2), %mm1 @@ -311,21 +311,21 @@ addq $SIZE, AO2 addq $2 * SIZE, B decq I - jg .L25 + jg L(25) ALIGN_4 -.L30: +L(30): testq $1, N - jle .L999 + jle L(999) movq A, AO1 movq M, I sarq $2, I - jle .L34 + jle L(34) ALIGN_4 -.L33: +L(33): MOVQ 0 * SIZE(AO1), %mm0 MOVQ 1 * SIZE(AO1), %mm1 MOVQ 2 * SIZE(AO1), %mm2 @@ -339,27 +339,27 @@ addq $4 * SIZE, AO1 subq $-4 * SIZE, B decq I - jg .L33 + jg L(33) ALIGN_4 -.L34: +L(34): movq M, I andq $3, I - jle .L999 + jle L(999) ALIGN_4 -.L35: +L(35): MOVQ 0 * SIZE(AO1), %mm0 addq $SIZE, AO1 MOVNTQ %mm0, 0 * SIZE(B) addq $1 * SIZE, B decq I - jg .L35 + jg L(35) ALIGN_4 -.L999: +L(999): EMMS #ifdef WINDOWS_ABI diff --git a/kernel/x86_64/gemm_tcopy_2.S b/kernel/x86_64/gemm_tcopy_2.S index f35427b798..2e970ce424 100644 --- a/kernel/x86_64/gemm_tcopy_2.S +++ b/kernel/x86_64/gemm_tcopy_2.S @@ -140,10 +140,10 @@ movq M, J sarq $1, J - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): movq A, AO1 leaq (A, LDA ), AO2 leaq (A, LDA, 2), A @@ -153,10 +153,10 @@ movq N, I sarq $1, I - jle .L14 + jle L(14) ALIGN_4 -.L12: +L(12): #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 movhps 0 * SIZE(AO2), %xmm0 @@ -179,12 +179,12 @@ addq $2 * SIZE, AO1 addq $2 * SIZE, AO2 decq I - jg .L12 + jg L(12) ALIGN_4 -.L14: +L(14): testq $1, N - jle .L19 + jle L(19) #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 @@ -202,26 +202,26 @@ addq $2 * SIZE, BO1 ALIGN_4 -.L19: +L(19): decq J - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M - jle .L999 + jle L(999) ALIGN_4 -.L31: +L(31): movq A, AO1 movq B, BO movq N, I sarq $1, I - jle .L33 + jle L(33) ALIGN_4 -.L32: +L(32): #ifndef DOUBLE movsd 0 * SIZE(AO1), %xmm0 movsd %xmm0, 0 * SIZE(BO) @@ -234,12 +234,12 @@ addq $2 * SIZE, AO1 leaq (BO, M8, 2), BO decq I - jg .L32 + jg L(32) ALIGN_4 -.L33: +L(33): testq $1, N - jle .L999 + jle L(999) #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 @@ -251,7 +251,7 @@ addq $1 * SIZE, BO1 ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 movups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/gemm_tcopy_2_bulldozer.S b/kernel/x86_64/gemm_tcopy_2_bulldozer.S index d7552042eb..89af090edd 100644 --- a/kernel/x86_64/gemm_tcopy_2_bulldozer.S +++ b/kernel/x86_64/gemm_tcopy_2_bulldozer.S @@ -126,10 +126,10 @@ movq M, J sarq $1, J - jle .L20 + jle L(20) ALIGN_4 -.L01: +L(01): movq A, AO1 leaq (A, LDA ), AO2 leaq (A, LDA, 2), A @@ -139,11 +139,11 @@ movq N, I sarq $3, I - jle .L10 + jle L(10) ALIGN_4 -.L08: +L(08): #ifndef DOUBLE vmovsd 0 * SIZE(AO1), %xmm0 @@ -206,14 +206,14 @@ addq $8 * SIZE, AO1 addq $8 * SIZE, AO2 decq I - jg .L08 + jg L(08) ALIGN_4 -.L10: +L(10): testq $4, N - jle .L12 + jle L(12) #ifndef DOUBLE vmovsd 0 * SIZE(AO1), %xmm0 @@ -252,9 +252,9 @@ ALIGN_4 -.L12: +L(12): testq $2, N - jle .L14 + jle L(14) #ifndef DOUBLE vmovsd 0 * SIZE(AO1), %xmm0 vmovsd 0 * SIZE(AO2), %xmm1 @@ -274,9 +274,9 @@ addq $2 * SIZE, AO2 ALIGN_4 -.L14: +L(14): testq $1, N - jle .L19 + jle L(19) #ifndef DOUBLE vmovss 0 * SIZE(AO1), %xmm0 @@ -294,26 +294,26 @@ addq $2 * SIZE, BO1 ALIGN_4 -.L19: +L(19): decq J - jg .L01 + jg L(01) ALIGN_4 -.L20: +L(20): testq $1, M - jle .L999 + jle L(999) ALIGN_4 -.L31: +L(31): movq A, AO1 movq B, BO movq N, I sarq $1, I - jle .L33 + jle L(33) ALIGN_4 -.L32: +L(32): #ifndef DOUBLE vmovsd 0 * SIZE(AO1), %xmm0 vmovsd %xmm0, 0 * SIZE(BO) @@ -325,12 +325,12 @@ addq $2 * SIZE, AO1 leaq (BO, M8, 2), BO decq I - jg .L32 + jg L(32) ALIGN_4 -.L33: +L(33): testq $1, N - jle .L999 + jle L(999) #ifndef DOUBLE vmovss 0 * SIZE(AO1), %xmm0 @@ -342,7 +342,7 @@ addq $1 * SIZE, BO1 ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI vmovups 0(%rsp), %xmm6 vmovups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/gemm_tcopy_4.S b/kernel/x86_64/gemm_tcopy_4.S index 177587c4b7..551c6e1089 100644 --- a/kernel/x86_64/gemm_tcopy_4.S +++ b/kernel/x86_64/gemm_tcopy_4.S @@ -174,10 +174,10 @@ movq M, J sarq $2, J - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): movq A, AO1 leaq (A, LDA ), AO2 leaq (A, LDA, 2), AO3 @@ -189,10 +189,10 @@ movq N, I sarq $2, I - jle .L13 + jle L(13) ALIGN_4 -.L12: +L(12): #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 movhps 2 * SIZE(AO1), %xmm0 @@ -274,12 +274,12 @@ addq $4 * SIZE, AO3 addq $4 * SIZE, AO4 decq I - jg .L12 + jg L(12) ALIGN_4 -.L13: +L(13): testq $2, N - jle .L14 + jle L(14) #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 @@ -314,9 +314,9 @@ addq $8 * SIZE, BO1 ALIGN_4 -.L14: +L(14): testq $1, N - jle .L19 + jle L(19) #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 @@ -341,17 +341,17 @@ addq $4 * SIZE, BO2 ALIGN_4 -.L19: +L(19): decq J - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M - jle .L30 + jle L(30) ALIGN_4 -.L21: +L(21): movq A, AO1 leaq (A, LDA ), AO2 leaq (A, LDA, 2), A @@ -361,10 +361,10 @@ movq N, I sarq $2, I - jle .L23 + jle L(23) ALIGN_4 -.L22: +L(22): #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 movhps 2 * SIZE(AO1), %xmm0 @@ -407,12 +407,12 @@ addq $4 * SIZE, AO2 leaq (BO, M8, 4), BO decq I - jg .L22 + jg L(22) ALIGN_4 -.L23: +L(23): testq $2, N - jle .L24 + jle L(24) #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 @@ -434,9 +434,9 @@ addq $4 * SIZE, BO1 ALIGN_4 -.L24: +L(24): testq $1, N - jle .L30 + jle L(30) #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 @@ -453,21 +453,21 @@ addq $2 * SIZE, BO2 ALIGN_4 -.L30: +L(30): testq $1, M - jle .L999 + jle L(999) ALIGN_4 -.L31: +L(31): movq A, AO1 movq B, BO movq N, I sarq $2, I - jle .L33 + jle L(33) ALIGN_4 -.L32: +L(32): #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 movhps 2 * SIZE(AO1), %xmm0 @@ -486,12 +486,12 @@ addq $4 * SIZE, AO1 leaq (BO, M8, 4), BO decq I - jg .L32 + jg L(32) ALIGN_4 -.L33: +L(33): testq $2, N - jle .L34 + jle L(34) #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 @@ -508,9 +508,9 @@ addq $2 * SIZE, BO1 ALIGN_4 -.L34: +L(34): testq $1, N - jle .L999 + jle L(999) #ifndef DOUBLE movss 0 * SIZE(AO1), %xmm0 @@ -522,7 +522,7 @@ addq $1 * SIZE, BO2 ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 movups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/gemm_tcopy_4_opteron.S b/kernel/x86_64/gemm_tcopy_4_opteron.S index e8207ace37..05f27f482f 100644 --- a/kernel/x86_64/gemm_tcopy_4_opteron.S +++ b/kernel/x86_64/gemm_tcopy_4_opteron.S @@ -149,10 +149,10 @@ leaq (, M, SIZE), M8 movq M, J sarq $2, J - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if 0 movq A, AO1 leaq (A, LDA, 1), AO2 @@ -161,10 +161,10 @@ movq N, I sarq $3, I - jle .L13 + jle L(13) ALIGN_4 -.L12: +L(12): MOVQ 0 * SIZE(AO1), %mm0 addq $8 * SIZE, AO1 MOVQ 0 * SIZE(AO2), %mm1 @@ -175,10 +175,10 @@ addq $8 * SIZE, AO4 decq I - jg .L12 + jg L(12) ALIGN_4 -.L13: +L(13): #endif movq A, AO1 @@ -192,10 +192,10 @@ movq N, I sarq $2, I - jle .L15 + jle L(15) ALIGN_4 -.L14: +L(14): RPREFETCH (RPREFETCHSIZE) * SIZE(AO1) @@ -253,12 +253,12 @@ leaq (BO, M8, 4), BO decq I - jg .L14 + jg L(14) ALIGN_4 -.L15: +L(15): testq $2, N - jle .L16 + jle L(16) MOVQ 0 * SIZE(AO1), %mm0 MOVQ 1 * SIZE(AO1), %mm1 @@ -286,9 +286,9 @@ addq $8 * SIZE, BO1 ALIGN_4 -.L16: +L(16): testq $1, N - jle .L19 + jle L(19) MOVQ 0 * SIZE(AO1), %mm0 MOVQ 0 * SIZE(AO2), %mm1 @@ -303,17 +303,17 @@ addq $4 * SIZE, BO2 ALIGN_4 -.L19: +L(19): decq J - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M - jle .L30 + jle L(30) ALIGN_4 -.L21: +L(21): movq A, AO1 leaq (A, LDA ), AO2 leaq (A, LDA, 2), A @@ -323,10 +323,10 @@ movq N, I sarq $2, I - jle .L23 + jle L(23) ALIGN_4 -.L22: +L(22): RPREFETCH (RPREFETCHSIZE) * SIZE(AO1) MOVQ 0 * SIZE(AO1), %mm0 MOVQ 1 * SIZE(AO1), %mm1 @@ -353,12 +353,12 @@ addq $4 * SIZE, AO2 leaq (BO, M8, 4), BO decq I - jg .L22 + jg L(22) ALIGN_4 -.L23: +L(23): testq $2, N - jle .L24 + jle L(24) MOVQ 0 * SIZE(AO1), %mm0 MOVQ 1 * SIZE(AO1), %mm1 @@ -375,9 +375,9 @@ addq $4 * SIZE, BO1 ALIGN_4 -.L24: +L(24): testq $1, N - jle .L30 + jle L(30) MOVQ 0 * SIZE(AO1), %mm0 MOVQ 0 * SIZE(AO2), %mm1 @@ -388,21 +388,21 @@ addq $2 * SIZE, BO2 ALIGN_4 -.L30: +L(30): testq $1, M - jle .L999 + jle L(999) ALIGN_4 -.L31: +L(31): movq A, AO1 movq B, BO movq N, I sarq $2, I - jle .L33 + jle L(33) ALIGN_4 -.L32: +L(32): MOVQ 0 * SIZE(AO1), %mm0 MOVQ 1 * SIZE(AO1), %mm1 MOVQ 2 * SIZE(AO1), %mm2 @@ -416,12 +416,12 @@ addq $4 * SIZE, AO1 leaq (BO, M8, 4), BO decq I - jg .L32 + jg L(32) ALIGN_4 -.L33: +L(33): testq $2, N - jle .L34 + jle L(34) MOVQ 0 * SIZE(AO1), %mm0 MOVQ 1 * SIZE(AO1), %mm1 @@ -433,9 +433,9 @@ addq $2 * SIZE, BO1 ALIGN_4 -.L34: +L(34): testq $1, N - jle .L999 + jle L(999) MOVQ 0 * SIZE(AO1), %mm0 MOVNTQ %mm0, 0 * SIZE(BO2) @@ -443,7 +443,7 @@ addq $1 * SIZE, BO2 ALIGN_4 -.L999: +L(999): EMMS #ifdef WINDOWS_ABI diff --git a/kernel/x86_64/iamax.S b/kernel/x86_64/iamax.S index 00999e25f2..a8a0669dad 100644 --- a/kernel/x86_64/iamax.S +++ b/kernel/x86_64/iamax.S @@ -70,9 +70,9 @@ xorq RET, RET testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) ffreep %st movq $2, NUM @@ -84,17 +84,17 @@ #endif addq INCX, X decq M - jle .L999 + jle L(999) cmpq $SIZE, INCX - jne .L40 + jne L(40) movq M, I sarq $3, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -190,17 +190,17 @@ addq $8 * SIZE, X decq I - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): movq M, I andq $7, I - jle .L999 + jle L(999) ALIGN_4 -.L21: +L(21): FLD 0 * SIZE(X) #ifdef USE_ABS fabs @@ -214,17 +214,17 @@ addq $1 * SIZE, X incq NUM decq I - jg .L21 - jmp .L999 + jg L(21) + jmp L(999) ALIGN_4 -.L40: +L(40): movq M, I sarq $3, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): FLD 0 * SIZE(X) addq INCX, X #ifdef USE_ABS @@ -322,17 +322,17 @@ incq NUM decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): movq M, I andq $7, I - jle .L999 + jle L(999) ALIGN_4 -.L61: +L(61): FLD 0 * SIZE(X) #ifdef USE_ABS fabs @@ -346,10 +346,10 @@ addq INCX, X decq I - jg .L61 + jg L(61) ALIGN_4 -.L999: +L(999): ffreep %st ret diff --git a/kernel/x86_64/iamax_sse.S b/kernel/x86_64/iamax_sse.S index 6a5da06d52..11251612a9 100644 --- a/kernel/x86_64/iamax_sse.S +++ b/kernel/x86_64/iamax_sse.S @@ -76,10 +76,10 @@ pxor %xmm0, %xmm0 /* Return Value(Float) */ xor RET, RET /* Return Value(Int) */ testq M, M - jle .L999 + jle L(999) leaq (, INCX, SIZE), INCX testq INCX, INCX - jle .L999 + jle L(999) movq M, MM movq X, XX @@ -100,17 +100,17 @@ movups %xmm0, %xmm2 movups %xmm0, %xmm3 /* Generating "seed value" */ cmpq $SIZE, INCX - jne .L80 /* Incx != 1 goto L80 */ + jne L(80) /* Incx != 1 goto L80 */ /* Analigned Check */ testq $3, X /* 00000011 */ - jne .L30 /* Purely Unaligned Mode */ + jne L(30) /* Purely Unaligned Mode */ cmpq $8, M - jle .L30 /* if M <= 8 goto Unaligned mode */ + jle L(30) /* if M <= 8 goto Unaligned mode */ testq $4, X /* bit test 000100 */ - je .L05 + je L(05) movss 0 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -121,9 +121,9 @@ addq $SIZE, X ALIGN_3 -.L05: +L(05): testq $8, X - je .L06 + je L(06) movsd 0 * SIZE(X), %xmm4 unpcklps %xmm4, %xmm4 @@ -135,13 +135,13 @@ addq $2 * SIZE, X ALIGN_3 -.L06: +L(06): movq M, I sarq $4, I - jle .L15 + jle L(15) ALIGN_4 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -172,15 +172,15 @@ addq $16 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_4 -.L15: +L(15): andq $15, M - jle .L20 + jle L(20) testq $8, M - je .L16 + je L(16) movups 0 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -196,9 +196,9 @@ addq $8 * SIZE, X ALIGN_3 -.L16: +L(16): testq $4, M - je .L17 + je L(17) movups 0 * SIZE(X), %xmm6 #ifdef USE_ABS @@ -208,9 +208,9 @@ addq $4 * SIZE, X ALIGN_3 -.L17: +L(17): testq $2, M - je .L18 + je L(18) movsd 0 * SIZE(X), %xmm7 unpcklps %xmm7, %xmm7 @@ -220,9 +220,9 @@ MAXPS %xmm7, %xmm3 addq $2 * SIZE, X -.L18: +L(18): testq $1, M - je .L20 + je L(20) movss 0 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -231,7 +231,7 @@ MAXSS %xmm4, %xmm0 ALIGN_3 -.L20: +L(20): movq XX, X movq MM, M @@ -247,7 +247,7 @@ shufps $0, %xmm0, %xmm0 testq $4, X - je .L21 + je L(21) movss 0 * SIZE(X), %xmm1 @@ -259,12 +259,12 @@ #endif incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) ALIGN_3 -.L21: +L(21): testq $8, X - je .L22 + je L(22) movss 0 * SIZE(X), %xmm1 movss 1 * SIZE(X), %xmm2 @@ -278,19 +278,19 @@ #endif incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm2 - je .L999 + je L(999) ALIGN_3 -.L22: +L(22): movq M, I sarq $3, I - jle .L25 + jle L(25) ALIGN_4 -.L23: +L(23): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -314,16 +314,16 @@ .long 0xd9500f4c #endif testq $15, %r11 - jne .L24 + jne L(24) addq $8 * SIZE, X addq $8, RET decq I - jg .L23 - jmp .L25 + jg L(23) + jmp L(25) ALIGN_3 -.L24: +L(24): movss 0 * SIZE(X), %xmm1 movss 1 * SIZE(X), %xmm2 movss 2 * SIZE(X), %xmm3 @@ -347,32 +347,32 @@ incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm2 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm3 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm4 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm5 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm6 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm7 - je .L999 + je L(999) incq RET - jmp .L999 + jmp L(999) ALIGN_4 -.L25: +L(25): testq $4, M - je .L26 + je L(26) movss 0 * SIZE(X), %xmm1 movss 1 * SIZE(X), %xmm2 @@ -387,21 +387,21 @@ addq $4 * SIZE, X incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm2 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm3 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm4 - je .L999 + je L(999) ALIGN_3 -.L26: +L(26): testq $2, M - je .L27 + je L(27) movss 0 * SIZE(X), %xmm1 movss 1 * SIZE(X), %xmm2 @@ -412,25 +412,25 @@ addq $2 * SIZE, X incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm2 - je .L999 + je L(999) ALIGN_3 -.L27: +L(27): incq RET - jmp .L999 + jmp L(999) ALIGN_3 /* Unaligned Mode */ -.L30: +L(30): movq M, I sarq $4, I - jle .L35 + jle L(35) ALIGN_4 -.L31: +L(31): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -465,15 +465,15 @@ addq $16 * SIZE, X decq I - jg .L31 + jg L(31) ALIGN_4 -.L35: +L(35): andq $15, M - jle .L40 + jle L(40) testq $8, M - je .L36 + je L(36) movsd 0 * SIZE(X), %xmm4 movhps 2 * SIZE(X), %xmm4 @@ -492,9 +492,9 @@ addq $8 * SIZE, X ALIGN_3 -.L36: +L(36): testq $4, M - je .L37 + je L(37) movsd 0 * SIZE(X), %xmm6 movhps 2 * SIZE(X), %xmm6 @@ -505,9 +505,9 @@ addq $4 * SIZE, X ALIGN_3 -.L37: +L(37): testq $2, M - je .L38 + je L(38) movsd 0 * SIZE(X), %xmm7 unpcklps %xmm7, %xmm7 @@ -517,19 +517,19 @@ MAXPS %xmm7, %xmm3 addq $2 * SIZE, X -.L38: +L(38): testq $1, M - je .L40 + je L(40) movss 0 * SIZE(X), %xmm4 #ifdef USE_ABS andps %xmm15, %xmm4 #endif MAXSS %xmm4, %xmm0 - jmp .L40 + jmp L(40) ALIGN_4 -.L40: +L(40): movq XX, X movq MM, M @@ -546,10 +546,10 @@ movq M, I sarq $3, I - jle .L45 + jle L(45) ALIGN_4 -.L43: +L(43): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -575,16 +575,16 @@ .long 0xd9500f4c #endif testq $15, %r11 - jne .L44 + jne L(44) addq $8 * SIZE, X addq $8, RET decq I - jg .L43 - jmp .L45 + jg L(43) + jmp L(45) ALIGN_3 -.L44: +L(44): movss 0 * SIZE(X), %xmm1 movss 1 * SIZE(X), %xmm2 movss 2 * SIZE(X), %xmm3 @@ -608,32 +608,32 @@ incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm2 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm3 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm4 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm5 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm6 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm7 - je .L999 + je L(999) incq RET - jmp .L999 + jmp L(999) ALIGN_4 -.L45: +L(45): testq $4, M - je .L46 + je L(46) movss 0 * SIZE(X), %xmm1 movss 1 * SIZE(X), %xmm2 @@ -648,21 +648,21 @@ addq $4 * SIZE, X incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm2 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm3 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm4 - je .L999 + je L(999) ALIGN_3 -.L46: +L(46): testq $2, M - je .L47 + je L(47) movss 0 * SIZE(X), %xmm1 movss 1 * SIZE(X), %xmm2 @@ -673,24 +673,24 @@ addq $2 * SIZE, X incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm2 - je .L999 + je L(999) ALIGN_3 -.L47: +L(47): incq RET - jmp .L999 + jmp L(999) ALIGN_3 -.L80: +L(80): movq M, I sarq $3, I - jle .L85 + jle L(85) ALIGN_4 -.L81: +L(81): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -752,15 +752,15 @@ MAXSS %xmm7, %xmm3 decq I - jg .L81 + jg L(81) ALIGN_4 -.L85: +L(85): andq $7, M - jle .L90 + jle L(90) testq $4, M - je .L86 + je L(86) movss 0 * SIZE(X), %xmm4 addq INCX, X @@ -791,9 +791,9 @@ MAXSS %xmm7, %xmm3 ALIGN_3 -.L86: +L(86): testq $2, M - je .L87 + je L(87) movss 0 * SIZE(X), %xmm4 addq INCX, X @@ -810,9 +810,9 @@ MAXSS %xmm5, %xmm1 ALIGN_3 -.L87: +L(87): testq $1, M - je .L90 + je L(90) movss 0 * SIZE(X), %xmm6 addq INCX, X @@ -822,7 +822,7 @@ MAXSS %xmm6, %xmm2 ALIGN_4 -.L90: +L(90): movq XX, X movq MM, M @@ -833,10 +833,10 @@ movq M, I sarq $3, I - jle .L95 + jle L(95) ALIGN_4 -.L93: +L(93): LOAD_AND_COMPARE_TO_MXX %xmm1 LOAD_AND_COMPARE_TO_MXX %xmm2 LOAD_AND_COMPARE_TO_MXX %xmm3 @@ -860,15 +860,15 @@ .long 0xd9500f4c #endif testq $15, %r11 - jne .L94 + jne L(94) addq $8, RET decq I - jg .L93 - jmp .L95 + jg L(93) + jmp L(95) ALIGN_3 -.L94: +L(94): subq INCX, X movss 0 * SIZE(X), %xmm8 subq INCX, X @@ -897,32 +897,32 @@ #endif incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm2 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm3 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm4 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm5 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm6 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm7 - je .L999 + je L(999) incq RET - jmp .L999 + jmp L(999) ALIGN_4 -.L95: +L(95): testq $4, M - je .L96 + je L(96) movss 0 * SIZE(X), %xmm1 addq INCX, X @@ -941,21 +941,21 @@ #endif incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm2 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm3 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm4 - je .L999 + je L(999) ALIGN_3 -.L96: +L(96): testq $2, M - je .L97 + je L(97) movss 0 * SIZE(X), %xmm1 addq INCX, X @@ -967,17 +967,17 @@ #endif incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm2 - je .L999 + je L(999) ALIGN_3 -.L97: +L(97): incq RET ALIGN_3 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/iamax_sse2.S b/kernel/x86_64/iamax_sse2.S index 6808f191ba..79ff582b67 100644 --- a/kernel/x86_64/iamax_sse2.S +++ b/kernel/x86_64/iamax_sse2.S @@ -63,10 +63,10 @@ pxor %xmm0, %xmm0 xor RET, RET testq M, M - jle .L999 + jle L(999) leaq (, INCX, SIZE), INCX testq INCX, INCX - jle .L999 + jle L(999) movq M, MM movq X, XX @@ -87,17 +87,17 @@ movapd %xmm0, %xmm2 movapd %xmm0, %xmm3 cmpq $SIZE, INCX - jne .L80 + jne L(80) /* Analigned Check */ cmpq $7, M - jle .L50 + jle L(50) testq $7, X - jne .L50 # Purely Unaligned Mode + jne L(50) # Purely Unaligned Mode testq $15, X # Checking for 128bit align - je .L05 + je L(05) movsd 0 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -109,13 +109,13 @@ addq $SIZE, X ALIGN_3 -.L05: +L(05): movq M, I sarq $4, I - jle .L15 + jle L(15) ALIGN_4 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -174,15 +174,15 @@ addq $16 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_4 -.L15: +L(15): andq $15, M - jle .L20 + jle L(20) testq $8, M - je .L16 + je L(16) movapd 0 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -210,9 +210,9 @@ addq $8 * SIZE, X ALIGN_3 -.L16: +L(16): testq $4, M - je .L17 + je L(17) movapd 0 * SIZE(X), %xmm4 #ifdef USE_ABS @@ -228,9 +228,9 @@ addq $4 * SIZE, X ALIGN_3 -.L17: +L(17): testq $2, M - je .L18 + je L(18) movapd 0 * SIZE(X), %xmm6 #ifdef USE_ABS @@ -239,9 +239,9 @@ maxpd %xmm6, %xmm2 addq $2 * SIZE, X -.L18: +L(18): testq $1, M - je .L20 + je L(20) movsd 0 * SIZE(X), %xmm7 #ifdef USE_ABS @@ -252,7 +252,7 @@ ALIGN_3 /* Finding Index */ -.L20: +L(20): movq XX, X movq MM, M @@ -266,7 +266,7 @@ ALIGN_3 testq $15, X # Checking for 128bit align - je .L21 + je L(21) movsd 0 * SIZE(X), %xmm1 #ifdef USE_ABS @@ -274,18 +274,18 @@ #endif incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) addq $SIZE, X decq M ALIGN_3 -.L21: +L(21): movq M, I sarq $3, I - jle .L25 + jle L(25) ALIGN_4 -.L22: +L(22): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -324,16 +324,16 @@ .long 0xd9500f4c #endif testq $3, %r11 - jne .L23 + jne L(23) addq $8 * SIZE, X addq $8, RET decq I - jg .L22 - jmp .L25 + jg L(22) + jmp L(25) ALIGN_4 -.L23: +L(23): movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 movsd 2 * SIZE(X), %xmm3 @@ -357,32 +357,32 @@ incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm2 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm3 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm4 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm5 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm6 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm7 - je .L999 + je L(999) incq RET - jmp .L999 + jmp L(999) ALIGN_3 -.L25: +L(25): testq $4, M - je .L27 + je L(27) movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 @@ -397,21 +397,21 @@ addq $4 * SIZE, X incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm2 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm3 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm4 - je .L999 + je L(999) ALIGN_3 -.L27: +L(27): testq $2, M - je .L28 + je L(28) movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 @@ -422,25 +422,25 @@ addq $2 * SIZE, X incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm2 - je .L999 + je L(999) ALIGN_3 -.L28: +L(28): incq RET - jmp .L999 + jmp L(999) ALIGN_3 /* Unaligned Mode */ -.L50: +L(50): movq M, I sarq $4, I - jle .L55 + jle L(55) ALIGN_4 -.L51: +L(51): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -507,15 +507,15 @@ addq $16 * SIZE, X decq I - jg .L51 + jg L(51) ALIGN_4 -.L55: +L(55): andq $15, M - jle .L60 + jle L(60) testq $8, M - je .L56 + je L(56) movsd 0 * SIZE(X), %xmm4 movhpd 1 * SIZE(X), %xmm4 @@ -548,9 +548,9 @@ addq $8 * SIZE, X ALIGN_3 -.L56: +L(56): testq $4, M - je .L57 + je L(57) movsd 0 * SIZE(X), %xmm4 movhpd 1 * SIZE(X), %xmm4 @@ -568,9 +568,9 @@ addq $4 * SIZE, X ALIGN_3 -.L57: +L(57): testq $2, M - je .L58 + je L(58) movsd 0 * SIZE(X), %xmm6 movhpd 1 * SIZE(X), %xmm6 @@ -580,9 +580,9 @@ maxpd %xmm6, %xmm2 addq $2 * SIZE, X -.L58: +L(58): testq $1, M - je .L60 + je L(60) movsd 0 * SIZE(X), %xmm7 unpcklpd %xmm7, %xmm7 @@ -592,7 +592,7 @@ maxpd %xmm7, %xmm3 ALIGN_3 -.L60: +L(60): movq XX, X movq MM, M @@ -606,10 +606,10 @@ movq M, I sarq $3, I - jle .L65 + jle L(65) ALIGN_4 -.L62: +L(62): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -652,16 +652,16 @@ .long 0xd9500f4c #endif testq $3, %r11 - jne .L63 + jne L(63) addq $8 * SIZE, X addq $8, RET decq I - jg .L62 - jmp .L65 + jg L(62) + jmp L(65) ALIGN_4 -.L63: +L(63): movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 movsd 2 * SIZE(X), %xmm3 @@ -685,32 +685,32 @@ incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm2 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm3 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm4 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm5 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm6 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm7 - je .L999 + je L(999) incq RET - jmp .L999 + jmp L(999) ALIGN_3 -.L65: +L(65): testq $4, M - je .L67 + je L(67) movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 @@ -725,21 +725,21 @@ addq $4 * SIZE, X incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm2 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm3 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm4 - je .L999 + je L(999) ALIGN_3 -.L67: +L(67): testq $2, M - je .L68 + je L(68) movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 @@ -750,24 +750,24 @@ addq $2 * SIZE, X incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm2 - je .L999 + je L(999) ALIGN_3 -.L68: +L(68): incq RET - jmp .L999 + jmp L(999) ALIGN_4 -.L80: +L(80): movq M, I sarq $4, I - jle .L85 + jle L(85) ALIGN_4 -.L81: +L(81): movsd 0 * SIZE(X), %xmm4 addq INCX, X movhpd 0 * SIZE(X), %xmm4 @@ -841,15 +841,15 @@ maxpd %xmm7, %xmm3 decq I - jg .L81 + jg L(81) ALIGN_4 -.L85: +L(85): andq $15, M - jle .L90 + jle L(90) testq $8, M - je .L86 + je L(86) movsd 0 * SIZE(X), %xmm4 addq INCX, X @@ -888,9 +888,9 @@ maxpd %xmm7, %xmm3 ALIGN_3 -.L86: +L(86): testq $4, M - je .L87 + je L(87) movsd 0 * SIZE(X), %xmm4 addq INCX, X @@ -911,9 +911,9 @@ maxpd %xmm5, %xmm1 ALIGN_3 -.L87: +L(87): testq $2, M - je .L88 + je L(88) movsd 0 * SIZE(X), %xmm6 addq INCX, X @@ -925,9 +925,9 @@ maxpd %xmm6, %xmm2 ALIGN_3 -.L88: +L(88): testq $1, M - je .L90 + je L(90) movsd 0 * SIZE(X), %xmm7 unpcklpd %xmm7, %xmm7 @@ -944,7 +944,7 @@ maxsd %xmm1, %xmm0 ALIGN_4 -.L90: +L(90): movq XX, X movq MM, M @@ -958,10 +958,10 @@ movq M, I sarq $3, I - jle .L95 + jle L(95) ALIGN_4 -.L92: +L(92): movsd 0 * SIZE(X), %xmm1 addq INCX, X movhpd 0 * SIZE(X), %xmm1 @@ -1008,15 +1008,15 @@ .long 0xd9500f4c #endif testq $3, %r11 - jne .L93 + jne L(93) addq $8, RET decq I - jg .L92 - jmp .L95 + jg L(92) + jmp L(95) ALIGN_4 -.L93: +L(93): subq INCX, X movsd 0 * SIZE(X), %xmm8 subq INCX, X @@ -1048,32 +1048,32 @@ incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm2 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm3 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm4 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm5 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm6 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm7 - je .L999 + je L(999) incq RET - jmp .L999 + jmp L(999) ALIGN_3 -.L95: +L(95): testq $4, M - je .L97 + je L(97) movsd 0 * SIZE(X), %xmm1 addq INCX, X @@ -1091,21 +1091,21 @@ #endif incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm2 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm3 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm4 - je .L999 + je L(999) ALIGN_3 -.L97: +L(97): testq $2, M - je .L98 + je L(98) movsd 0 * SIZE(X), %xmm1 addq INCX, X @@ -1117,17 +1117,17 @@ #endif incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm2 - je .L999 + je L(999) ALIGN_3 -.L98: +L(98): incq RET ALIGN_3 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/izamax.S b/kernel/x86_64/izamax.S index b24b2e6925..c76a7e8bb0 100644 --- a/kernel/x86_64/izamax.S +++ b/kernel/x86_64/izamax.S @@ -70,9 +70,9 @@ xorq RET, RET testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) ffreep %st movq $2, NUM @@ -85,17 +85,17 @@ faddp %st, %st(1) addq INCX, X decq M - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX - jne .L40 + jne L(40) movq M, I sarq $2, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -151,16 +151,16 @@ addq $8 * SIZE, X decq I - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): movq M, I andq $3, I - jle .L999 + jle L(999) ALIGN_4 -.L21: +L(21): FLD 0 * SIZE(X) fabs FLD 1 * SIZE(X) @@ -175,17 +175,17 @@ addq $2 * SIZE, X decq I - jg .L21 - jmp .L999 + jg L(21) + jmp L(999) ALIGN_4 -.L40: +L(40): movq M, I sarq $2, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): FLD 0 * SIZE(X) fabs FLD 1 * SIZE(X) @@ -239,17 +239,17 @@ incq NUM decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): movq M, I andq $3, I - jle .L999 + jle L(999) ALIGN_4 -.L61: +L(61): FLD 0 * SIZE(X) fabs FLD 1 * SIZE(X) @@ -264,10 +264,10 @@ addq INCX, X decq I - jg .L61 + jg L(61) ALIGN_4 -.L999: +L(999): ffreep %st ret diff --git a/kernel/x86_64/izamax_sse.S b/kernel/x86_64/izamax_sse.S index e273b8cc6e..84ffee6a55 100644 --- a/kernel/x86_64/izamax_sse.S +++ b/kernel/x86_64/izamax_sse.S @@ -63,9 +63,9 @@ pxor %xmm0, %xmm0 xor RET, RET testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) salq $ZBASE_SHIFT, INCX movq M, MM @@ -84,15 +84,15 @@ shufps $0, %xmm0, %xmm0 movaps %xmm0, %xmm1 cmpq $2 * SIZE, INCX - jne .L70 + jne L(70) -.L30: +L(30): movq M, I sarq $3, I - jle .L35 + jle L(35) ALIGN_4 -.L31: +L(31): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -128,15 +128,15 @@ addq $16 * SIZE, X decq I - jg .L31 + jg L(31) ALIGN_4 -.L35: +L(35): andq $7, M - jle .L40 + jle L(40) testq $4, M - je .L36 + je L(36) movsd 0 * SIZE(X), %xmm4 movhps 2 * SIZE(X), %xmm4 @@ -155,9 +155,9 @@ addq $8 * SIZE, X ALIGN_3 -.L36: +L(36): testq $2, M - je .L37 + je L(37) movss 0 * SIZE(X), %xmm4 movss 1 * SIZE(X), %xmm5 @@ -174,9 +174,9 @@ addq $4 * SIZE, X ALIGN_3 -.L37: +L(37): testq $1, M - je .L40 + je L(40) movss 0 * SIZE(X), %xmm4 movss 1 * SIZE(X), %xmm5 @@ -186,7 +186,7 @@ maxss %xmm4, %xmm0 ALIGN_4 -.L40: +L(40): movq XX, X movq MM, M @@ -201,10 +201,10 @@ movq M, I sarq $2, I - jle .L45 + jle L(45) ALIGN_4 -.L41: +L(41): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -230,16 +230,16 @@ .long 0xd9500f4c #endif testq $15, %r11 - jne .L43 + jne L(43) addq $8 * SIZE, X addq $4, RET decq I - jg .L41 - jmp .L45 + jg L(41) + jmp L(45) ALIGN_4 -.L43: +L(43): movss 0 * SIZE(X), %xmm1 movss 1 * SIZE(X), %xmm2 movss 2 * SIZE(X), %xmm3 @@ -266,21 +266,21 @@ incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm3 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm5 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm7 - je .L999 + je L(999) ALIGN_3 -.L45: +L(45): testq $2, M - je .L47 + je L(47) movss 0 * SIZE(X), %xmm1 movss 1 * SIZE(X), %xmm2 @@ -297,24 +297,24 @@ incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm3 - je .L999 + je L(999) ALIGN_3 -.L47: +L(47): incq RET - jmp .L999 + jmp L(999) ALIGN_3 -.L70: +L(70): movq M, I sarq $3, I - jle .L75 + jle L(75) ALIGN_4 -.L71: +L(71): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -357,15 +357,15 @@ maxps %xmm7, %xmm0 decq I - jg .L71 + jg L(71) ALIGN_4 -.L75: +L(75): andq $7, M - jle .L80 + jle L(80) testq $4, M - je .L76 + je L(76) movsd 0 * SIZE(X), %xmm4 addq INCX, X @@ -386,9 +386,9 @@ maxps %xmm4, %xmm0 ALIGN_3 -.L76: +L(76): testq $2, M - je .L77 + je L(77) movss 0 * SIZE(X), %xmm4 movss 1 * SIZE(X), %xmm5 @@ -406,9 +406,9 @@ maxss %xmm6, %xmm1 ALIGN_3 -.L77: +L(77): testq $1, M - je .L80 + je L(80) movss 0 * SIZE(X), %xmm4 movss 1 * SIZE(X), %xmm5 @@ -418,7 +418,7 @@ maxss %xmm4, %xmm0 ALIGN_4 -.L80: +L(80): movq XX, X movq MM, M @@ -433,10 +433,10 @@ movq M, I sarq $2, I - jle .L85 + jle L(85) ALIGN_4 -.L81: +L(81): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -466,15 +466,15 @@ .long 0xd9500f4c #endif testq $15, %r11 - jne .L83 + jne L(83) addq $4, RET decq I - jg .L81 - jmp .L85 + jg L(81) + jmp L(85) ALIGN_4 -.L83: +L(83): subq INCX, X movss 0 * SIZE(X), %xmm7 movss 1 * SIZE(X), %xmm8 @@ -504,21 +504,21 @@ incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm3 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm5 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm7 - je .L999 + je L(999) ALIGN_3 -.L85: +L(85): testq $2, M - je .L87 + je L(87) movss 0 * SIZE(X), %xmm1 movss 1 * SIZE(X), %xmm2 @@ -536,17 +536,17 @@ incq RET comiss %xmm0, %xmm1 - je .L999 + je L(999) incq RET comiss %xmm0, %xmm3 - je .L999 + je L(999) ALIGN_3 -.L87: +L(87): incq RET ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/izamax_sse2.S b/kernel/x86_64/izamax_sse2.S index c656a652dc..9f8dc3af8f 100644 --- a/kernel/x86_64/izamax_sse2.S +++ b/kernel/x86_64/izamax_sse2.S @@ -63,9 +63,9 @@ pxor %xmm0, %xmm0 xor RET, RET testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) salq $ZBASE_SHIFT, INCX movq M, MM @@ -86,14 +86,14 @@ movapd %xmm0, %xmm2 movapd %xmm0, %xmm3 cmpq $2 * SIZE, INCX - jne .L60 + jne L(60) movq M, I sarq $3, I - jle .L25 + jle L(25) ALIGN_4 -.L21: +L(21): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -142,15 +142,15 @@ addq $16 * SIZE, X decq I - jg .L21 + jg L(21) ALIGN_4 -.L25: +L(25): andq $7, M - jle .L30 + jle L(30) testq $4, M - je .L26 + je L(26) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -174,9 +174,9 @@ addq $8 * SIZE, X ALIGN_3 -.L26: +L(26): testq $2, M - je .L27 + je L(27) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -190,9 +190,9 @@ maxpd %xmm4, %xmm0 ALIGN_3 -.L27: +L(27): testq $1, M - je .L30 + je L(30) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -202,7 +202,7 @@ maxsd %xmm4, %xmm2 ALIGN_4 -.L30: +L(30): movq XX, X movq MM, M @@ -216,10 +216,10 @@ movq M, I sarq $2, I - jle .L35 + jle L(35) ALIGN_4 -.L31: +L(31): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -252,16 +252,16 @@ .long 0xd9500f4c #endif testq $3, %r11 - jne .L33 + jne L(33) addq $8 * SIZE, X addq $4, RET decq I - jg .L31 - jmp .L35 + jg L(31) + jmp L(35) ALIGN_4 -.L33: +L(33): movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 movsd 2 * SIZE(X), %xmm3 @@ -288,21 +288,21 @@ incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm3 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm5 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm7 - je .L999 + je L(999) ALIGN_3 -.L35: +L(35): testq $2, M - je .L36 + je L(36) movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 @@ -319,24 +319,24 @@ addpd %xmm4, %xmm3 incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm3 - je .L999 + je L(999) ALIGN_3 -.L36: +L(36): incq RET - jmp .L999 + jmp L(999) ALIGN_3 -.L60: +L(60): movq M, I sarq $3, I - jle .L65 + jle L(65) ALIGN_4 -.L61: +L(61): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -390,15 +390,15 @@ maxpd %xmm6, %xmm3 decq I - jg .L61 + jg L(61) ALIGN_4 -.L65: +L(65): andq $7, M - jle .L70 + jle L(70) testq $4, M - je .L66 + je L(66) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -423,9 +423,9 @@ maxpd %xmm6, %xmm1 ALIGN_3 -.L66: +L(66): testq $2, M - je .L67 + je L(67) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -439,9 +439,9 @@ maxpd %xmm4, %xmm2 ALIGN_3 -.L67: +L(67): testq $1, M - je .L70 + je L(70) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -451,7 +451,7 @@ maxsd %xmm4, %xmm3 ALIGN_3 -.L70: +L(70): movq XX, X movq MM, M @@ -465,10 +465,10 @@ movq M, I sarq $2, I - jle .L75 + jle L(75) ALIGN_4 -.L71: +L(71): #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) prefetch PREFETCHSIZE * SIZE(X) #endif @@ -509,15 +509,15 @@ .long 0xd9500f4c #endif testq $3, %r11 - jne .L73 + jne L(73) addq $4, RET decq I - jg .L71 - jmp .L75 + jg L(71) + jmp L(75) ALIGN_4 -.L73: +L(73): subq INCX, X movsd 0 * SIZE(X), %xmm7 movsd 1 * SIZE(X), %xmm8 @@ -547,21 +547,21 @@ incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm3 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm5 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm7 - je .L999 + je L(999) ALIGN_3 -.L75: +L(75): testq $2, M - je .L76 + je L(76) movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 @@ -579,17 +579,17 @@ addpd %xmm4, %xmm3 incq RET comisd %xmm0, %xmm1 - je .L999 + je L(999) incq RET comisd %xmm0, %xmm3 - je .L999 + je L(999) ALIGN_3 -.L76: +L(76): incq RET ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/nrm2.S b/kernel/x86_64/nrm2.S index 61cf8c4529..165aa36215 100644 --- a/kernel/x86_64/nrm2.S +++ b/kernel/x86_64/nrm2.S @@ -56,9 +56,9 @@ fldz testq M, M - jle .L999 + jle L(999) testq INCX, INCX - je .L999 + je L(999) salq $BASE_SHIFT, INCX @@ -66,14 +66,14 @@ fldz fldz cmpq $SIZE, INCX - jne .L40 + jne L(40) movq M, I sarq $3, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -109,32 +109,32 @@ faddp %st, %st(1) decq I - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): andq $7, M - jle .L998 + jle L(998) ALIGN_4 -.L21: +L(21): FLD (X) fmul %st(0), %st faddp %st,%st(1) addq $1 * SIZE, X decq M - jg .L21 - jmp .L998 + jg L(21) + jmp L(998) ALIGN_4 -.L40: +L(40): movq M, I sarq $3, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): FLD (X) addq INCX, X fmul %st(0), %st @@ -172,31 +172,31 @@ faddp %st, %st(1) decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): andq $7, M - jle .L998 + jle L(998) ALIGN_4 -.L61: +L(61): FLD (X) addq INCX, X fmul %st(0), %st faddp %st,%st(1) decq M - jg .L61 + jg L(61) ALIGN_4 -.L998: +L(998): faddp %st,%st(2) faddp %st,%st(1) faddp %st,%st(1) ALIGN_4 -.L999: +L(999): fsqrt #ifndef XDOUBLE sub $2 * SIZE, %rsp diff --git a/kernel/x86_64/nrm2_sse.S b/kernel/x86_64/nrm2_sse.S index c1f3a45fc5..42334b51c8 100644 --- a/kernel/x86_64/nrm2_sse.S +++ b/kernel/x86_64/nrm2_sse.S @@ -54,19 +54,19 @@ pxor %xmm0, %xmm0 testq M, M - jle .L999 + jle L(999) pxor %xmm1, %xmm1 testq INCX, INCX - je .L999 + je L(999) pxor %xmm2, %xmm2 leaq (, INCX, SIZE), INCX pxor %xmm3, %xmm3 cmpq $SIZE, INCX - jne .L40 + jne L(40) testq $SIZE, X - je .L05 + je L(05) movss 0 * SIZE(X), %xmm4 cvtss2sd %xmm4, %xmm6 @@ -74,13 +74,13 @@ addsd %xmm6, %xmm3 addq INCX, X decq M - jle .L998 + jle L(998) ALIGN_3 -.L05: +L(05): movq M, I sarq $3, I - jle .L14 + jle L(14) movsd 0 * SIZE(X), %xmm4 movsd 2 * SIZE(X), %xmm5 @@ -88,10 +88,10 @@ movsd 6 * SIZE(X), %xmm7 addq $8 * SIZE, X decq I - jle .L12 + jle L(12) ALIGN_3 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -118,10 +118,10 @@ addq $8 * SIZE, X decq I - jg .L10 + jg L(10) ALIGN_3 -.L12: +L(12): cvtps2pd %xmm4, %xmm8 cvtps2pd %xmm5, %xmm9 cvtps2pd %xmm6, %xmm10 @@ -139,9 +139,9 @@ ALIGN_3 -.L14: +L(14): testq $4, M - je .L15 + je L(15) movsd 0 * SIZE(X), %xmm4 movsd 2 * SIZE(X), %xmm5 @@ -154,9 +154,9 @@ addq $4 * SIZE, X ALIGN_3 -.L15: +L(15): testq $2, M - je .L16 + je L(16) movsd 0 * SIZE(X), %xmm4 cvtps2pd %xmm4, %xmm6 @@ -165,24 +165,24 @@ addq $2 * SIZE, X ALIGN_3 -.L16: +L(16): testq $1, M - je .L998 + je L(998) movss 0 * SIZE(X), %xmm4 cvtss2sd %xmm4, %xmm6 mulsd %xmm6, %xmm6 addsd %xmm6, %xmm3 - jmp .L998 + jmp L(998) ALIGN_4 -.L40: +L(40): movq M, I sarq $3, I - jle .L44 + jle L(44) ALIGN_4 -.L41: +L(41): movss (X), %xmm4 addq INCX, X movss (X), %xmm5 @@ -230,12 +230,12 @@ addsd %xmm11, %xmm3 decq I - jg .L41 + jg L(41) ALIGN_3 -.L44: +L(44): testq $4, M - je .L45 + je L(45) movss (X), %xmm4 addq INCX, X @@ -262,9 +262,9 @@ addsd %xmm11, %xmm3 ALIGN_3 -.L45: +L(45): testq $2, M - je .L46 + je L(46) movss (X), %xmm4 addq INCX, X @@ -279,9 +279,9 @@ addsd %xmm7, %xmm2 ALIGN_3 -.L46: +L(46): testq $1, M - je .L998 + je L(998) movss (X), %xmm4 cvtss2sd %xmm4, %xmm6 @@ -289,7 +289,7 @@ addsd %xmm6, %xmm3 ALIGN_4 -.L998: +L(998): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -303,7 +303,7 @@ #endif ALIGN_4 -.L999: +L(999): sqrtsd %xmm0, %xmm0 cvtsd2ss %xmm0, %xmm0 diff --git a/kernel/x86_64/qdot.S b/kernel/x86_64/qdot.S index 2243b6b6d8..59e46a17c0 100644 --- a/kernel/x86_64/qdot.S +++ b/kernel/x86_64/qdot.S @@ -83,16 +83,16 @@ fldz cmpl $SIZE, INCX - jne .L14 + jne L(14) cmpl $SIZE, INCY - jne .L14 + jne L(14) movl N, %eax sarl $2, %eax - jle .L15 + jle L(15) ALIGN_3 -.L16: +L(16): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -121,16 +121,16 @@ addl $4 * SIZE, X addl $4 * SIZE, Y decl %eax - jg .L16 + jg L(16) ALIGN_3 -.L15: +L(15): movl N, %eax andl $3, %eax - jle .L27 + jle L(27) ALIGN_3 -.L22: +L(22): FLD (X) addl $SIZE, X FLD (Y) @@ -138,18 +138,18 @@ addl $SIZE, Y faddp %st,%st(1) decl %eax - jg .L22 + jg L(22) - jmp .L27 + jmp L(27) ALIGN_3 -.L14: +L(14): movl N, %eax sarl $2, %eax - jle .L30 + jle L(30) ALIGN_3 -.L31: +L(31): FLD (X) addl INCX, X FLD (Y) @@ -179,16 +179,16 @@ faddp %st,%st(4) decl %eax - jg .L31 + jg L(31) ALIGN_3 -.L30: +L(30): movl N, %eax andl $3, %eax - jle .L27 + jle L(27) ALIGN_3 -.L37: +L(37): FLD (X) addl INCX, X FLD (Y) @@ -196,10 +196,10 @@ addl INCY, Y faddp %st, %st(1) decl %eax - jg .L37 + jg L(37) ALIGN_3 -.L27: +L(27): faddp %st,%st(2) faddp %st,%st(2) faddp %st,%st(1) diff --git a/kernel/x86_64/qgemm_kernel_2x2.S b/kernel/x86_64/qgemm_kernel_2x2.S index c11f3a91d6..340e4fe352 100644 --- a/kernel/x86_64/qgemm_kernel_2x2.S +++ b/kernel/x86_64/qgemm_kernel_2x2.S @@ -102,10 +102,10 @@ movq N, %rax sarq $1, %rax movq %rax, J - je .L30 + je L(30) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -119,10 +119,10 @@ movq M, I sarq $1, I - je .L20 + je L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -163,10 +163,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -247,20 +247,20 @@ addq $8 * SIZE,BO decq %rax - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif and $3, %rax - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -283,10 +283,10 @@ addq $2 * SIZE,BO decq %rax - jne .L16 + jne L(16) ALIGN_4 -.L18: +L(18): #ifndef TRMMKERNEL FLD ALPHA @@ -332,16 +332,16 @@ addq $2 * SIZE, CO decq I - jne .L11 + jne L(11) ALIGN_4 -.L20: +L(20): movq M, %rax andq $1, %rax - je .L29 + je L(29) ALIGN_4 -.L21: +L(21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -372,10 +372,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -422,20 +422,20 @@ addq $8 * SIZE,BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif and $3, %rax - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -450,10 +450,10 @@ addq $2 * SIZE,BO decq %rax - jne .L26 + jne L(26) ALIGN_4 -.L28: +L(28): #ifndef TRMMKERNEL FLD ALPHA @@ -488,20 +488,20 @@ addq $1 * SIZE, CO ALIGN_4 -.L29: +L(29): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif movq BO, B decq J - jne .L01 + jne L(01) ALIGN_4 -.L30: +L(30): movq N, %rax testq $1, %rax - je .L999 + je L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -515,10 +515,10 @@ movq M, I sarq $1, I - je .L40 + je L(40) ALIGN_4 -.L31: +L(31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -555,10 +555,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(BO) @@ -601,20 +601,20 @@ addq $4 * SIZE,BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif and $3, %rax - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): FLD -8 * SIZE(BO) FLD -8 * SIZE(AO) @@ -629,10 +629,10 @@ addq $1 * SIZE,BO decq %rax - jne .L36 + jne L(36) ALIGN_4 -.L38: +L(38): #ifndef TRMMKERNEL FLD ALPHA @@ -666,16 +666,16 @@ addq $2 * SIZE, CO decq I - jne .L31 + jne L(31) ALIGN_4 -.L40: +L(40): movq M, %rax andq $1, %rax - je .L49 + je L(49) ALIGN_4 -.L41: +L(41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -705,10 +705,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -735,20 +735,20 @@ addq $4 * SIZE,BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif and $3, %rax - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -759,10 +759,10 @@ addq $1 * SIZE,BO decq %rax - jne .L46 + jne L(46) ALIGN_4 -.L48: +L(48): #ifndef TRMMKERNEL FLD ALPHA @@ -791,7 +791,7 @@ addq $1 * SIZE, CO ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addq $1, KK #endif @@ -799,7 +799,7 @@ movq BO, B ALIGN_4 -.L999: +L(999): EMMS movq 0(%rsp), %rbx diff --git a/kernel/x86_64/qgemv_n.S b/kernel/x86_64/qgemv_n.S index c9d345cb18..7e24b931c3 100644 --- a/kernel/x86_64/qgemv_n.S +++ b/kernel/x86_64/qgemv_n.S @@ -100,9 +100,9 @@ movq $0, IS test M, M - jle .L79 # goto END + jle L(79) # goto END test N, N - jle .L79 # goto END + jle L(79) # goto END movq LDA, %rax imulq $P, %rax # P * lda @@ -113,7 +113,7 @@ salq $BASE_SHIFT, LDA ALIGN_2 -.L32: +L(32): movq $P, %rax movq N, MIN_N subq IS, MIN_N @@ -125,17 +125,17 @@ leaq (X,XP, 1), XP cmpq $SIZE, INCX - je .L34 # if incx == 1 goto L34 + je L(34) # if incx == 1 goto L34 movq BUFFER, XP movq XP, X1 movq MIN_N, I sarq $2,I - jle .L35 + jle L(35) ALIGN_2 -.L36: +L(36): FLD (X) addq INCX,X FLD (X) @@ -152,33 +152,33 @@ addq $4 * SIZE, X1 decq I - jg .L36 + jg L(36) ALIGN_3 -.L35: +L(35): movq MIN_N, I andq $3, I - jle .L34 + jle L(34) ALIGN_2 -.L42: +L(42): FLD (X) addq INCX, X FST (X1) addq $SIZE, X1 decq I - jg .L42 + jg L(42) ALIGN_3 /* Main Routine */ -.L34: +L(34): movq Y, Y1 movq M, J sarq $2, J - jle .L47 + jle L(47) ALIGN_2 -.L48: +L(48): movq A, A1 # a_offset = a fldz addq $4 * SIZE, A # a += 4 @@ -189,10 +189,10 @@ fldz FLD (X1) # bt1 = b_offset sarq $1, I - jle .L51 + jle L(51) ALIGN_2 -.L80: +L(80): FLD 0 * SIZE(A1) # at1 = *(a_offset + 0) fmul %st(1), %st # at1 *= bt1 faddp %st, %st(2) # ct1 += at1 @@ -233,12 +233,12 @@ addq $2 * SIZE, X1 # b_offset += 2 decq I - jg .L80 + jg L(80) -.L51: +L(51): movq MIN_N, I andq $1, I - je .L57 + je L(57) FLD 0 * SIZE(A1) # at1 = *(a_offset + 0) fmul %st(1), %st # at1 *= bt1 @@ -258,7 +258,7 @@ fldz ALIGN_2 -.L57: +L(57): ffreep %st(0) fxch %st(4) @@ -289,16 +289,16 @@ addq INCY, Y1 decq J # j -- - jg .L48 + jg L(48) ALIGN_3 -.L47: +L(47): movq M, J andq $3, J # j = (m & 3) - jle .L60 + jle L(60) ALIGN_2 -.L61: +L(61): movq A, A1 # a_offset = a fldz addq $SIZE, A # a++ @@ -308,10 +308,10 @@ fldz movq MIN_N, I sarq $3, I - jle .L64 + jle L(64) ALIGN_2 -.L65: +L(65): FLD 0 * SIZE(X1) FLD (A1) fmulp %st, %st(1) @@ -362,15 +362,15 @@ addq $8 * SIZE, X1 decq I - jg .L65 + jg L(65) -.L64: +L(64): movq MIN_N,I andq $7, I - jle .L70 + jle L(70) ALIGN_2 -.L71: +L(71): FLD (X1) addq $SIZE, X1 FLD (A1) @@ -378,10 +378,10 @@ addq LDA, A1 # a_offset += lda faddp %st, %st(1) decq I - jg .L71 + jg L(71) ALIGN_2 -.L70: +L(70): faddp %st, %st(1) faddp %st, %st(1) faddp %st, %st(1) @@ -392,15 +392,15 @@ FST (Y1) addq INCY, Y1 decq J - jg .L61 + jg L(61) -.L60: +L(60): addq PLDA_M, A addq $P, IS cmpq N, IS - jl .L32 + jl L(32) -.L79: +L(79): EMMS movq 0(%rsp), %rbx diff --git a/kernel/x86_64/qgemv_t.S b/kernel/x86_64/qgemv_t.S index 32372ff15d..969f8163bf 100644 --- a/kernel/x86_64/qgemv_t.S +++ b/kernel/x86_64/qgemv_t.S @@ -99,9 +99,9 @@ movq $0, IS test M, M - jle .L79 # goto END + jle L(79) # goto END test N, N - jle .L79 # goto END + jle L(79) # goto END movq N, %rax imulq LDA, %rax @@ -112,7 +112,7 @@ salq $BASE_SHIFT, LDA ALIGN_2 -.L32: +L(32): movq $P, %rax movq M, MIN_M subq IS , MIN_M @@ -126,17 +126,17 @@ movq X1, XP cmpq $SIZE, INCX - je .L34 + je L(34) movq BUFFER, X1 movq X1, XP movq MIN_M, I sarq $2, I - jle .L35 + jle L(35) ALIGN_3 -.L36: +L(36): FLD (X) addq INCX, X FST 0 * SIZE(X1) @@ -155,35 +155,35 @@ addq $4 * SIZE, X1 decq I - jg .L36 + jg L(36) ALIGN_3 -.L35: +L(35): movq MIN_M, I andq $3,I - jle .L34 + jle L(34) ALIGN_2 -.L42: +L(42): FLD (X) addq INCX, X FST (X1) addq $SIZE, X1 decq I - jg .L42 + jg L(42) ALIGN_3 /* Main Routine */ -.L34: +L(34): movq Y, Y1 # coffset = y movq N, J sarq $2, J - jle .L47 + jle L(47) ALIGN_3 -.L48: +L(48): movq A, A1 leaq (A, LDA), A2 leaq (A, LDA, 4), A @@ -198,10 +198,10 @@ movq MIN_M, I sarq $2,I - jle .L51 + jle L(51) ALIGN_3 -.L80: +L(80): FLD 0 * SIZE(A1) # at = *(a_offset + 0 * lda) fmul %st(1),%st # at1 *= bt1 @@ -278,16 +278,16 @@ addq $4 * SIZE, X1 decq I - jg .L80 + jg L(80) ALIGN_3 -.L51: +L(51): movq MIN_M, I andq $3, I - je .L81 + je L(81) ALIGN_3 -.L52: +L(52): FLD (A1) # at = *(a_offset + 0 * lda) fmul %st(1),%st # at1 *= bt1 faddp %st,%st(2) # ct1 += at1 @@ -309,10 +309,10 @@ addq $SIZE, A2 addq $SIZE, X1 decq I - jg .L52 + jg L(52) ALIGN_3 -.L81: +L(81): ffreep %st(0) fxch %st(4) @@ -343,16 +343,16 @@ addq INCY, Y1 decq J - jg .L48 + jg L(48) ALIGN_3 -.L47: +L(47): movq N, J andq $3, J - jle .L60 + jle L(60) ALIGN_2 -.L61: +L(61): movq A, A1 # a_offset = a fldz # ct1 = ZERO fldz # ct1 = ZERO @@ -365,10 +365,10 @@ movq MIN_M, I sarq $3,I - jle .L64 + jle L(64) ALIGN_3 -.L65: +L(65): FLD 0 * SIZE(X1) FLD 0 * SIZE(A1) fmulp %st, %st(1) @@ -413,16 +413,16 @@ addq $8 * SIZE, A1 decq I - jg .L65 + jg L(65) ALIGN_3 -.L64: +L(64): movq MIN_M, I andq $7, I - jle .L70 + jle L(70) ALIGN_3 -.L71: +L(71): FLD (X1) FLD (A1) fmulp %st, %st(1) @@ -431,10 +431,10 @@ addq $SIZE, X1 addq $SIZE, A1 decq I - jg .L71 + jg L(71) ALIGN_3 -.L70: +L(70): faddp %st, %st(1) faddp %st, %st(1) faddp %st, %st(1) @@ -445,18 +445,18 @@ FST (Y1) addq INCY, Y1 decq J - jg .L61 + jg L(61) ALIGN_3 -.L60: +L(60): addq NLDA, A addq $P, IS cmpq M, IS - jl .L32 + jl L(32) ALIGN_3 -.L79: +L(79): EMMS movq 0(%rsp), %rbx diff --git a/kernel/x86_64/qtrsm_kernel_LN_2x2.S b/kernel/x86_64/qtrsm_kernel_LN_2x2.S index 0a545faf87..591f3ad457 100644 --- a/kernel/x86_64/qtrsm_kernel_LN_2x2.S +++ b/kernel/x86_64/qtrsm_kernel_LN_2x2.S @@ -133,10 +133,10 @@ movq N, %rax sarq $1, %rax movq %rax, J - je .L30 + je L(30) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -173,10 +173,10 @@ movq M, %rax andq $1, %rax - je .L20 + je L(20) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -203,10 +203,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -253,10 +253,10 @@ addq $8 * SIZE,BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -264,10 +264,10 @@ subq KK, %rax #endif and $3, %rax - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -282,10 +282,10 @@ addq $2 * SIZE,BO decq %rax - jne .L26 + jne L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -393,13 +393,13 @@ #endif ALIGN_4 -.L20: +L(20): movq M, I sarq $1, I - je .L29 + je L(29) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -436,10 +436,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -520,10 +520,10 @@ addq $8 * SIZE,BO decq %rax - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -531,10 +531,10 @@ subq KK, %rax #endif and $3, %rax - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -557,10 +557,10 @@ addq $2 * SIZE,BO decq %rax - jne .L16 + jne L(16) ALIGN_4 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -735,10 +735,10 @@ #endif decq I - jne .L11 + jne L(11) ALIGN_4 -.L29: +L(29): #ifdef LN movq K, %rax salq $BASE_SHIFT, %rax @@ -758,13 +758,13 @@ #endif decq J - jne .L01 + jne L(01) ALIGN_4 -.L30: +L(30): movq N, %rax testq $1, %rax - je .L999 + je L(999) #if defined(LT) || defined(RN) movq A, AO @@ -800,10 +800,10 @@ movq M, %rax andq $1, %rax - je .L40 + je L(40) ALIGN_4 -.L41: +L(41): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -829,10 +829,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -859,10 +859,10 @@ addq $4 * SIZE,BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -870,10 +870,10 @@ subq KK, %rax #endif and $3, %rax - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -884,10 +884,10 @@ addq $1 * SIZE,BO decq %rax - jne .L46 + jne L(46) ALIGN_4 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -972,13 +972,13 @@ #endif ALIGN_4 -.L40: +L(40): movq M, I sarq $1, I - je .L49 + je L(49) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -1011,10 +1011,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(BO) @@ -1057,10 +1057,10 @@ addq $4 * SIZE,BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1068,10 +1068,10 @@ subq KK, %rax #endif and $3, %rax - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): FLD -8 * SIZE(BO) FLD -8 * SIZE(AO) @@ -1086,10 +1086,10 @@ addq $1 * SIZE,BO decq %rax - jne .L36 + jne L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1202,10 +1202,10 @@ #endif decq I - jne .L31 + jne L(31) ALIGN_4 -.L49: +L(49): #ifdef LN movq K, %rax salq $BASE_SHIFT, %rax @@ -1225,7 +1225,7 @@ #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/qtrsm_kernel_LT_2x2.S b/kernel/x86_64/qtrsm_kernel_LT_2x2.S index 16063fbcdd..ab62f709c0 100644 --- a/kernel/x86_64/qtrsm_kernel_LT_2x2.S +++ b/kernel/x86_64/qtrsm_kernel_LT_2x2.S @@ -133,10 +133,10 @@ movq N, %rax sarq $1, %rax movq %rax, J - je .L30 + je L(30) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -173,10 +173,10 @@ movq M, I sarq $1, I - je .L20 + je L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -213,10 +213,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -297,10 +297,10 @@ addq $8 * SIZE,BO decq %rax - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -308,10 +308,10 @@ subq KK, %rax #endif and $3, %rax - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -334,10 +334,10 @@ addq $2 * SIZE,BO decq %rax - jne .L16 + jne L(16) ALIGN_4 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -512,16 +512,16 @@ #endif decq I - jne .L11 + jne L(11) ALIGN_4 -.L20: +L(20): movq M, %rax andq $1, %rax - je .L29 + je L(29) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -548,10 +548,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -598,10 +598,10 @@ addq $8 * SIZE,BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -609,10 +609,10 @@ subq KK, %rax #endif and $3, %rax - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -627,10 +627,10 @@ addq $2 * SIZE,BO decq %rax - jne .L26 + jne L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -738,7 +738,7 @@ #endif ALIGN_4 -.L29: +L(29): #ifdef LN movq K, %rax salq $BASE_SHIFT, %rax @@ -758,13 +758,13 @@ #endif decq J - jne .L01 + jne L(01) ALIGN_4 -.L30: +L(30): movq N, %rax testq $1, %rax - je .L999 + je L(999) #if defined(LT) || defined(RN) movq A, AO @@ -800,10 +800,10 @@ movq M, I sarq $1, I - je .L40 + je L(40) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -836,10 +836,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(BO) @@ -882,10 +882,10 @@ addq $4 * SIZE,BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -893,10 +893,10 @@ subq KK, %rax #endif and $3, %rax - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): FLD -8 * SIZE(BO) FLD -8 * SIZE(AO) @@ -911,10 +911,10 @@ addq $1 * SIZE,BO decq %rax - jne .L36 + jne L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1027,16 +1027,16 @@ #endif decq I - jne .L31 + jne L(31) ALIGN_4 -.L40: +L(40): movq M, %rax andq $1, %rax - je .L49 + je L(49) ALIGN_4 -.L41: +L(41): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1062,10 +1062,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -1092,10 +1092,10 @@ addq $4 * SIZE,BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1103,10 +1103,10 @@ subq KK, %rax #endif and $3, %rax - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -1117,10 +1117,10 @@ addq $1 * SIZE,BO decq %rax - jne .L46 + jne L(46) ALIGN_4 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1205,7 +1205,7 @@ #endif ALIGN_4 -.L49: +L(49): #ifdef LN movq K, %rax salq $BASE_SHIFT, %rax @@ -1225,7 +1225,7 @@ #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/qtrsm_kernel_RT_2x2.S b/kernel/x86_64/qtrsm_kernel_RT_2x2.S index 4c94ac02cf..571425f25f 100644 --- a/kernel/x86_64/qtrsm_kernel_RT_2x2.S +++ b/kernel/x86_64/qtrsm_kernel_RT_2x2.S @@ -132,7 +132,7 @@ movq N, %rax testq $1, %rax - je .L30 + je L(30) #if defined(LT) || defined(RN) movq A, AO @@ -168,10 +168,10 @@ movq M, I sarq $1, I - je .L40 + je L(40) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -204,10 +204,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(BO) @@ -250,10 +250,10 @@ addq $4 * SIZE,BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -261,10 +261,10 @@ subq KK, %rax #endif and $3, %rax - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): FLD -8 * SIZE(BO) FLD -8 * SIZE(AO) @@ -279,10 +279,10 @@ addq $1 * SIZE,BO decq %rax - jne .L36 + jne L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -395,16 +395,16 @@ #endif decq I - jne .L31 + jne L(31) ALIGN_4 -.L40: +L(40): movq M, %rax andq $1, %rax - je .L49 + je L(49) ALIGN_4 -.L41: +L(41): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -430,10 +430,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -460,10 +460,10 @@ addq $4 * SIZE,BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -471,10 +471,10 @@ subq KK, %rax #endif and $3, %rax - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -485,10 +485,10 @@ addq $1 * SIZE,BO decq %rax - jne .L46 + jne L(46) ALIGN_4 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -573,7 +573,7 @@ #endif ALIGN_4 -.L49: +L(49): #ifdef LN movq K, %rax salq $BASE_SHIFT, %rax @@ -593,14 +593,14 @@ #endif ALIGN_4 -.L30: +L(30): movq N, %rax sarq $1, %rax movq %rax, J - je .L999 + je L(999) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -637,10 +637,10 @@ movq M, I sarq $1, I - je .L20 + je L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -677,10 +677,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -761,10 +761,10 @@ addq $8 * SIZE,BO decq %rax - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -772,10 +772,10 @@ subq KK, %rax #endif and $3, %rax - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -798,10 +798,10 @@ addq $2 * SIZE,BO decq %rax - jne .L16 + jne L(16) ALIGN_4 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -976,16 +976,16 @@ #endif decq I - jne .L11 + jne L(11) ALIGN_4 -.L20: +L(20): movq M, %rax andq $1, %rax - je .L29 + je L(29) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1012,10 +1012,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -1062,10 +1062,10 @@ addq $8 * SIZE,BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1073,10 +1073,10 @@ subq KK, %rax #endif and $3, %rax - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -1091,10 +1091,10 @@ addq $2 * SIZE,BO decq %rax - jne .L26 + jne L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1202,7 +1202,7 @@ #endif ALIGN_4 -.L29: +L(29): #ifdef LN movq K, %rax salq $BASE_SHIFT, %rax @@ -1222,10 +1222,10 @@ #endif decq J - jne .L01 + jne L(01) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/rot.S b/kernel/x86_64/rot.S index 6b2ad7fd87..61167fa998 100644 --- a/kernel/x86_64/rot.S +++ b/kernel/x86_64/rot.S @@ -69,19 +69,19 @@ salq $BASE_SHIFT, INCY testq N, N - jle .L999 + jle L(999) cmpq $SIZE, INCX - jne .L50 + jne L(50) cmpq $SIZE, INCY - jne .L50 + jne L(50) movq N, I sarq $2, I - jle .L15 + jle L(15) ALIGN_4 -.L10: +L(10): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -170,16 +170,16 @@ addq $4 * SIZE, Y decq I - jg .L10 + jg L(10) ALIGN_4 -.L15: +L(15): movq N, I andq $3, I - jle .L999 + jle L(999) ALIGN_4 -.L16: +L(16): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -203,17 +203,17 @@ addq $SIZE, Y decq I - jg .L16 - jmp .L999 + jg L(16) + jmp L(999) ALIGN_4 -.L50: +L(50): movq N, I sarq $2, I - jle .L55 + jle L(55) ALIGN_4 -.L51: +L(51): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -303,16 +303,16 @@ addq INCY, Y decq I - jg .L51 + jg L(51) ALIGN_4 -.L55: +L(55): movq N, I andq $3, I - jle .L999 + jle L(999) ALIGN_4 -.L56: +L(56): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -336,11 +336,11 @@ addq INCY, Y decq I - jg .L56 + jg L(56) ALIGN_4 -.L999: +L(999): ffreep %st ffreep %st ret diff --git a/kernel/x86_64/rot_sse.S b/kernel/x86_64/rot_sse.S index 6e37292ed9..4d82fde56a 100644 --- a/kernel/x86_64/rot_sse.S +++ b/kernel/x86_64/rot_sse.S @@ -72,15 +72,15 @@ pshufd $0x0, %xmm1, S cmpq $0, N - jle .L999 + jle L(999) cmpq $SIZE, INCX - jne .L50 + jne L(50) cmpq $SIZE, INCY - jne .L50 + jne L(50) testq $SIZE, X - je .L05 + je L(05) movss 0 * SIZE(Y), %xmm1 movss 0 * SIZE(X), %xmm0 @@ -103,14 +103,14 @@ addq $1 * SIZE, X addq $1 * SIZE, Y decq N - jle .L999 + jle L(999) -.L05: +L(05): testq $2 * SIZE, X - je .L10 + je L(10) cmpq $1, N - je .L17 + je L(17) movsd 0 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -133,16 +133,16 @@ addq $2 * SIZE, X addq $2 * SIZE, Y subq $2, N - jle .L999 + jle L(999) ALIGN_2 -.L10: +L(10): testq $3 * SIZE, Y - jne .L20 + jne L(20) movq N, %rax sarq $5, %rax - jle .L14 + jle L(14) movaps 0 * SIZE(Y), %xmm1 movaps 4 * SIZE(Y), %xmm3 @@ -155,10 +155,10 @@ movaps 12 * SIZE(X), %xmm10 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -291,10 +291,10 @@ addq $32 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movaps %xmm1, %xmm4 mulps S, %xmm1 movaps %xmm3, %xmm6 @@ -404,12 +404,12 @@ addq $32 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $31, N - jle .L999 + jle L(999) testq $16, N - jle .L15 + jle L(15) movaps 0 * SIZE(Y), %xmm1 movaps 0 * SIZE(X), %xmm0 @@ -475,9 +475,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $8, N - jle .L16 + jle L(16) movaps 0 * SIZE(Y), %xmm1 movaps 0 * SIZE(X), %xmm0 @@ -513,9 +513,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $4, N - jle .L17 + jle L(17) movaps 0 * SIZE(Y), %xmm1 movaps 0 * SIZE(X), %xmm0 @@ -539,9 +539,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $2, N - jle .L18 + jle L(18) movsd 0 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -565,9 +565,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L18: +L(18): testq $1, N - jle .L999 + jle L(999) movss 0 * SIZE(Y), %xmm1 movss 0 * SIZE(X), %xmm0 @@ -586,16 +586,16 @@ movss %xmm0, 0 * SIZE(X) movss %xmm2, 0 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L20: +L(20): movq N, %rax sarq $5, %rax - jle .L24 + jle L(24) ALIGN_3 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -751,15 +751,15 @@ addq $32 * SIZE, X addq $32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L24: +L(24): testq $31, N - jle .L999 + jle L(999) testq $16, N - jle .L25 + jle L(25) movsd 0 * SIZE(Y), %xmm1 movhps 2 * SIZE(Y), %xmm1 @@ -833,9 +833,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $8, N - jle .L26 + jle L(26) movsd 0 * SIZE(Y), %xmm1 movhps 2 * SIZE(Y), %xmm1 @@ -876,9 +876,9 @@ ALIGN_3 -.L26: +L(26): testq $4, N - jle .L27 + jle L(27) movsd 0 * SIZE(Y), %xmm1 movhps 2 * SIZE(Y), %xmm1 @@ -904,9 +904,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $2, N - jle .L28 + jle L(28) movsd 0 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -930,9 +930,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L28: +L(28): testq $1, N - jle .L999 + jle L(999) movss 0 * SIZE(Y), %xmm1 movss 0 * SIZE(X), %xmm0 @@ -951,17 +951,17 @@ movss %xmm0, 0 * SIZE(X) movss %xmm2, 0 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L50: +L(50): movq N, %rax sarq $2, %rax - jle .L55 + jle L(55) ALIGN_3 -.L53: +L(53): movss (Y), %xmm1 movss (X), %xmm0 @@ -1047,16 +1047,16 @@ addq INCY, Y decq %rax - jg .L53 + jg L(53) ALIGN_3 -.L55: +L(55): movq N, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_3 -.L56: +L(56): movss (Y), %xmm1 movss (X), %xmm0 @@ -1079,10 +1079,10 @@ addq INCY, Y decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/rot_sse2.S b/kernel/x86_64/rot_sse2.S index aa5852cac1..e41eebfc74 100644 --- a/kernel/x86_64/rot_sse2.S +++ b/kernel/x86_64/rot_sse2.S @@ -72,15 +72,15 @@ pshufd $0x44, %xmm1, S cmpq $0, N - jle .L999 + jle L(999) cmpq $SIZE, INCX - jne .L50 + jne L(50) cmpq $SIZE, INCY - jne .L50 + jne L(50) testq $SIZE, X - je .L10 + je L(10) movsd 0 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -103,16 +103,16 @@ addq $1 * SIZE, X addq $1 * SIZE, Y decq N - jle .L999 + jle L(999) ALIGN_2 -.L10: +L(10): testq $SIZE, Y - jne .L20 + jne L(20) movq N, %rax sarq $4, %rax - jle .L14 + jle L(14) movaps 0 * SIZE(Y), %xmm1 movaps 2 * SIZE(Y), %xmm3 @@ -125,10 +125,10 @@ movaps 6 * SIZE(X), %xmm10 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -261,10 +261,10 @@ addq $16 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movaps %xmm1, %xmm4 mulpd S, %xmm1 movaps %xmm3, %xmm6 @@ -375,12 +375,12 @@ ALIGN_3 -.L14: +L(14): testq $15, N - jle .L999 + jle L(999) testq $8, N - jle .L15 + jle L(15) movaps 0 * SIZE(Y), %xmm1 movaps 0 * SIZE(X), %xmm0 @@ -446,9 +446,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $4, N - jle .L16 + jle L(16) movaps 0 * SIZE(Y), %xmm1 movaps 0 * SIZE(X), %xmm0 @@ -484,9 +484,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $2, N - jle .L17 + jle L(17) movaps 0 * SIZE(Y), %xmm1 movaps 0 * SIZE(X), %xmm0 @@ -510,9 +510,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $1, N - jle .L999 + jle L(999) movsd 0 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -531,18 +531,18 @@ movsd %xmm0, 0 * SIZE(X) movsd %xmm2, 0 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L20: +L(20): movaps -1 * SIZE(Y), %xmm1 movq N, %rax sarq $4, %rax - jle .L24 + jle L(24) ALIGN_3 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -703,15 +703,15 @@ addq $16 * SIZE, X addq $16 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L24: +L(24): testq $15, N - jle .L999 + jle L(999) testq $8, N - jle .L25 + jle L(25) movaps 1 * SIZE(Y), %xmm3 movaps 3 * SIZE(Y), %xmm8 @@ -787,9 +787,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $4, N - jle .L26 + jle L(26) movaps 1 * SIZE(Y), %xmm3 movaps 3 * SIZE(Y), %xmm8 @@ -831,9 +831,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $2, N - jle .L27 + jle L(27) movaps 1 * SIZE(Y), %xmm4 movaps 0 * SIZE(X), %xmm0 @@ -860,9 +860,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $1, N - jle .L999 + jle L(999) unpckhpd %xmm1, %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -881,21 +881,21 @@ movsd %xmm0, 0 * SIZE(X) movsd %xmm2, 0 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L50: +L(50): movq N, %rax cmpq $0, INCX - je .L56 + je L(56) cmpq $0, INCY - je .L56 + je L(56) sarq $2, %rax - jle .L55 + jle L(55) ALIGN_3 -.L53: +L(53): movsd (Y), %xmm1 movhps (Y, INCY), %xmm1 movsd (X), %xmm0 @@ -947,16 +947,16 @@ leaq (Y, INCY, 2), Y decq %rax - jg .L53 + jg L(53) ALIGN_3 -.L55: +L(55): movq N, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_3 -.L56: +L(56): movsd (Y), %xmm1 movsd (X), %xmm0 @@ -979,10 +979,10 @@ addq INCY, Y decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/scal.S b/kernel/x86_64/scal.S index 1f8e4d4447..8956802152 100644 --- a/kernel/x86_64/scal.S +++ b/kernel/x86_64/scal.S @@ -55,18 +55,18 @@ ftst fnstsw %ax andb $68, %ah - je .L300 + je L(300) /* Alpha == ZERO */ cmpq $1, INCX - jne .L104 + jne L(104) movq M, I sarq $3, I - jle .L102 + jle L(102) ALIGN_4 -.L101: +L(101): fld %st FST 0 * SIZE(X) fld %st @@ -86,34 +86,34 @@ addq $8 * SIZE, X decq I - jg .L101 + jg L(101) ALIGN_4 -.L102: +L(102): movq M, I andq $7, I - jle .L999 + jle L(999) ALIGN_4 -.L103: +L(103): fld %st FST 0 * SIZE(X) addq $SIZE, X decq I - jg .L103 - jmp .L999 + jg L(103) + jmp L(999) ALIGN_4 -.L104: +L(104): salq $BASE_SHIFT, INCX movq M, I sarq $3, I - jle .L106 + jle L(106) ALIGN_4 -.L105: +L(105): fld %st FST 0 * SIZE(X) addq INCX, X @@ -140,36 +140,36 @@ addq INCX, X decq I - jg .L105 + jg L(105) ALIGN_4 -.L106: +L(106): movq M, I andq $7, I - jle .L999 + jle L(999) ALIGN_4 -.L107: +L(107): fld %st FST 0 * SIZE(X) addq INCX, X decq I - jg .L107 - jmp .L999 + jg L(107) + jmp L(999) ALIGN_4 /* Alpha != ZERO */ -.L300: +L(300): cmpq $1,INCX - jne .L304 + jne L(304) movq M, I sarq $3, I - jle .L302 + jle L(302) ALIGN_4 -.L301: +L(301): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -208,34 +208,34 @@ addq $8 * SIZE, X decq I - jg .L301 + jg L(301) ALIGN_4 -.L302: +L(302): movq M, I andq $7, I - jle .L999 + jle L(999) ALIGN_4 -.L303: +L(303): FLD 0 * SIZE(X) fmul %st(1), %st FST 0 * SIZE(X) addq $SIZE, X decq I - jg .L303 - jmp .L999 + jg L(303) + jmp L(999) ALIGN_4 -.L304: +L(304): salq $BASE_SHIFT, INCX movq M, I sarq $3, I - jle .L306 + jle L(306) ALIGN_4 -.L305: +L(305): FLD 0 * SIZE(X) fmul %st(1), %st FST 0 * SIZE(X) @@ -277,25 +277,25 @@ addq INCX, X decq I - jg .L305 + jg L(305) ALIGN_4 -.L306: +L(306): movq M, I andq $7, I - jle .L999 + jle L(999) ALIGN_4 -.L307: +L(307): FLD 0 * SIZE(X) fmul %st(1), %st FST 0 * SIZE(X) addq INCX, X decq I - jg .L307 + jg L(307) ALIGN_4 -.L999: +L(999): ffreep %st(0) ret diff --git a/kernel/x86_64/scal_atom.S b/kernel/x86_64/scal_atom.S index 284ea4515a..a1c26cafb7 100644 --- a/kernel/x86_64/scal_atom.S +++ b/kernel/x86_64/scal_atom.S @@ -69,27 +69,27 @@ SAVEREGISTERS testq M, M - jle .L999 + jle L(999) pxor %xmm1, %xmm1 lea (, INCX, SIZE), INCX comisd %xmm0, %xmm1 - jne .L100 - jp .L100 + jne L(100) + jp L(100) cmpq $1, %r9 - je .L100 + je L(100) /* Alpha == ZERO */ cmpq $SIZE, INCX - jne .L50 + jne L(50) movq M, I sarq $3, I - jle .L12 + jle L(12) ALIGN_4 -.L11: +L(11): movsd %xmm1, 0 * SIZE(X) movsd %xmm1, 1 * SIZE(X) movsd %xmm1, 2 * SIZE(X) @@ -102,12 +102,12 @@ addq $8 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_4 -.L12: +L(12): testq $4, M - je .L14 + je L(14) movsd %xmm1, 0 * SIZE(X) movsd %xmm1, 1 * SIZE(X) @@ -117,9 +117,9 @@ addq $4 * SIZE, X ALIGN_3 -.L14: +L(14): testq $2, M - je .L15 + je L(15) movsd %xmm1, 0 * SIZE(X) movsd %xmm1, 1 * SIZE(X) @@ -127,21 +127,21 @@ addq $2 * SIZE, X ALIGN_3 -.L15: +L(15): testq $1, M - je .L999 + je L(999) movsd %xmm1, 0 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_4 -.L50: +L(50): movq M, I sarq $3, I - jle .L52 + jle L(52) ALIGN_4 -.L51: +L(51): movsd %xmm1, 0 * SIZE(X) addq INCX, X movsd %xmm1, 0 * SIZE(X) @@ -160,15 +160,15 @@ addq INCX, X decq I - jg .L51 + jg L(51) ALIGN_4 -.L52: +L(52): testq $7, M - je .L999 + je L(999) testq $4, M - je .L53 + je L(53) movsd %xmm1, 0 * SIZE(X) addq INCX, X @@ -180,9 +180,9 @@ addq INCX, X ALIGN_3 -.L53: +L(53): testq $2, M - je .L54 + je L(54) movsd %xmm1, 0 * SIZE(X) addq INCX, X @@ -190,25 +190,25 @@ addq INCX, X ALIGN_3 -.L54: +L(54): testq $1, M - je .L999 + je L(999) movsd %xmm1, 0 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_4 /* Alpha != ZERO */ -.L100: +L(100): cmpq $SIZE, INCX - jne .L150 + jne L(150) unpcklpd %xmm0, %xmm0 movq M, I sarq $3, I - jle .L113 + jle L(113) movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 @@ -225,10 +225,10 @@ mulsd %xmm0, %xmm4 decq I - jle .L112 + jle L(112) ALIGN_4 -.L111: +L(111): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -263,10 +263,10 @@ addq $8 * SIZE, X decq I - jg .L111 + jg L(111) ALIGN_4 -.L112: +L(112): movsd %xmm1, 0 * SIZE(X) mulsd %xmm0, %xmm5 movsd %xmm2, 1 * SIZE(X) @@ -283,9 +283,9 @@ addq $8 * SIZE, X ALIGN_3 -.L113: +L(113): testq $4, M - je .L115 + je L(115) movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 @@ -305,9 +305,9 @@ addq $4 * SIZE, X ALIGN_3 -.L115: +L(115): testq $2, M - je .L116 + je L(116) movsd 0 * SIZE(X), %xmm1 movsd 1 * SIZE(X), %xmm2 @@ -321,26 +321,26 @@ addq $2 * SIZE, X ALIGN_3 -.L116: +L(116): testq $1, M - je .L999 + je L(999) movsd 0 * SIZE(X), %xmm1 mulsd %xmm0, %xmm1 movsd %xmm1, 0 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_3 /* incx != 1 */ -.L150: +L(150): movq X, XX movq M, I # rcx = n sarq $3, I # (n >> 3) - jle .L152 + jle L(152) ALIGN_4 -.L151: +L(151): movsd 0 * SIZE(X), %xmm1 addq INCX, X movsd 0 * SIZE(X), %xmm2 @@ -383,15 +383,15 @@ movsd %xmm8, 0 * SIZE(XX) addq INCX, XX decq I - jg .L151 + jg L(151) ALIGN_4 -.L152: +L(152): testq $7, M - je .L999 + je L(999) testq $4, M - je .L153 + je L(153) movsd 0 * SIZE(X), %xmm1 addq INCX, X @@ -416,9 +416,9 @@ addq INCX, XX ALIGN_3 -.L153: +L(153): testq $2, M - je .L154 + je L(154) movsd 0 * SIZE(X), %xmm1 addq INCX, X @@ -433,16 +433,16 @@ addq INCX, XX ALIGN_3 -.L154: +L(154): testq $1, M - je .L999 + je L(999) movsd 0 * SIZE(X), %xmm1 mulsd %xmm0, %xmm1 movsd %xmm1, 0 * SIZE(X) ALIGN_4 -.L999: +L(999): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/scal_sse.S b/kernel/x86_64/scal_sse.S index 88ef4a3eda..2c3d910dbf 100644 --- a/kernel/x86_64/scal_sse.S +++ b/kernel/x86_64/scal_sse.S @@ -69,7 +69,7 @@ SAVEREGISTERS testq M, M - jle .L999 + jle L(999) lea (, INCX, SIZE), INCX @@ -77,44 +77,44 @@ comiss %xmm0, %xmm1 shufps $0, %xmm0, %xmm0 - jne .L100 # Alpha != ZERO + jne L(100) # Alpha != ZERO cmpq $1, %r9 - je .L100 + je L(100) /* Alpha == ZERO */ cmpq $SIZE, INCX - jne .L50 + jne L(50) /* INCX == 1 */ cmpq $3, M - jle .L14 + jle L(14) testq $4, X # aligned for double word? - je .L05 + je L(05) movss %xmm1, 0 * SIZE(X) addq $SIZE, X decq M - jle .L999 + jle L(999) ALIGN_3 -.L05: +L(05): testq $8, X # aligned for quad word? - je .L06 + je L(06) movsd %xmm1, 0 * SIZE(X) addq $2 * SIZE, X subq $2, M - jle .L999 + jle L(999) ALIGN_3 -.L06: +L(06): movq M, I sarq $4, I - jle .L12 + jle L(12) ALIGN_4 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -125,52 +125,52 @@ movaps %xmm1, 12 * SIZE(X) addq $16 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_4 -.L12: +L(12): testq $15, M - je .L999 + je L(999) testq $8, M - je .L13 + je L(13) movaps %xmm1, 0 * SIZE(X) movaps %xmm1, 4 * SIZE(X) addq $8 * SIZE, X ALIGN_3 -.L13: +L(13): testq $4, M - je .L14 + je L(14) movaps %xmm1, 0 * SIZE(X) addq $4 * SIZE, X ALIGN_3 -.L14: +L(14): testq $2, M - je .L15 + je L(15) movsd %xmm1, 0 * SIZE(X) addq $2 * SIZE, X ALIGN_3 -.L15: +L(15): testq $1, M - je .L999 + je L(999) movss %xmm1, 0 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_4 /* incx != 1 */ -.L50: +L(50): movq M, I # rcx = n sarq $3, I # (n >> 3) - jle .L52 + jle L(52) ALIGN_4 -.L51: +L(51): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -193,15 +193,15 @@ addq INCX, X decq I - jg .L51 + jg L(51) ALIGN_4 -.L52: +L(52): testq $7, M - je .L999 + je L(999) testq $4, M - je .L53 + je L(53) movss %xmm1, (X) addq INCX, X @@ -213,9 +213,9 @@ addq INCX, X ALIGN_3 -.L53: +L(53): testq $2, M - je .L54 + je L(54) movss %xmm1, (X) addq INCX, X @@ -223,52 +223,52 @@ addq INCX, X ALIGN_3 -.L54: +L(54): testq $1, M - je .L999 + je L(999) movss %xmm1, (X) - jmp .L999 + jmp L(999) ALIGN_4 /* Alpha != ZERO */ -.L100: +L(100): cmpq $SIZE, INCX - jne .L150 + jne L(150) subq $-32 * SIZE, X cmpq $3, M - jle .L116 + jle L(116) testq $SIZE, X - je .L105 + je L(105) movss -32 * SIZE(X), %xmm1 mulss %xmm0, %xmm1 movss %xmm1, -32 * SIZE(X) addq $SIZE, X decq M - jle .L999 + jle L(999) ALIGN_3 -.L105: +L(105): testq $2 * SIZE, X - je .L110 + je L(110) movsd -32 * SIZE(X), %xmm1 mulps %xmm0, %xmm1 movsd %xmm1, -32 * SIZE(X) addq $2 * SIZE, X subq $2, M - jle .L999 + jle L(999) ALIGN_3 -.L110: +L(110): movq M, I sarq $5, I - jle .L113 + jle L(113) #if defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) @@ -290,10 +290,10 @@ mulps -4 * SIZE(X), %xmm8 decq I - jle .L112 + jle L(112) ALIGN_4 -.L111: +L(111): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -332,10 +332,10 @@ subq $-32 * SIZE, X decq I - jg .L111 + jg L(111) ALIGN_4 -.L112: +L(112): movaps %xmm1, -32 * SIZE(X) movaps %xmm2, -28 * SIZE(X) movaps %xmm3, -24 * SIZE(X) @@ -357,10 +357,10 @@ movaps -8 * SIZE(X), %xmm7 movaps -4 * SIZE(X), %xmm8 decq I - jle .L112 + jle L(112) ALIGN_4 -.L111: +L(111): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -397,10 +397,10 @@ subq $-32 * SIZE, X decq I - jg .L111 + jg L(111) ALIGN_4 -.L112: +L(112): mulps %xmm0, %xmm1 movaps %xmm1, -32 * SIZE(X) mulps %xmm0, %xmm2 @@ -424,12 +424,12 @@ subq $-32 * SIZE, X ALIGN_3 -.L113: +L(113): testq $31, M - je .L999 + je L(999) testq $16, M - je .L114 + je L(114) movaps -32 * SIZE(X), %xmm1 movaps -28 * SIZE(X), %xmm3 @@ -448,9 +448,9 @@ addq $16 * SIZE, X ALIGN_3 -.L114: +L(114): testq $8, M - je .L115 + je L(115) movaps -32 * SIZE(X), %xmm1 movaps -28 * SIZE(X), %xmm3 @@ -462,9 +462,9 @@ addq $8 * SIZE, X ALIGN_3 -.L115: +L(115): testq $4, M - je .L116 + je L(116) movaps -32 * SIZE(X), %xmm1 mulps %xmm0, %xmm1 @@ -472,9 +472,9 @@ addq $4 * SIZE, X ALIGN_3 -.L116: +L(116): testq $2, M - je .L117 + je L(117) movsd -32 * SIZE(X), %xmm1 mulps %xmm0, %xmm1 @@ -482,26 +482,26 @@ addq $2 * SIZE, X ALIGN_3 -.L117: +L(117): testq $1, M - je .L999 + je L(999) movss -32 * SIZE(X), %xmm1 mulss %xmm0, %xmm1 movss %xmm1, -32 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_3 /* incx != 1 */ -.L150: +L(150): movq X, XX movq M, I # rcx = n sarq $3, I # (n >> 3) - jle .L152 + jle L(152) ALIGN_4 -.L151: +L(151): movss (X), %xmm1 addq INCX, X movss (X), %xmm2 @@ -545,15 +545,15 @@ movss %xmm8, (XX) addq INCX, XX decq I - jg .L151 + jg L(151) ALIGN_4 -.L152: +L(152): testq $7, M - je .L999 + je L(999) testq $4, M - je .L153 + je L(153) movss (X), %xmm1 addq INCX, X @@ -579,9 +579,9 @@ addq INCX, XX ALIGN_3 -.L153: +L(153): testq $2, M - je .L154 + je L(154) movss (X), %xmm1 addq INCX, X @@ -597,16 +597,16 @@ addq INCX, XX ALIGN_3 -.L154: +L(154): testq $1, M - je .L999 + je L(999) movss (X), %xmm1 mulss %xmm0, %xmm1 movss %xmm1, (X) ALIGN_4 -.L999: +L(999): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/scal_sse2.S b/kernel/x86_64/scal_sse2.S index 485e6ef462..6d5bb5b585 100644 --- a/kernel/x86_64/scal_sse2.S +++ b/kernel/x86_64/scal_sse2.S @@ -70,39 +70,39 @@ SAVEREGISTERS testq M, M - jle .L999 + jle L(999) leaq (, INCX, SIZE), INCX xorps %xmm1, %xmm1 comisd %xmm0, %xmm1 - jne .L100 # Alpha != ZERO - jp .L100 # For Alpha = NaN + jne L(100) # Alpha != ZERO + jp L(100) # For Alpha = NaN cmpq $1, FLAG - je .L100 # disable the Alpha=zero path as it does not handle x=inf or nan + je L(100) # disable the Alpha=zero path as it does not handle x=inf or nan /* Alpha == ZERO */ cmpq $SIZE, INCX - jne .L50 + jne L(50) /* INCX == 1 */ testq $15, X # aligned for quad word? - je .L05 + je L(05) movsd %xmm1, 0 * SIZE(X) addq $SIZE, X decq M - jle .L999 + jle L(999) ALIGN_3 -.L05: +L(05): /* Aligned Mode */ movq M, I # rcx = n sarq $4, I - jle .L12 + jle L(12) ALIGN_4 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -123,14 +123,14 @@ addq $16 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_4 -.L12: +L(12): testq $15, M - je .L999 + je L(999) testq $8, M - je .L13 + je L(13) movaps %xmm1, 0 * SIZE(X) movaps %xmm1, 2 * SIZE(X) @@ -139,38 +139,38 @@ addq $8 * SIZE, X ALIGN_3 -.L13: +L(13): testq $4, M - je .L14 + je L(14) movaps %xmm1, 0 * SIZE(X) movaps %xmm1, 2 * SIZE(X) addq $4 * SIZE, X ALIGN_3 -.L14: +L(14): testq $2, M - je .L15 + je L(15) movaps %xmm1, 0 * SIZE(X) addq $2 * SIZE, X ALIGN_3 -.L15: +L(15): testq $1, M - je .L999 + je L(999) movsd %xmm1, 0 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_4 -.L50: +L(50): movq M, I sarq $3, I - jle .L52 + jle L(52) ALIGN_4 -.L51: +L(51): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -193,15 +193,15 @@ addq INCX, X decq I - jg .L51 + jg L(51) ALIGN_4 -.L52: +L(52): testq $7, M - je .L999 + je L(999) testq $4, M - je .L53 + je L(53) movsd %xmm1, (X) addq INCX, X @@ -213,9 +213,9 @@ addq INCX, X ALIGN_3 -.L53: +L(53): testq $2, M - je .L54 + je L(54) movsd %xmm1, (X) addq INCX, X @@ -223,38 +223,38 @@ addq INCX, X ALIGN_3 -.L54: +L(54): testq $1, M - je .L999 + je L(999) movsd %xmm1, (X) - jmp .L999 + jmp L(999) ALIGN_4 /* Alpha != ZERO */ -.L100: +L(100): unpcklpd %xmm0, %xmm0 cmpq $SIZE, INCX - jne .L150 + jne L(150) testq $SIZE, X - je .L105 + je L(105) movsd 0 * SIZE(X), %xmm1 mulsd %xmm0, %xmm1 movsd %xmm1, 0 * SIZE(X) addq $SIZE, X decq M - jle .L999 + jle L(999) ALIGN_3 -.L105: +L(105): subq $-16 * SIZE, X movq M, I # rcx = n sarq $4, I - jle .L113 + jle L(113) #if defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER) @@ -276,10 +276,10 @@ mulpd -2 * SIZE(X), %xmm8 decq I - jle .L112 + jle L(112) ALIGN_4 -.L111: +L(111): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -318,10 +318,10 @@ subq $-16 * SIZE, X decq I - jg .L111 + jg L(111) ALIGN_4 -.L112: +L(112): movaps %xmm1, -16 * SIZE(X) movaps %xmm2, -14 * SIZE(X) movaps %xmm3, -12 * SIZE(X) @@ -342,10 +342,10 @@ movaps -2 * SIZE(X), %xmm8 decq I - jle .L112 + jle L(112) ALIGN_4 -.L111: +L(111): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -384,10 +384,10 @@ subq $-16 * SIZE, X decq I - jg .L111 + jg L(111) ALIGN_4 -.L112: +L(112): mulpd %xmm0, %xmm1 movaps %xmm1, -16 * SIZE(X) mulpd %xmm0, %xmm2 @@ -410,12 +410,12 @@ subq $-16 * SIZE, X ALIGN_3 -.L113: +L(113): testq $15, M - je .L999 + je L(999) testq $8, M - je .L114 + je L(114) movaps -16 * SIZE(X), %xmm1 movaps -14 * SIZE(X), %xmm2 @@ -433,9 +433,9 @@ addq $8 * SIZE, X ALIGN_3 -.L114: +L(114): testq $4, M - je .L115 + je L(115) movaps -16 * SIZE(X), %xmm1 movaps -14 * SIZE(X), %xmm2 @@ -447,9 +447,9 @@ addq $4 * SIZE, X ALIGN_3 -.L115: +L(115): testq $2, M - je .L116 + je L(116) movaps -16 * SIZE(X), %xmm1 mulpd %xmm0, %xmm1 @@ -457,26 +457,26 @@ addq $2 * SIZE, X ALIGN_3 -.L116: +L(116): testq $1, M - je .L999 + je L(999) movsd -16 * SIZE(X), %xmm1 mulsd %xmm0, %xmm1 movsd %xmm1, -16 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_3 /* incx != 1 */ -.L150: +L(150): movq X, XX movq M, I # rcx = n sarq $3, I # (n >> 3) - jle .L152 + jle L(152) ALIGN_4 -.L151: +L(151): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(X) #endif @@ -523,15 +523,15 @@ movsd %xmm8, (XX) addq INCX, XX decq I - jg .L151 + jg L(151) ALIGN_4 -.L152: +L(152): testq $7, M - je .L999 + je L(999) testq $4, M - je .L153 + je L(153) movsd (X), %xmm1 addq INCX, X @@ -557,9 +557,9 @@ addq INCX, XX ALIGN_3 -.L153: +L(153): testq $2, M - je .L154 + je L(154) movsd (X), %xmm1 addq INCX, X @@ -575,16 +575,16 @@ addq INCX, XX ALIGN_3 -.L154: +L(154): testq $1, M - je .L999 + je L(999) movsd (X), %xmm1 mulsd %xmm0, %xmm1 movsd %xmm1, (X) ALIGN_4 -.L999: +L(999): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/sgemm_kernel_16x2_bulldozer.S b/kernel/x86_64/sgemm_kernel_16x2_bulldozer.S index b31a934f23..0c26923eb8 100644 --- a/kernel/x86_64/sgemm_kernel_16x2_bulldozer.S +++ b/kernel/x86_64/sgemm_kernel_16x2_bulldozer.S @@ -1033,13 +1033,13 @@ STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -1059,10 +1059,10 @@ movq Ndiv6, J cmpq $0, J - je .L2_0 + je L(2_0) ALIGN_4 -.L6_01: +L(6_01): // copy to sub buffer movq K, %rax salq $1,%rax // K * 2 ; read 2 values @@ -1071,10 +1071,10 @@ leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $3 , %rax // K / 8 - jz .L6_01a_2 + jz L(6_01a_2) ALIGN_4 -.L6_01a_1: +L(6_01a_1): prefetcht0 512(BO1) prefetcht0 512(BO2) @@ -1121,19 +1121,19 @@ addq $12*SIZE,BO decq %rax - jnz .L6_01a_1 + jnz L(6_01a_1) -.L6_01a_2: +L(6_01a_2): movq K, %rax andq $7, %rax // K % 8 - jz .L6_02c + jz L(6_02c) ALIGN_4 -.L6_02b: +L(6_02b): vmovsd 0 * SIZE(BO1), %xmm0 vmovss 0 * SIZE(BO2), %xmm2 @@ -1143,9 +1143,9 @@ addq $2*SIZE,BO2 addq $3*SIZE,BO decq %rax - jnz .L6_02b + jnz L(6_02b) -.L6_02c: +L(6_02c): movq K, %rax salq $1,%rax // K * 2 @@ -1154,10 +1154,10 @@ leaq BUFFER2, BO // second buffer to BO movq K, %rax sarq $3 , %rax // K / 8 - jz .L6_02c_2 + jz L(6_02c_2) ALIGN_4 -.L6_02c_1: +L(6_02c_1): prefetcht0 512(BO2) prefetchw 512(BO) @@ -1204,17 +1204,17 @@ addq $12*SIZE,BO decq %rax - jnz .L6_02c_1 + jnz L(6_02c_1) -.L6_02c_2: +L(6_02c_2): movq K, %rax andq $7, %rax // K % 8 - jz .L6_03c + jz L(6_03c) ALIGN_4 -.L6_03b: +L(6_03b): vmovss 1*SIZE(BO1), %xmm0 vmovsd 0*SIZE(BO2), %xmm1 @@ -1224,14 +1224,14 @@ addq $2*SIZE,BO2 addq $3*SIZE,BO decq %rax - jnz .L6_03b + jnz L(6_03b) -.L6_03c: +L(6_03c): movq BO2, B // next offset of B -.L6_10: +L(6_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -1242,11 +1242,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L6_20 + je L(6_20) ALIGN_4 -.L6_11: +L(6_11): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1255,7 +1255,7 @@ movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L6_16 + je L(6_16) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1266,7 +1266,7 @@ negq %rax ALIGN_4 -.L6_12: +L(6_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x3_1(xxx) @@ -1280,7 +1280,7 @@ KERNEL16x3_3(xxx) KERNEL16x3_4(xxx) - je .L6_16 + je L(6_16) KERNEL16x3_1(xxx) KERNEL16x3_2(xxx) @@ -1293,16 +1293,16 @@ KERNEL16x3_3(xxx) KERNEL16x3_4(xxx) - je .L6_16 + je L(6_16) - jmp .L6_12 + jmp L(6_12) ALIGN_4 -.L6_16: +L(6_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_19 + je L(6_19) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1314,16 +1314,16 @@ negq %rax ALIGN_4 -.L6_17: +L(6_17): KERNEL16x3_SUB(xxx) addq $3, BI addq $16, %rax - jl .L6_17 + jl L(6_17) ALIGN_4 -.L6_19: +L(6_19): vbroadcastss ALPHA, %xmm0 @@ -1361,25 +1361,25 @@ addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L6_11 + jg L(6_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L6_20: +L(6_20): // Test rest of M testq $15, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N testq $8, M - jz .L6_21pre + jz L(6_21pre) ALIGN_4 /**************************************************************************/ -.L6_20_1: +L(6_20_1): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1388,7 +1388,7 @@ movq K, %rax andq $-8, %rax - je .L6_20_6 + je L(6_20_6) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1399,7 +1399,7 @@ negq %rax ALIGN_4 -.L6_20_2: +L(6_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x3_1(xxx) @@ -1413,7 +1413,7 @@ KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L6_20_6 + je L(6_20_6) KERNEL8x3_1(xxx) KERNEL8x3_2(xxx) @@ -1426,16 +1426,16 @@ KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L6_20_6 + je L(6_20_6) - jmp .L6_20_2 + jmp L(6_20_2) ALIGN_4 -.L6_20_6: +L(6_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_20_9 + je L(6_20_9) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1447,16 +1447,16 @@ negq %rax ALIGN_4 -.L6_20_7: +L(6_20_7): KERNEL8x3_SUB(xxx) addq $3, BI addq $8, %rax - jl .L6_20_7 + jl L(6_20_7) ALIGN_4 -.L6_20_9: +L(6_20_9): vbroadcastss ALPHA, %xmm0 @@ -1487,13 +1487,13 @@ /**************************************************************************/ -.L6_21pre: +L(6_21pre): testq $4, M - jz .L6_30 + jz L(6_30) ALIGN_4 -.L6_21: +L(6_21): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1502,7 +1502,7 @@ movq K, %rax andq $-8, %rax - je .L6_26 + je L(6_26) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1513,7 +1513,7 @@ negq %rax ALIGN_4 -.L6_22: +L(6_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x3_1(xxx) @@ -1527,7 +1527,7 @@ KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L6_26 + je L(6_26) KERNEL4x3_1(xxx) KERNEL4x3_2(xxx) @@ -1540,16 +1540,16 @@ KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L6_26 + je L(6_26) - jmp .L6_22 + jmp L(6_22) ALIGN_4 -.L6_26: +L(6_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_29 + je L(6_29) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1561,16 +1561,16 @@ negq %rax ALIGN_4 -.L6_27: +L(6_27): KERNEL4x3_SUB(xxx) addq $3, BI addq $4, %rax - jl .L6_27 + jl L(6_27) ALIGN_4 -.L6_29: +L(6_29): vbroadcastss ALPHA, %xmm0 @@ -1587,13 +1587,13 @@ ALIGN_4 -.L6_30: +L(6_30): testq $2, M - jz .L6_40 + jz L(6_40) ALIGN_4 -.L6_31: +L(6_31): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1602,7 +1602,7 @@ movq K, %rax andq $-8, %rax - je .L6_36 + je L(6_36) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1613,7 +1613,7 @@ negq %rax ALIGN_4 -.L6_32: +L(6_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x3_1(xxx) @@ -1627,7 +1627,7 @@ KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L6_36 + je L(6_36) KERNEL2x3_1(xxx) KERNEL2x3_2(xxx) @@ -1640,16 +1640,16 @@ KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L6_36 + je L(6_36) - jmp .L6_32 + jmp L(6_32) ALIGN_4 -.L6_36: +L(6_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_39 + je L(6_39) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1661,16 +1661,16 @@ negq %rax ALIGN_4 -.L6_37: +L(6_37): KERNEL2x3_SUB(xxx) addq $3, BI addq $2, %rax - jl .L6_37 + jl L(6_37) ALIGN_4 -.L6_39: +L(6_39): vmovss ALPHA, %xmm0 @@ -1692,13 +1692,13 @@ addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L6_40: +L(6_40): testq $1, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N ALIGN_4 -.L6_41: +L(6_41): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1707,7 +1707,7 @@ movq K, %rax andq $-8, %rax - je .L6_46 + je L(6_46) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1717,7 +1717,7 @@ negq %rax ALIGN_4 -.L6_42: +L(6_42): KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -1729,7 +1729,7 @@ KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L6_46 + je L(6_46) KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -1741,16 +1741,16 @@ KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L6_46 + je L(6_46) - jmp .L6_42 + jmp L(6_42) ALIGN_4 -.L6_46: +L(6_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_49 + je L(6_49) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1761,16 +1761,16 @@ negq %rax ALIGN_4 -.L6_47: +L(6_47): KERNEL1x3_SUB(xxx) addq $3, BI addq $1, %rax - jl .L6_47 + jl L(6_47) ALIGN_4 -.L6_49: +L(6_49): vmovss ALPHA, %xmm0 @@ -1791,7 +1791,7 @@ /***************************************************************************************************************/ -.L7_10: +L(7_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -1802,11 +1802,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L7_20 + je L(7_20) ALIGN_4 -.L7_11: +L(7_11): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -1815,7 +1815,7 @@ movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L7_16 + je L(7_16) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1826,7 +1826,7 @@ negq %rax ALIGN_4 -.L7_12: +L(7_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x3_1(xxx) @@ -1840,7 +1840,7 @@ KERNEL16x3_3(xxx) KERNEL16x3_4(xxx) - je .L7_16 + je L(7_16) KERNEL16x3_1(xxx) KERNEL16x3_2(xxx) @@ -1853,16 +1853,16 @@ KERNEL16x3_3(xxx) KERNEL16x3_4(xxx) - je .L7_16 + je L(7_16) - jmp .L7_12 + jmp L(7_12) ALIGN_4 -.L7_16: +L(7_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_19 + je L(7_19) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1874,16 +1874,16 @@ negq %rax ALIGN_4 -.L7_17: +L(7_17): KERNEL16x3_SUB(xxx) addq $3, BI addq $16, %rax - jl .L7_17 + jl L(7_17) ALIGN_4 -.L7_19: +L(7_19): vbroadcastss ALPHA, %xmm0 @@ -1921,25 +1921,25 @@ addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L7_11 + jg L(7_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L7_20: +L(7_20): // Test rest of M testq $15, M - jz .L7_60 // to next 3 lines of N + jz L(7_60) // to next 3 lines of N testq $8, M - jz .L7_21pre + jz L(7_21pre) ALIGN_4 /**************************************************************************/ -.L7_20_1: +L(7_20_1): leaq BUFFER2, BO // first buffer to BO addq $6 * SIZE, BO @@ -1948,7 +1948,7 @@ movq K, %rax andq $-8, %rax - je .L7_20_6 + je L(7_20_6) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1959,7 +1959,7 @@ negq %rax ALIGN_4 -.L7_20_2: +L(7_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x3_1(xxx) @@ -1973,7 +1973,7 @@ KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L7_20_6 + je L(7_20_6) KERNEL8x3_1(xxx) KERNEL8x3_2(xxx) @@ -1986,16 +1986,16 @@ KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L7_20_6 + je L(7_20_6) - jmp .L7_20_2 + jmp L(7_20_2) ALIGN_4 -.L7_20_6: +L(7_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_20_9 + je L(7_20_9) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2007,15 +2007,15 @@ negq %rax ALIGN_4 -.L7_20_7: +L(7_20_7): KERNEL8x3_SUB(xxx) addq $3, BI addq $8, %rax - jl .L7_20_7 + jl L(7_20_7) ALIGN_4 -.L7_20_9: +L(7_20_9): vbroadcastss ALPHA, %xmm0 @@ -2046,13 +2046,13 @@ /**************************************************************************/ -.L7_21pre: +L(7_21pre): testq $4, M - jz .L7_30 + jz L(7_30) ALIGN_4 -.L7_21: +L(7_21): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -2061,7 +2061,7 @@ movq K, %rax andq $-8, %rax - je .L7_26 + je L(7_26) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2072,7 +2072,7 @@ negq %rax ALIGN_4 -.L7_22: +L(7_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x3_1(xxx) @@ -2086,7 +2086,7 @@ KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L7_26 + je L(7_26) KERNEL4x3_1(xxx) KERNEL4x3_2(xxx) @@ -2099,16 +2099,16 @@ KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L7_26 + je L(7_26) - jmp .L7_22 + jmp L(7_22) ALIGN_4 -.L7_26: +L(7_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_29 + je L(7_29) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2120,16 +2120,16 @@ negq %rax ALIGN_4 -.L7_27: +L(7_27): KERNEL4x3_SUB(xxx) addq $3, BI addq $4, %rax - jl .L7_27 + jl L(7_27) ALIGN_4 -.L7_29: +L(7_29): vbroadcastss ALPHA, %xmm0 @@ -2145,13 +2145,13 @@ ALIGN_4 -.L7_30: +L(7_30): testq $2, M - jz .L7_40 + jz L(7_40) ALIGN_4 -.L7_31: +L(7_31): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -2160,7 +2160,7 @@ movq K, %rax andq $-8, %rax - je .L7_36 + je L(7_36) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2171,7 +2171,7 @@ negq %rax ALIGN_4 -.L7_32: +L(7_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x3_1(xxx) @@ -2185,7 +2185,7 @@ KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L7_36 + je L(7_36) KERNEL2x3_1(xxx) KERNEL2x3_2(xxx) @@ -2198,16 +2198,16 @@ KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L7_36 + je L(7_36) - jmp .L7_32 + jmp L(7_32) ALIGN_4 -.L7_36: +L(7_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_39 + je L(7_39) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2219,16 +2219,16 @@ negq %rax ALIGN_4 -.L7_37: +L(7_37): KERNEL2x3_SUB(xxx) addq $3, BI addq $2, %rax - jl .L7_37 + jl L(7_37) ALIGN_4 -.L7_39: +L(7_39): vmovss ALPHA, %xmm0 @@ -2249,13 +2249,13 @@ addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L7_40: +L(7_40): testq $1, M - jz .L7_60 // to next 3 lines of N + jz L(7_60) // to next 3 lines of N ALIGN_4 -.L7_41: +L(7_41): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -2264,7 +2264,7 @@ movq K, %rax andq $-8, %rax - je .L7_46 + je L(7_46) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2274,7 +2274,7 @@ negq %rax ALIGN_4 -.L7_42: +L(7_42): KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -2286,7 +2286,7 @@ KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L7_46 + je L(7_46) KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -2298,16 +2298,16 @@ KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L7_46 + je L(7_46) - jmp .L7_42 + jmp L(7_42) ALIGN_4 -.L7_46: +L(7_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_49 + je L(7_49) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2318,16 +2318,16 @@ negq %rax ALIGN_4 -.L7_47: +L(7_47): KERNEL1x3_SUB(xxx) addq $3, BI addq $1, %rax - jl .L7_47 + jl L(7_47) ALIGN_4 -.L7_49: +L(7_49): vmovss ALPHA, %xmm0 @@ -2344,15 +2344,15 @@ -.L7_60: +L(7_60): decq J // j -- - jg .L6_01 + jg L(6_01) -.L2_0: +L(2_0): cmpq $0, Nmod6 // N % 6 == 0 - je .L999 + je L(999) /************************************************************************************************ * Loop for Nmod6 / 2 > 0 @@ -2360,30 +2360,30 @@ movq Nmod6, J sarq $1, J // j = j / 2 - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -2393,11 +2393,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2406,7 +2406,7 @@ movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2417,7 +2417,7 @@ negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x2_1(xxx) @@ -2430,7 +2430,7 @@ KERNEL16x2_3(xxx) KERNEL16x2_4(xxx) - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x2_1(xxx) @@ -2443,16 +2443,16 @@ KERNEL16x2_3(xxx) KERNEL16x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2464,16 +2464,16 @@ negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB(xxx) addq $2, BI addq $16, %rax - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vbroadcastss ALPHA, %xmm0 @@ -2499,25 +2499,25 @@ addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 3 lines of N + jz L(2_60) // to next 3 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2526,7 +2526,7 @@ movq K, %rax andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2537,7 +2537,7 @@ negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x2_1(xxx) @@ -2550,7 +2550,7 @@ KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_20_6 + je L(2_20_6) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x2_1(xxx) @@ -2563,16 +2563,16 @@ KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2584,16 +2584,16 @@ negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB(xxx) addq $2, BI addq $8, %rax - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): vbroadcastss ALPHA, %xmm0 @@ -2617,13 +2617,13 @@ /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2632,7 +2632,7 @@ movq K, %rax andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -2643,7 +2643,7 @@ negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x2_1(xxx) @@ -2656,7 +2656,7 @@ KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x2_1(xxx) @@ -2669,16 +2669,16 @@ KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2690,16 +2690,16 @@ negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB(xxx) addq $2, BI addq $4, %rax - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): vbroadcastss ALPHA, %xmm0 @@ -2714,13 +2714,13 @@ ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2729,7 +2729,7 @@ movq K, %rax andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2740,7 +2740,7 @@ negq %rax ALIGN_4 -.L2_32: +L(2_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -2753,7 +2753,7 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -2766,16 +2766,16 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2787,16 +2787,16 @@ negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB(xxx) addq $2, BI addq $2, %rax - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): vmovss ALPHA, %xmm0 @@ -2813,13 +2813,13 @@ addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2828,7 +2828,7 @@ movq K, %rax andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2838,7 +2838,7 @@ negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -2850,7 +2850,7 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -2862,16 +2862,16 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2882,16 +2882,16 @@ negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) addq $2, BI addq $1, %rax - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vmovss ALPHA, %xmm0 @@ -2904,14 +2904,14 @@ addq $1 * SIZE, CO1 # coffset += 1 ALIGN_4 -.L2_60: +L(2_60): decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -2919,30 +2919,30 @@ movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovss (BO1), %xmm0 vmovss %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -2952,11 +2952,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -2965,7 +2965,7 @@ movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -2975,7 +2975,7 @@ negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x1_1(xxx) @@ -2988,7 +2988,7 @@ KERNEL16x1_3(xxx) KERNEL16x1_4(xxx) - je .L1_16 + je L(1_16) KERNEL16x1_1(xxx) KERNEL16x1_2(xxx) @@ -3000,16 +3000,16 @@ KERNEL16x1_3(xxx) KERNEL16x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -3020,16 +3020,16 @@ negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB(xxx) addq $1, BI addq $16, %rax - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vbroadcastss ALPHA, %xmm0 @@ -3045,25 +3045,25 @@ addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3072,7 +3072,7 @@ movq K, %rax andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -3082,7 +3082,7 @@ negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x1_1(xxx) @@ -3095,7 +3095,7 @@ KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_20_6 + je L(1_20_6) KERNEL8x1_1(xxx) KERNEL8x1_2(xxx) @@ -3107,16 +3107,16 @@ KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -3127,16 +3127,16 @@ negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB(xxx) addq $1, BI addq $8, %rax - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): vbroadcastss ALPHA, %xmm0 @@ -3153,13 +3153,13 @@ /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3168,7 +3168,7 @@ movq K, %rax andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -3178,7 +3178,7 @@ negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x1_1(xxx) @@ -3191,7 +3191,7 @@ KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) KERNEL4x1_1(xxx) KERNEL4x1_2(xxx) @@ -3203,16 +3203,16 @@ KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -3223,16 +3223,16 @@ negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB(xxx) addq $1, BI addq $4, %rax - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): vbroadcastss ALPHA, %xmm0 @@ -3244,13 +3244,13 @@ ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3259,7 +3259,7 @@ movq K, %rax andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -3269,7 +3269,7 @@ negq %rax ALIGN_4 -.L1_32: +L(1_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -3282,7 +3282,7 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) KERNEL2x1_1(xxx) KERNEL2x1_2(xxx) @@ -3294,16 +3294,16 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -3314,16 +3314,16 @@ negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB(xxx) addq $1, BI addq $2, %rax - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): vmovss ALPHA, %xmm0 @@ -3336,13 +3336,13 @@ addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3351,7 +3351,7 @@ movq K, %rax andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -3360,7 +3360,7 @@ negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -3372,7 +3372,7 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -3384,16 +3384,16 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -3403,16 +3403,16 @@ negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) addq $1, BI addq $1, %rax - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vmovss ALPHA, %xmm0 @@ -3424,7 +3424,7 @@ ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp @@ -3514,13 +3514,13 @@ STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -3549,30 +3549,30 @@ movq Ndiv6, J cmpq $0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -3586,11 +3586,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3627,7 +3627,7 @@ #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3638,7 +3638,7 @@ negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x2_1(xxx) @@ -3651,7 +3651,7 @@ KERNEL16x2_3(xxx) KERNEL16x2_4(xxx) - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x2_1(xxx) @@ -3664,12 +3664,12 @@ KERNEL16x2_3(xxx) KERNEL16x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3677,7 +3677,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3689,16 +3689,16 @@ negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB(xxx) addq $2, BI addq $16, %rax - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vbroadcastss ALPHA, %xmm0 @@ -3756,25 +3756,25 @@ addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 3 lines of N + jz L(2_60) // to next 3 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3812,7 +3812,7 @@ andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3823,7 +3823,7 @@ negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x2_1(xxx) @@ -3836,7 +3836,7 @@ KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_20_6 + je L(2_20_6) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x2_1(xxx) @@ -3849,12 +3849,12 @@ KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -3862,7 +3862,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3874,16 +3874,16 @@ negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB(xxx) addq $2, BI addq $8, %rax - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): vbroadcastss ALPHA, %xmm0 @@ -3934,13 +3934,13 @@ /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3978,7 +3978,7 @@ andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -3989,7 +3989,7 @@ negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x2_1(xxx) @@ -4002,7 +4002,7 @@ KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x2_1(xxx) @@ -4015,12 +4015,12 @@ KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4028,7 +4028,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4040,16 +4040,16 @@ negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB(xxx) addq $2, BI addq $4, %rax - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): vbroadcastss ALPHA, %xmm0 @@ -4088,13 +4088,13 @@ ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4132,7 +4132,7 @@ andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4143,7 +4143,7 @@ negq %rax ALIGN_4 -.L2_32: +L(2_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -4156,7 +4156,7 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -4169,12 +4169,12 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -4182,7 +4182,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4194,16 +4194,16 @@ negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB(xxx) addq $2, BI addq $2, %rax - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): vmovss ALPHA, %xmm0 @@ -4246,13 +4246,13 @@ addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4288,7 +4288,7 @@ #endif andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4298,7 +4298,7 @@ negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -4310,7 +4310,7 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -4322,12 +4322,12 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -4335,7 +4335,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4346,16 +4346,16 @@ negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) addq $2, BI addq $1, %rax - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vmovss ALPHA, %xmm0 @@ -4395,17 +4395,17 @@ -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -4413,30 +4413,30 @@ movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovss (BO1), %xmm0 vmovss %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -4450,11 +4450,11 @@ movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4490,7 +4490,7 @@ #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -4500,7 +4500,7 @@ negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x1_1(xxx) @@ -4513,7 +4513,7 @@ KERNEL16x1_3(xxx) KERNEL16x1_4(xxx) - je .L1_16 + je L(1_16) KERNEL16x1_1(xxx) KERNEL16x1_2(xxx) @@ -4525,12 +4525,12 @@ KERNEL16x1_3(xxx) KERNEL16x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -4538,7 +4538,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -4549,16 +4549,16 @@ negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB(xxx) addq $1, BI addq $16, %rax - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vbroadcastss ALPHA, %xmm0 @@ -4600,25 +4600,25 @@ addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4655,7 +4655,7 @@ andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -4665,7 +4665,7 @@ negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x1_1(xxx) @@ -4678,7 +4678,7 @@ KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_20_6 + je L(1_20_6) KERNEL8x1_1(xxx) KERNEL8x1_2(xxx) @@ -4690,12 +4690,12 @@ KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -4703,7 +4703,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -4714,16 +4714,16 @@ negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB(xxx) addq $1, BI addq $8, %rax - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): vbroadcastss ALPHA, %xmm0 @@ -4763,13 +4763,13 @@ /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4806,7 +4806,7 @@ andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -4816,7 +4816,7 @@ negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x1_1(xxx) @@ -4829,7 +4829,7 @@ KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) KERNEL4x1_1(xxx) KERNEL4x1_2(xxx) @@ -4841,12 +4841,12 @@ KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4854,7 +4854,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -4865,16 +4865,16 @@ negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB(xxx) addq $1, BI addq $4, %rax - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): vbroadcastss ALPHA, %xmm0 @@ -4909,13 +4909,13 @@ ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4952,7 +4952,7 @@ andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -4962,7 +4962,7 @@ negq %rax ALIGN_4 -.L1_32: +L(1_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -4975,7 +4975,7 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) KERNEL2x1_1(xxx) KERNEL2x1_2(xxx) @@ -4987,12 +4987,12 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -5000,7 +5000,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -5011,16 +5011,16 @@ negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB(xxx) addq $1, BI addq $2, %rax - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): vmovss ALPHA, %xmm0 @@ -5056,13 +5056,13 @@ addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5097,7 +5097,7 @@ #endif andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -5106,7 +5106,7 @@ negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -5118,7 +5118,7 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -5130,12 +5130,12 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -5143,7 +5143,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -5153,16 +5153,16 @@ negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) addq $1, BI addq $1, %rax - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vmovss ALPHA, %xmm0 @@ -5195,7 +5195,7 @@ ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/sgemm_kernel_16x2_piledriver.S b/kernel/x86_64/sgemm_kernel_16x2_piledriver.S index 35b01de07b..32125266e8 100644 --- a/kernel/x86_64/sgemm_kernel_16x2_piledriver.S +++ b/kernel/x86_64/sgemm_kernel_16x2_piledriver.S @@ -1060,13 +1060,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -1086,10 +1086,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv6, J cmpq $0, J - je .L2_0 + je L(2_0) ALIGN_4 -.L6_01: +L(6_01): // copy to sub buffer movq K, %rax salq $1,%rax // K * 2 ; read 2 values @@ -1098,10 +1098,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $3 , %rax // K / 8 - jz .L6_01a_2 + jz L(6_01a_2) ALIGN_4 -.L6_01a_1: +L(6_01a_1): prefetcht0 512(BO1) prefetcht0 512(BO2) @@ -1148,19 +1148,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $12*SIZE,BO decq %rax - jnz .L6_01a_1 + jnz L(6_01a_1) -.L6_01a_2: +L(6_01a_2): movq K, %rax andq $7, %rax // K % 8 - jz .L6_02c + jz L(6_02c) ALIGN_4 -.L6_02b: +L(6_02b): vmovsd 0 * SIZE(BO1), %xmm0 vmovss 0 * SIZE(BO2), %xmm2 @@ -1170,9 +1170,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2*SIZE,BO2 addq $3*SIZE,BO decq %rax - jnz .L6_02b + jnz L(6_02b) -.L6_02c: +L(6_02c): movq K, %rax salq $1,%rax // K * 2 @@ -1181,10 +1181,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. leaq BUFFER2, BO // second buffer to BO movq K, %rax sarq $3 , %rax // K / 8 - jz .L6_02c_2 + jz L(6_02c_2) ALIGN_4 -.L6_02c_1: +L(6_02c_1): prefetcht0 512(BO2) prefetchw 512(BO) @@ -1231,17 +1231,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $12*SIZE,BO decq %rax - jnz .L6_02c_1 + jnz L(6_02c_1) -.L6_02c_2: +L(6_02c_2): movq K, %rax andq $7, %rax // K % 8 - jz .L6_03c + jz L(6_03c) ALIGN_4 -.L6_03b: +L(6_03b): vmovss 1*SIZE(BO1), %xmm0 vmovsd 0*SIZE(BO2), %xmm1 @@ -1251,14 +1251,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2*SIZE,BO2 addq $3*SIZE,BO decq %rax - jnz .L6_03b + jnz L(6_03b) -.L6_03c: +L(6_03c): movq BO2, B // next offset of B -.L6_10: +L(6_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -1269,11 +1269,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L6_20 + je L(6_20) ALIGN_4 -.L6_11: +L(6_11): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1282,7 +1282,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L6_16 + je L(6_16) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1293,7 +1293,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_12: +L(6_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x3_1(xxx) @@ -1307,7 +1307,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x3_3(xxx) KERNEL16x3_4(xxx) - je .L6_16 + je L(6_16) KERNEL16x3_1(xxx) KERNEL16x3_2(xxx) @@ -1320,16 +1320,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x3_3(xxx) KERNEL16x3_4(xxx) - je .L6_16 + je L(6_16) - jmp .L6_12 + jmp L(6_12) ALIGN_4 -.L6_16: +L(6_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_19 + je L(6_19) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1341,16 +1341,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_17: +L(6_17): KERNEL16x3_SUB(xxx) addq $3, BI addq $16, %rax - jl .L6_17 + jl L(6_17) ALIGN_4 -.L6_19: +L(6_19): vbroadcastss ALPHA, %xmm0 @@ -1388,25 +1388,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L6_11 + jg L(6_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L6_20: +L(6_20): // Test rest of M testq $15, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N testq $8, M - jz .L6_21pre + jz L(6_21pre) ALIGN_4 /**************************************************************************/ -.L6_20_1: +L(6_20_1): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1415,7 +1415,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_20_6 + je L(6_20_6) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1426,7 +1426,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_20_2: +L(6_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x3_1(xxx) @@ -1440,7 +1440,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L6_20_6 + je L(6_20_6) KERNEL8x3_1(xxx) KERNEL8x3_2(xxx) @@ -1453,16 +1453,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L6_20_6 + je L(6_20_6) - jmp .L6_20_2 + jmp L(6_20_2) ALIGN_4 -.L6_20_6: +L(6_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_20_9 + je L(6_20_9) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1474,16 +1474,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_20_7: +L(6_20_7): KERNEL8x3_SUB(xxx) addq $3, BI addq $8, %rax - jl .L6_20_7 + jl L(6_20_7) ALIGN_4 -.L6_20_9: +L(6_20_9): vbroadcastss ALPHA, %xmm0 @@ -1514,13 +1514,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L6_21pre: +L(6_21pre): testq $4, M - jz .L6_30 + jz L(6_30) ALIGN_4 -.L6_21: +L(6_21): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1529,7 +1529,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_26 + je L(6_26) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1540,7 +1540,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_22: +L(6_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x3_1(xxx) @@ -1554,7 +1554,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L6_26 + je L(6_26) KERNEL4x3_1(xxx) KERNEL4x3_2(xxx) @@ -1567,16 +1567,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L6_26 + je L(6_26) - jmp .L6_22 + jmp L(6_22) ALIGN_4 -.L6_26: +L(6_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_29 + je L(6_29) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1588,16 +1588,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_27: +L(6_27): KERNEL4x3_SUB(xxx) addq $3, BI addq $4, %rax - jl .L6_27 + jl L(6_27) ALIGN_4 -.L6_29: +L(6_29): vbroadcastss ALPHA, %xmm0 @@ -1614,13 +1614,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L6_30: +L(6_30): testq $2, M - jz .L6_40 + jz L(6_40) ALIGN_4 -.L6_31: +L(6_31): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1629,7 +1629,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_36 + je L(6_36) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1640,7 +1640,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_32: +L(6_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x3_1(xxx) @@ -1654,7 +1654,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L6_36 + je L(6_36) KERNEL2x3_1(xxx) KERNEL2x3_2(xxx) @@ -1667,16 +1667,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L6_36 + je L(6_36) - jmp .L6_32 + jmp L(6_32) ALIGN_4 -.L6_36: +L(6_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_39 + je L(6_39) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1688,16 +1688,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_37: +L(6_37): KERNEL2x3_SUB(xxx) addq $3, BI addq $2, %rax - jl .L6_37 + jl L(6_37) ALIGN_4 -.L6_39: +L(6_39): vmovss ALPHA, %xmm0 @@ -1719,13 +1719,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L6_40: +L(6_40): testq $1, M - jz .L7_10 // to next 3 lines of N + jz L(7_10) // to next 3 lines of N ALIGN_4 -.L6_41: +L(6_41): leaq BUFFER1, BO // first buffer to BO addq $6 * SIZE, BO @@ -1734,7 +1734,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_46 + je L(6_46) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1744,7 +1744,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_42: +L(6_42): KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -1756,7 +1756,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L6_46 + je L(6_46) KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -1768,16 +1768,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L6_46 + je L(6_46) - jmp .L6_42 + jmp L(6_42) ALIGN_4 -.L6_46: +L(6_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_49 + je L(6_49) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1788,16 +1788,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L6_47: +L(6_47): KERNEL1x3_SUB(xxx) addq $3, BI addq $1, %rax - jl .L6_47 + jl L(6_47) ALIGN_4 -.L6_49: +L(6_49): vmovss ALPHA, %xmm0 @@ -1818,7 +1818,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /***************************************************************************************************************/ -.L7_10: +L(7_10): movq C, CO1 leaq (C, LDC, 2), C leaq (C, LDC, 1), C // c += 3 * ldc @@ -1829,11 +1829,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L7_20 + je L(7_20) ALIGN_4 -.L7_11: +L(7_11): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -1842,7 +1842,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L7_16 + je L(7_16) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1853,7 +1853,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_12: +L(7_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x3_1(xxx) @@ -1867,7 +1867,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x3_3(xxx) KERNEL16x3_4(xxx) - je .L7_16 + je L(7_16) KERNEL16x3_1(xxx) KERNEL16x3_2(xxx) @@ -1880,16 +1880,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x3_3(xxx) KERNEL16x3_4(xxx) - je .L7_16 + je L(7_16) - jmp .L7_12 + jmp L(7_12) ALIGN_4 -.L7_16: +L(7_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_19 + je L(7_19) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1901,16 +1901,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_17: +L(7_17): KERNEL16x3_SUB(xxx) addq $3, BI addq $16, %rax - jl .L7_17 + jl L(7_17) ALIGN_4 -.L7_19: +L(7_19): vbroadcastss ALPHA, %xmm0 @@ -1948,25 +1948,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L7_11 + jg L(7_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L7_20: +L(7_20): // Test rest of M testq $15, M - jz .L7_60 // to next 3 lines of N + jz L(7_60) // to next 3 lines of N testq $8, M - jz .L7_21pre + jz L(7_21pre) ALIGN_4 /**************************************************************************/ -.L7_20_1: +L(7_20_1): leaq BUFFER2, BO // first buffer to BO addq $6 * SIZE, BO @@ -1975,7 +1975,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_20_6 + je L(7_20_6) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -1986,7 +1986,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_20_2: +L(7_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x3_1(xxx) @@ -2000,7 +2000,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L7_20_6 + je L(7_20_6) KERNEL8x3_1(xxx) KERNEL8x3_2(xxx) @@ -2013,16 +2013,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x3_3(xxx) KERNEL8x3_4(xxx) - je .L7_20_6 + je L(7_20_6) - jmp .L7_20_2 + jmp L(7_20_2) ALIGN_4 -.L7_20_6: +L(7_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_20_9 + je L(7_20_9) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2034,15 +2034,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_20_7: +L(7_20_7): KERNEL8x3_SUB(xxx) addq $3, BI addq $8, %rax - jl .L7_20_7 + jl L(7_20_7) ALIGN_4 -.L7_20_9: +L(7_20_9): vbroadcastss ALPHA, %xmm0 @@ -2073,13 +2073,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L7_21pre: +L(7_21pre): testq $4, M - jz .L7_30 + jz L(7_30) ALIGN_4 -.L7_21: +L(7_21): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -2088,7 +2088,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_26 + je L(7_26) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2099,7 +2099,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_22: +L(7_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x3_1(xxx) @@ -2113,7 +2113,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L7_26 + je L(7_26) KERNEL4x3_1(xxx) KERNEL4x3_2(xxx) @@ -2126,16 +2126,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_3(xxx) KERNEL4x3_4(xxx) - je .L7_26 + je L(7_26) - jmp .L7_22 + jmp L(7_22) ALIGN_4 -.L7_26: +L(7_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_29 + je L(7_29) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2147,16 +2147,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_27: +L(7_27): KERNEL4x3_SUB(xxx) addq $3, BI addq $4, %rax - jl .L7_27 + jl L(7_27) ALIGN_4 -.L7_29: +L(7_29): vbroadcastss ALPHA, %xmm0 @@ -2172,13 +2172,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L7_30: +L(7_30): testq $2, M - jz .L7_40 + jz L(7_40) ALIGN_4 -.L7_31: +L(7_31): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -2187,7 +2187,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_36 + je L(7_36) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2198,7 +2198,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_32: +L(7_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x3_1(xxx) @@ -2212,7 +2212,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L7_36 + je L(7_36) KERNEL2x3_1(xxx) KERNEL2x3_2(xxx) @@ -2225,16 +2225,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_3(xxx) KERNEL2x3_4(xxx) - je .L7_36 + je L(7_36) - jmp .L7_32 + jmp L(7_32) ALIGN_4 -.L7_36: +L(7_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_39 + je L(7_39) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2246,16 +2246,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_37: +L(7_37): KERNEL2x3_SUB(xxx) addq $3, BI addq $2, %rax - jl .L7_37 + jl L(7_37) ALIGN_4 -.L7_39: +L(7_39): vmovss ALPHA, %xmm0 @@ -2276,13 +2276,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L7_40: +L(7_40): testq $1, M - jz .L7_60 // to next 3 lines of N + jz L(7_60) // to next 3 lines of N ALIGN_4 -.L7_41: +L(7_41): leaq BUFFER2, BO // second buffer to BO addq $6 * SIZE, BO @@ -2291,7 +2291,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_46 + je L(7_46) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2301,7 +2301,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_42: +L(7_42): KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -2313,7 +2313,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L7_46 + je L(7_46) KERNEL1x3_1(xxx) KERNEL1x3_2(xxx) @@ -2325,16 +2325,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_3(xxx) KERNEL1x3_4(xxx) - je .L7_46 + je L(7_46) - jmp .L7_42 + jmp L(7_42) ALIGN_4 -.L7_46: +L(7_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_49 + je L(7_49) movq %rax, BI // Index for BO leaq (BI,BI,2), BI // BI = BI * 3 ; number of values @@ -2345,16 +2345,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L7_47: +L(7_47): KERNEL1x3_SUB(xxx) addq $3, BI addq $1, %rax - jl .L7_47 + jl L(7_47) ALIGN_4 -.L7_49: +L(7_49): vmovss ALPHA, %xmm0 @@ -2371,15 +2371,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L7_60: +L(7_60): decq J // j -- - jg .L6_01 + jg L(6_01) -.L2_0: +L(2_0): cmpq $0, Nmod6 // N % 6 == 0 - je .L999 + je L(999) /************************************************************************************************ * Loop for Nmod6 / 2 > 0 @@ -2387,30 +2387,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J sarq $1, J // j = j / 2 - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -2420,11 +2420,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2433,7 +2433,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2444,7 +2444,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x2_1(xxx) @@ -2457,7 +2457,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_3(xxx) KERNEL16x2_4(xxx) - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x2_1(xxx) @@ -2470,16 +2470,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_3(xxx) KERNEL16x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2491,16 +2491,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB(xxx) addq $2, BI addq $16, %rax - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vbroadcastss ALPHA, %xmm0 @@ -2526,25 +2526,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 3 lines of N + jz L(2_60) // to next 3 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2553,7 +2553,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2564,7 +2564,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x2_1(xxx) @@ -2577,7 +2577,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_20_6 + je L(2_20_6) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x2_1(xxx) @@ -2590,16 +2590,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2611,16 +2611,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB(xxx) addq $2, BI addq $8, %rax - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): vbroadcastss ALPHA, %xmm0 @@ -2644,13 +2644,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2659,7 +2659,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -2670,7 +2670,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x2_1(xxx) @@ -2683,7 +2683,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x2_1(xxx) @@ -2696,16 +2696,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2717,16 +2717,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB(xxx) addq $2, BI addq $4, %rax - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): vbroadcastss ALPHA, %xmm0 @@ -2741,13 +2741,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2756,7 +2756,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2767,7 +2767,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_32: +L(2_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -2780,7 +2780,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -2793,16 +2793,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2814,16 +2814,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB(xxx) addq $2, BI addq $2, %rax - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): vmovss ALPHA, %xmm0 @@ -2840,13 +2840,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2855,7 +2855,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2865,7 +2865,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -2877,7 +2877,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -2889,16 +2889,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2909,16 +2909,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) addq $2, BI addq $1, %rax - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vmovss ALPHA, %xmm0 @@ -2931,14 +2931,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $1 * SIZE, CO1 # coffset += 1 ALIGN_4 -.L2_60: +L(2_60): decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -2946,30 +2946,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovss (BO1), %xmm0 vmovss %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -2979,11 +2979,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -2992,7 +2992,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -3002,7 +3002,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x1_1(xxx) @@ -3015,7 +3015,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_3(xxx) KERNEL16x1_4(xxx) - je .L1_16 + je L(1_16) KERNEL16x1_1(xxx) KERNEL16x1_2(xxx) @@ -3027,16 +3027,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_3(xxx) KERNEL16x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -3047,16 +3047,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB(xxx) addq $1, BI addq $16, %rax - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vbroadcastss ALPHA, %xmm0 @@ -3072,25 +3072,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3099,7 +3099,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -3109,7 +3109,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x1_1(xxx) @@ -3122,7 +3122,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_20_6 + je L(1_20_6) KERNEL8x1_1(xxx) KERNEL8x1_2(xxx) @@ -3134,16 +3134,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -3154,16 +3154,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB(xxx) addq $1, BI addq $8, %rax - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): vbroadcastss ALPHA, %xmm0 @@ -3180,13 +3180,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3195,7 +3195,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -3205,7 +3205,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x1_1(xxx) @@ -3218,7 +3218,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) KERNEL4x1_1(xxx) KERNEL4x1_2(xxx) @@ -3230,16 +3230,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -3250,16 +3250,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB(xxx) addq $1, BI addq $4, %rax - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): vbroadcastss ALPHA, %xmm0 @@ -3271,13 +3271,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3286,7 +3286,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -3296,7 +3296,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_32: +L(1_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -3309,7 +3309,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) KERNEL2x1_1(xxx) KERNEL2x1_2(xxx) @@ -3321,16 +3321,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -3341,16 +3341,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB(xxx) addq $1, BI addq $2, %rax - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): vmovss ALPHA, %xmm0 @@ -3363,13 +3363,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): leaq BUFFER1, BO // first buffer to BO addq $2 * SIZE, BO @@ -3378,7 +3378,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -3387,7 +3387,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -3399,7 +3399,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -3411,16 +3411,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -3430,16 +3430,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) addq $1, BI addq $1, %rax - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vmovss ALPHA, %xmm0 @@ -3451,7 +3451,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp @@ -3541,13 +3541,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -3576,30 +3576,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv6, J cmpq $0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -3613,11 +3613,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3654,7 +3654,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3665,7 +3665,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x2_1(xxx) @@ -3678,7 +3678,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_3(xxx) KERNEL16x2_4(xxx) - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x2_1(xxx) @@ -3691,12 +3691,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_3(xxx) KERNEL16x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3704,7 +3704,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3716,16 +3716,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB(xxx) addq $2, BI addq $16, %rax - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vbroadcastss ALPHA, %xmm0 @@ -3783,25 +3783,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 3 lines of N + jz L(2_60) // to next 3 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3839,7 +3839,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3850,7 +3850,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x2_1(xxx) @@ -3863,7 +3863,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_20_6 + je L(2_20_6) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x2_1(xxx) @@ -3876,12 +3876,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_3(xxx) KERNEL8x2_4(xxx) - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -3889,7 +3889,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3901,16 +3901,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB(xxx) addq $2, BI addq $8, %rax - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): vbroadcastss ALPHA, %xmm0 @@ -3961,13 +3961,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4005,7 +4005,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -4016,7 +4016,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_22: +L(2_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x2_1(xxx) @@ -4029,7 +4029,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x2_1(xxx) @@ -4042,12 +4042,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_3(xxx) KERNEL4x2_4(xxx) - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4055,7 +4055,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4067,16 +4067,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB(xxx) addq $2, BI addq $4, %rax - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): vbroadcastss ALPHA, %xmm0 @@ -4115,13 +4115,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4159,7 +4159,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4170,7 +4170,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_32: +L(2_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -4183,7 +4183,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -4196,12 +4196,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -4209,7 +4209,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4221,16 +4221,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB(xxx) addq $2, BI addq $2, %rax - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): vmovss ALPHA, %xmm0 @@ -4273,13 +4273,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4315,7 +4315,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4325,7 +4325,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -4337,7 +4337,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) KERNEL1x2_1(xxx) KERNEL1x2_2(xxx) @@ -4349,12 +4349,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -4362,7 +4362,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -4373,16 +4373,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) addq $2, BI addq $1, %rax - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vmovss ALPHA, %xmm0 @@ -4422,17 +4422,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -4440,30 +4440,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovss (BO1), %xmm0 vmovss %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -4477,11 +4477,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4517,7 +4517,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -4527,7 +4527,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL16x1_1(xxx) @@ -4540,7 +4540,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_3(xxx) KERNEL16x1_4(xxx) - je .L1_16 + je L(1_16) KERNEL16x1_1(xxx) KERNEL16x1_2(xxx) @@ -4552,12 +4552,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_3(xxx) KERNEL16x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -4565,7 +4565,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -4576,16 +4576,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB(xxx) addq $1, BI addq $16, %rax - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vbroadcastss ALPHA, %xmm0 @@ -4627,25 +4627,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4682,7 +4682,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -4692,7 +4692,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL8x1_1(xxx) @@ -4705,7 +4705,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_20_6 + je L(1_20_6) KERNEL8x1_1(xxx) KERNEL8x1_2(xxx) @@ -4717,12 +4717,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_3(xxx) KERNEL8x1_4(xxx) - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -4730,7 +4730,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -4741,16 +4741,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB(xxx) addq $1, BI addq $8, %rax - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): vbroadcastss ALPHA, %xmm0 @@ -4790,13 +4790,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4833,7 +4833,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -4843,7 +4843,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_22: +L(1_22): prefetcht0 B_PR1(BO,BI, SIZE) KERNEL4x1_1(xxx) @@ -4856,7 +4856,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) KERNEL4x1_1(xxx) KERNEL4x1_2(xxx) @@ -4868,12 +4868,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_3(xxx) KERNEL4x1_4(xxx) - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4881,7 +4881,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -4892,16 +4892,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB(xxx) addq $1, BI addq $4, %rax - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): vbroadcastss ALPHA, %xmm0 @@ -4936,13 +4936,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4979,7 +4979,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -4989,7 +4989,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_32: +L(1_32): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -5002,7 +5002,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) KERNEL2x1_1(xxx) KERNEL2x1_2(xxx) @@ -5014,12 +5014,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -5027,7 +5027,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -5038,16 +5038,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB(xxx) addq $1, BI addq $2, %rax - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): vmovss ALPHA, %xmm0 @@ -5083,13 +5083,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5124,7 +5124,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -5133,7 +5133,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -5145,7 +5145,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) KERNEL1x1_1(xxx) KERNEL1x1_2(xxx) @@ -5157,12 +5157,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -5170,7 +5170,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -5180,16 +5180,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) addq $1, BI addq $1, %rax - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vmovss ALPHA, %xmm0 @@ -5222,7 +5222,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/sgemm_kernel_16x4_haswell.S b/kernel/x86_64/sgemm_kernel_16x4_haswell.S index 76ea12fee7..e6579f11c2 100644 --- a/kernel/x86_64/sgemm_kernel_16x4_haswell.S +++ b/kernel/x86_64/sgemm_kernel_16x4_haswell.S @@ -1191,13 +1191,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -1216,13 +1216,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv6, J cmpq $0, J - je .L4_00 + je L(4_00) ALIGN_4 /*******************************************************************************************/ -.L6_01: +L(6_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO @@ -1235,7 +1235,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L6_02c: +L(6_02c): vmovups (BO1), %xmm0 vmovsd (BO2), %xmm1 @@ -1245,10 +1245,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4*SIZE,BO2 addq $ 6*SIZE,BO decq %rax - jnz .L6_02c + jnz L(6_02c) -.L6_10: +L(6_10): movq C, CO1 leaq (C, LDC, 2), CO2 leaq (CO2, LDC, 1), CO2 // co2 = c + 3 * ldc @@ -1260,11 +1260,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L6_20 + je L(6_20) ALIGN_4 -.L6_11: +L(6_11): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1273,11 +1273,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L6_16 + je L(6_16) ALIGN_4 -.L6_12: +L(6_12): KERNEL16x6_SUB KERNEL16x6_SUB @@ -1289,7 +1289,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x6_SUB KERNEL16x6_SUB - je .L6_16 + je L(6_16) KERNEL16x6_SUB KERNEL16x6_SUB @@ -1301,53 +1301,53 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x6_SUB KERNEL16x6_SUB - je .L6_16 + je L(6_16) - jmp .L6_12 + jmp L(6_12) ALIGN_4 -.L6_16: +L(6_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_19 + je L(6_19) ALIGN_4 -.L6_17: +L(6_17): KERNEL16x6_SUB - jnz .L6_17 + jnz L(6_17) ALIGN_4 -.L6_19: +L(6_19): SAVE16x6 addq $16 * SIZE, CO1 # coffset += 16 addq $16 * SIZE, CO2 # coffset += 16 decq I # i -- - jg .L6_11 + jg L(6_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L6_20: +L(6_20): // Test rest of M testq $15, M - jz .L6_60 // to next 6 lines of N + jz L(6_60) // to next 6 lines of N testq $8, M - jz .L6_21pre + jz L(6_21pre) ALIGN_4 /**************************************************************************/ -.L6_20_1: +L(6_20_1): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1356,11 +1356,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_20_6 + je L(6_20_6) ALIGN_4 -.L6_20_2: +L(6_20_2): prefetcht0 A_PR1(AO) KERNEL8x6_SUB @@ -1376,7 +1376,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x6_SUB KERNEL8x6_SUB - je .L6_20_6 + je L(6_20_6) prefetcht0 A_PR1(AO) KERNEL8x6_SUB @@ -1392,28 +1392,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x6_SUB KERNEL8x6_SUB - je .L6_20_6 + je L(6_20_6) - jmp .L6_20_2 + jmp L(6_20_2) ALIGN_4 -.L6_20_6: +L(6_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_20_9 + je L(6_20_9) ALIGN_4 -.L6_20_7: +L(6_20_7): KERNEL8x6_SUB - jnz .L6_20_7 + jnz L(6_20_7) ALIGN_4 -.L6_20_9: +L(6_20_9): SAVE8x6 @@ -1425,13 +1425,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L6_21pre: +L(6_21pre): testq $4, M - jz .L6_30 + jz L(6_30) ALIGN_4 -.L6_21: +L(6_21): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1440,11 +1440,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_26 + je L(6_26) ALIGN_4 -.L6_22: +L(6_22): prefetcht0 A_PR1(AO) KERNEL4x6_SUB @@ -1458,7 +1458,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x6_SUB KERNEL4x6_SUB - je .L6_26 + je L(6_26) prefetcht0 A_PR1(AO) KERNEL4x6_SUB @@ -1472,28 +1472,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x6_SUB KERNEL4x6_SUB - je .L6_26 + je L(6_26) - jmp .L6_22 + jmp L(6_22) ALIGN_4 -.L6_26: +L(6_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_29 + je L(6_29) ALIGN_4 -.L6_27: +L(6_27): KERNEL4x6_SUB - jnz .L6_27 + jnz L(6_27) ALIGN_4 -.L6_29: +L(6_29): SAVE4x6 @@ -1502,13 +1502,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L6_30: +L(6_30): testq $2, M - jz .L6_40 + jz L(6_40) ALIGN_4 -.L6_31: +L(6_31): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1517,11 +1517,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_36 + je L(6_36) ALIGN_4 -.L6_32: +L(6_32): prefetcht0 A_PR1(AO) KERNEL2x6_SUB @@ -1534,7 +1534,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x6_SUB KERNEL2x6_SUB - je .L6_36 + je L(6_36) prefetcht0 A_PR1(AO) KERNEL2x6_SUB @@ -1547,28 +1547,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x6_SUB KERNEL2x6_SUB - je .L6_36 + je L(6_36) - jmp .L6_32 + jmp L(6_32) ALIGN_4 -.L6_36: +L(6_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_39 + je L(6_39) ALIGN_4 -.L6_37: +L(6_37): KERNEL2x6_SUB - jnz .L6_37 + jnz L(6_37) ALIGN_4 -.L6_39: +L(6_39): SAVE2x6 @@ -1576,13 +1576,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO2 # coffset += 2 ALIGN_4 -.L6_40: +L(6_40): testq $1, M - jz .L6_60 // to next 4 lines of N + jz L(6_60) // to next 4 lines of N ALIGN_4 -.L6_41: +L(6_41): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1591,11 +1591,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_46 + je L(6_46) ALIGN_4 -.L6_42: +L(6_42): prefetcht0 A_PR1(AO) KERNEL1x6_SUB @@ -1608,7 +1608,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x6_SUB KERNEL1x6_SUB - je .L6_46 + je L(6_46) KERNEL1x6_SUB KERNEL1x6_SUB @@ -1620,28 +1620,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x6_SUB KERNEL1x6_SUB - je .L6_46 + je L(6_46) - jmp .L6_42 + jmp L(6_42) ALIGN_4 -.L6_46: +L(6_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_49 + je L(6_49) ALIGN_4 -.L6_47: +L(6_47): KERNEL1x6_SUB - jnz .L6_47 + jnz L(6_47) ALIGN_4 -.L6_49: +L(6_49): SAVE1x6 @@ -1653,13 +1653,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L6_60: +L(6_60): /*******************************************************************************************/ -.L7_01: +L(7_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO @@ -1671,7 +1671,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L7_02c: +L(7_02c): vmovsd 2*SIZE(BO1), %xmm0 vmovups (BO2), %xmm1 @@ -1681,11 +1681,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4*SIZE,BO2 addq $ 6*SIZE,BO decq %rax - jnz .L7_02c + jnz L(7_02c) movq BO2, B // next offset of B -.L7_10: +L(7_10): movq C, CO1 leaq (C, LDC, 2), CO2 leaq (CO2, LDC, 1), CO2 // co2 = c + 3 * ldc @@ -1697,11 +1697,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L7_20 + je L(7_20) ALIGN_4 -.L7_11: +L(7_11): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1710,11 +1710,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L7_16 + je L(7_16) ALIGN_4 -.L7_12: +L(7_12): KERNEL16x6_SUB KERNEL16x6_SUB @@ -1726,7 +1726,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x6_SUB KERNEL16x6_SUB - je .L7_16 + je L(7_16) KERNEL16x6_SUB KERNEL16x6_SUB @@ -1738,53 +1738,53 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x6_SUB KERNEL16x6_SUB - je .L7_16 + je L(7_16) - jmp .L7_12 + jmp L(7_12) ALIGN_4 -.L7_16: +L(7_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_19 + je L(7_19) ALIGN_4 -.L7_17: +L(7_17): KERNEL16x6_SUB - jnz .L7_17 + jnz L(7_17) ALIGN_4 -.L7_19: +L(7_19): SAVE16x6 addq $16 * SIZE, CO1 # coffset += 16 addq $16 * SIZE, CO2 # coffset += 16 decq I # i -- - jg .L7_11 + jg L(7_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L7_20: +L(7_20): // Test rest of M testq $15, M - jz .L7_60 // to next 6 lines of N + jz L(7_60) // to next 6 lines of N testq $8, M - jz .L7_21pre + jz L(7_21pre) ALIGN_4 /**************************************************************************/ -.L7_20_1: +L(7_20_1): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1793,11 +1793,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_20_6 + je L(7_20_6) ALIGN_4 -.L7_20_2: +L(7_20_2): prefetcht0 A_PR1(AO) KERNEL8x6_SUB @@ -1813,7 +1813,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x6_SUB KERNEL8x6_SUB - je .L7_20_6 + je L(7_20_6) prefetcht0 A_PR1(AO) KERNEL8x6_SUB @@ -1829,28 +1829,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x6_SUB KERNEL8x6_SUB - je .L7_20_6 + je L(7_20_6) - jmp .L7_20_2 + jmp L(7_20_2) ALIGN_4 -.L7_20_6: +L(7_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_20_9 + je L(7_20_9) ALIGN_4 -.L7_20_7: +L(7_20_7): KERNEL8x6_SUB - jnz .L7_20_7 + jnz L(7_20_7) ALIGN_4 -.L7_20_9: +L(7_20_9): SAVE8x6 @@ -1862,13 +1862,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L7_21pre: +L(7_21pre): testq $4, M - jz .L7_30 + jz L(7_30) ALIGN_4 -.L7_21: +L(7_21): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1877,11 +1877,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_26 + je L(7_26) ALIGN_4 -.L7_22: +L(7_22): prefetcht0 A_PR1(AO) KERNEL4x6_SUB @@ -1895,7 +1895,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x6_SUB KERNEL4x6_SUB - je .L7_26 + je L(7_26) prefetcht0 A_PR1(AO) KERNEL4x6_SUB @@ -1909,28 +1909,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x6_SUB KERNEL4x6_SUB - je .L7_26 + je L(7_26) - jmp .L7_22 + jmp L(7_22) ALIGN_4 -.L7_26: +L(7_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_29 + je L(7_29) ALIGN_4 -.L7_27: +L(7_27): KERNEL4x6_SUB - jnz .L7_27 + jnz L(7_27) ALIGN_4 -.L7_29: +L(7_29): SAVE4x6 @@ -1939,13 +1939,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L7_30: +L(7_30): testq $2, M - jz .L7_40 + jz L(7_40) ALIGN_4 -.L7_31: +L(7_31): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1954,11 +1954,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_36 + je L(7_36) ALIGN_4 -.L7_32: +L(7_32): prefetcht0 A_PR1(AO) KERNEL2x6_SUB @@ -1971,7 +1971,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x6_SUB KERNEL2x6_SUB - je .L7_36 + je L(7_36) prefetcht0 A_PR1(AO) KERNEL2x6_SUB @@ -1984,28 +1984,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x6_SUB KERNEL2x6_SUB - je .L7_36 + je L(7_36) - jmp .L7_32 + jmp L(7_32) ALIGN_4 -.L7_36: +L(7_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_39 + je L(7_39) ALIGN_4 -.L7_37: +L(7_37): KERNEL2x6_SUB - jnz .L7_37 + jnz L(7_37) ALIGN_4 -.L7_39: +L(7_39): SAVE2x6 @@ -2013,13 +2013,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO2 # coffset += 2 ALIGN_4 -.L7_40: +L(7_40): testq $1, M - jz .L7_60 // to next 4 lines of N + jz L(7_60) // to next 4 lines of N ALIGN_4 -.L7_41: +L(7_41): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2028,11 +2028,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_46 + je L(7_46) ALIGN_4 -.L7_42: +L(7_42): prefetcht0 A_PR1(AO) KERNEL1x6_SUB @@ -2045,7 +2045,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x6_SUB KERNEL1x6_SUB - je .L7_46 + je L(7_46) KERNEL1x6_SUB KERNEL1x6_SUB @@ -2057,28 +2057,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x6_SUB KERNEL1x6_SUB - je .L7_46 + je L(7_46) - jmp .L7_42 + jmp L(7_42) ALIGN_4 -.L7_46: +L(7_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_49 + je L(7_49) ALIGN_4 -.L7_47: +L(7_47): KERNEL1x6_SUB - jnz .L7_47 + jnz L(7_47) ALIGN_4 -.L7_49: +L(7_49): SAVE1x6 @@ -2090,35 +2090,35 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L7_60: +L(7_60): decq J // j -- - jg .L6_01 // next 12 lines of N + jg L(6_01) // next 12 lines of N /*******************************************************************************************/ -.L4_00: +L(4_00): movq Nmod6, J sarq $2, J // j = j / 4 cmpq $ 0, J - je .L2_00 + je L(2_00) ALIGN_4 -.L4_01: +L(4_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L4_01b + jz L(4_01b) ALIGN_4 -.L4_01a: +L(4_01a): prefetcht0 512(BO1) prefetchw 512(BO) @@ -2135,30 +2135,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 16*SIZE,BO1 addq $ 16*SIZE,BO decq %rax - jnz .L4_01a + jnz L(4_01a) -.L4_01b: +L(4_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L4_02d + jz L(4_02d) ALIGN_4 -.L4_02c: +L(4_02c): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $ 4*SIZE,BO1 addq $ 4*SIZE,BO decq %rax - jnz .L4_02c + jnz L(4_02c) -.L4_02d: +L(4_02d): movq BO1, B // next offset of B -.L4_10: +L(4_10): movq C, CO1 leaq (C, LDC, 2), CO2 leaq (C, LDC, 4), C // c += 4 * ldc @@ -2173,11 +2173,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L4_20 + je L(4_20) ALIGN_4 -.L4_11: +L(4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2214,7 +2214,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L4_16 + je L(4_16) movq %rax, BI // Index for BO leaq (,BI,4) , BI // BI = BI * 4 ; number of values @@ -2225,7 +2225,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_12: +L(4_12): prefetcht0 A_PR1(AO, %rax, SIZE) prefetcht0 B_PR1(BO, BI , SIZE) @@ -2247,7 +2247,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO, %rax, SIZE) KERNEL16x4_SUB - je .L4_16 + je L(4_16) prefetcht0 A_PR1(AO, %rax, SIZE) prefetcht0 B_PR1(BO, BI , SIZE) @@ -2269,12 +2269,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO, %rax, SIZE) KERNEL16x4_SUB - je .L4_16 + je L(4_16) - jmp .L4_12 + jmp L(4_12) ALIGN_4 -.L4_16: +L(4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -2282,7 +2282,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_19 + je L(4_19) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2294,15 +2294,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_17: +L(4_17): KERNEL16x4_SUB - jl .L4_17 + jl L(4_17) ALIGN_4 -.L4_19: +L(4_19): SAVE16x4 @@ -2325,25 +2325,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 addq $16 * SIZE, CO2 # coffset += 16 decq I # i -- - jg .L4_11 + jg L(4_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L4_20: +L(4_20): // Test rest of M testq $15, M - jz .L4_60 // to next 3 lines of N + jz L(4_60) // to next 3 lines of N testq $8, M - jz .L4_21pre + jz L(4_21pre) ALIGN_4 /**************************************************************************/ -.L4_20_1: +L(4_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2381,7 +2381,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_20_6 + je L(4_20_6) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2392,7 +2392,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_20_2: +L(4_20_2): KERNEL8x4_SUB KERNEL8x4_SUB @@ -2404,7 +2404,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_SUB KERNEL8x4_SUB - je .L4_20_6 + je L(4_20_6) KERNEL8x4_SUB KERNEL8x4_SUB @@ -2416,12 +2416,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_SUB KERNEL8x4_SUB - je .L4_20_6 + je L(4_20_6) - jmp .L4_20_2 + jmp L(4_20_2) ALIGN_4 -.L4_20_6: +L(4_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -2429,7 +2429,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_20_9 + je L(4_20_9) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2441,15 +2441,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_20_7: +L(4_20_7): KERNEL8x4_SUB - jl .L4_20_7 + jl L(4_20_7) ALIGN_4 -.L4_20_9: +L(4_20_9): SAVE8x4 @@ -2477,13 +2477,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L4_21pre: +L(4_21pre): testq $4, M - jz .L4_30 + jz L(4_30) ALIGN_4 -.L4_21: +L(4_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2521,7 +2521,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_26 + je L(4_26) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2532,7 +2532,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_22: +L(4_22): KERNEL4x4_SUB KERNEL4x4_SUB @@ -2544,7 +2544,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB KERNEL4x4_SUB - je .L4_26 + je L(4_26) KERNEL4x4_SUB KERNEL4x4_SUB @@ -2556,12 +2556,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB KERNEL4x4_SUB - je .L4_26 + je L(4_26) - jmp .L4_22 + jmp L(4_22) ALIGN_4 -.L4_26: +L(4_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -2569,7 +2569,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_29 + je L(4_29) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2581,15 +2581,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_27: +L(4_27): KERNEL4x4_SUB - jl .L4_27 + jl L(4_27) ALIGN_4 -.L4_29: +L(4_29): SAVE4x4 @@ -2614,13 +2614,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L4_30: +L(4_30): testq $2, M - jz .L4_40 + jz L(4_40) ALIGN_4 -.L4_31: +L(4_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2658,7 +2658,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_36 + je L(4_36) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2669,7 +2669,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_32: +L(4_32): KERNEL2x4_SUB KERNEL2x4_SUB @@ -2681,7 +2681,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB KERNEL2x4_SUB - je .L4_36 + je L(4_36) KERNEL2x4_SUB KERNEL2x4_SUB @@ -2693,12 +2693,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB KERNEL2x4_SUB - je .L4_36 + je L(4_36) - jmp .L4_32 + jmp L(4_32) ALIGN_4 -.L4_36: +L(4_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -2706,7 +2706,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_39 + je L(4_39) movq %rax, BI // Index for BO leaq (,BI, 4), BI // BI = BI * 4 ; number of values @@ -2718,15 +2718,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_37: +L(4_37): KERNEL2x4_SUB - jl .L4_37 + jl L(4_37) ALIGN_4 -.L4_39: +L(4_39): SAVE2x4 @@ -2750,13 +2750,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO2 # coffset += 2 ALIGN_4 -.L4_40: +L(4_40): testq $1, M - jz .L4_60 // to next 4 lines of N + jz L(4_60) // to next 4 lines of N ALIGN_4 -.L4_41: +L(4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2792,7 +2792,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L4_46 + je L(4_46) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2802,7 +2802,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_42: +L(4_42): KERNEL1x4_SUB KERNEL1x4_SUB @@ -2814,7 +2814,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB KERNEL1x4_SUB - je .L4_46 + je L(4_46) KERNEL1x4_SUB KERNEL1x4_SUB @@ -2826,12 +2826,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB KERNEL1x4_SUB - je .L4_46 + je L(4_46) - jmp .L4_42 + jmp L(4_42) ALIGN_4 -.L4_46: +L(4_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -2839,7 +2839,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_49 + je L(4_49) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2850,15 +2850,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_47: +L(4_47): KERNEL1x4_SUB - jl .L4_47 + jl L(4_47) ALIGN_4 -.L4_49: +L(4_49): SAVE1x4 @@ -2885,38 +2885,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L4_60: +L(4_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif decq J // j -- - jg .L4_01 // next 4 lines of N + jg L(4_01) // next 4 lines of N /*******************************************************************************************/ -.L2_00: +L(2_00): movq Nmod6, J andq $3, J // j % 4 - je .L999 + je L(999) movq Nmod6, J andq $2, J // j % 4 - je .L1_0 + je L(1_0) -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L2_01b + jz L(2_01b) ALIGN_4 -.L2_01a: +L(2_01a): vmovsd (BO1), %xmm0 vmovsd 2*SIZE(BO1), %xmm1 @@ -2931,30 +2931,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $8*SIZE,BO1 addq $8*SIZE,BO decq %rax - jnz .L2_01a + jnz L(2_01a) -.L2_01b: +L(2_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L2_02d + jz L(2_02d) ALIGN_4 -.L2_02c: +L(2_02c): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02c + jnz L(2_02c) -.L2_02d: +L(2_02d): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -2968,11 +2968,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3009,7 +3009,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3020,7 +3020,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_12: +L(2_12): KERNEL16x2_SUB KERNEL16x2_SUB @@ -3032,7 +3032,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB KERNEL16x2_SUB - je .L2_16 + je L(2_16) KERNEL16x2_SUB KERNEL16x2_SUB @@ -3044,12 +3044,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB KERNEL16x2_SUB - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3057,7 +3057,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3069,15 +3069,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE16x2 @@ -3099,25 +3099,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3155,7 +3155,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3166,7 +3166,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): KERNEL8x2_SUB @@ -3179,7 +3179,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB KERNEL8x2_SUB - je .L2_20_6 + je L(2_20_6) KERNEL8x2_SUB KERNEL8x2_SUB @@ -3191,12 +3191,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB KERNEL8x2_SUB - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -3204,7 +3204,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3216,15 +3216,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): SAVE8x2 @@ -3251,13 +3251,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3295,7 +3295,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -3306,7 +3306,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_22: +L(2_22): KERNEL4x2_SUB @@ -3319,7 +3319,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_26 + je L(2_26) KERNEL4x2_SUB KERNEL4x2_SUB @@ -3331,12 +3331,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -3344,7 +3344,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3356,15 +3356,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): SAVE4x2 @@ -3388,13 +3388,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3432,7 +3432,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3443,7 +3443,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_SUB KERNEL2x2_SUB @@ -3455,7 +3455,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_36 + je L(2_36) KERNEL2x2_SUB KERNEL2x2_SUB @@ -3467,12 +3467,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -3480,7 +3480,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3492,15 +3492,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): SAVE2x2 @@ -3523,13 +3523,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3565,7 +3565,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3575,7 +3575,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_SUB KERNEL1x2_SUB @@ -3587,7 +3587,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_46 + je L(2_46) KERNEL1x2_SUB KERNEL1x2_SUB @@ -3599,12 +3599,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -3612,7 +3612,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3623,15 +3623,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): SAVE1x2 @@ -3657,7 +3657,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -3665,7 +3665,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -3673,30 +3673,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovss (BO1), %xmm0 vmovss %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -3710,11 +3710,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3750,7 +3750,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -3760,7 +3760,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_12: +L(1_12): KERNEL16x1_SUB KERNEL16x1_SUB @@ -3772,7 +3772,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB KERNEL16x1_SUB - je .L1_16 + je L(1_16) KERNEL16x1_SUB KERNEL16x1_SUB @@ -3784,12 +3784,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB KERNEL16x1_SUB - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3797,7 +3797,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -3808,15 +3808,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE16x1 @@ -3837,25 +3837,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3892,7 +3892,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -3902,7 +3902,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): KERNEL8x1_SUB KERNEL8x1_SUB @@ -3914,7 +3914,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB KERNEL8x1_SUB - je .L1_20_6 + je L(1_20_6) KERNEL8x1_SUB KERNEL8x1_SUB @@ -3926,12 +3926,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB KERNEL8x1_SUB - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -3939,7 +3939,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -3950,15 +3950,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): SAVE8x1 @@ -3984,13 +3984,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4027,7 +4027,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -4037,7 +4037,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_22: +L(1_22): KERNEL4x1_SUB KERNEL4x1_SUB @@ -4049,7 +4049,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_26 + je L(1_26) KERNEL4x1_SUB KERNEL4x1_SUB @@ -4061,12 +4061,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4074,7 +4074,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -4085,15 +4085,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): SAVE4x1 @@ -4116,13 +4116,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4159,7 +4159,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -4169,7 +4169,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_SUB KERNEL2x1_SUB @@ -4181,7 +4181,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_36 + je L(1_36) KERNEL2x1_SUB KERNEL2x1_SUB @@ -4193,12 +4193,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -4206,7 +4206,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -4217,15 +4217,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): SAVE2x1 @@ -4247,13 +4247,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4288,7 +4288,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -4297,7 +4297,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_SUB KERNEL1x1_SUB @@ -4309,7 +4309,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_46 + je L(1_46) KERNEL1x1_SUB KERNEL1x1_SUB @@ -4321,12 +4321,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -4334,7 +4334,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -4344,15 +4344,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): SAVE1x1 @@ -4374,7 +4374,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp @@ -4466,13 +4466,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -4501,22 +4501,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv6, J cmpq $0, J - je .L2_0 + je L(2_0) ALIGN_4 /*******************************************************************************************/ -.L4_01: +L(4_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L4_01b + jz L(4_01b) ALIGN_4 -.L4_01a: +L(4_01a): prefetcht0 512(BO1) prefetchw 512(BO) @@ -4533,30 +4533,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 16*SIZE,BO1 addq $ 16*SIZE,BO decq %rax - jnz .L4_01a + jnz L(4_01a) -.L4_01b: +L(4_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L4_02d + jz L(4_02d) ALIGN_4 -.L4_02c: +L(4_02c): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $ 4*SIZE,BO1 addq $ 4*SIZE,BO decq %rax - jnz .L4_02c + jnz L(4_02c) -.L4_02d: +L(4_02d): movq BO1, B // next offset of B -.L4_10: +L(4_10): movq C, CO1 leaq (C, LDC, 2), CO2 leaq (C, LDC, 4), C // c += 4 * ldc @@ -4571,11 +4571,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L4_20 + je L(4_20) ALIGN_4 -.L4_11: +L(4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4612,7 +4612,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L4_16 + je L(4_16) movq %rax, BI // Index for BO leaq (,BI,4) , BI // BI = BI * 4 ; number of values @@ -4623,7 +4623,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_12: +L(4_12): prefetcht0 A_PR1(AO, %rax, SIZE) prefetcht0 B_PR1(BO, BI , SIZE) @@ -4645,7 +4645,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO, %rax, SIZE) KERNEL16x4_SUB - je .L4_16 + je L(4_16) prefetcht0 A_PR1(AO, %rax, SIZE) prefetcht0 B_PR1(BO, BI , SIZE) @@ -4667,12 +4667,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO, %rax, SIZE) KERNEL16x4_SUB - je .L4_16 + je L(4_16) - jmp .L4_12 + jmp L(4_12) ALIGN_4 -.L4_16: +L(4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -4680,7 +4680,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_19 + je L(4_19) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -4692,15 +4692,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_17: +L(4_17): KERNEL16x4_SUB - jl .L4_17 + jl L(4_17) ALIGN_4 -.L4_19: +L(4_19): SAVE16x4 @@ -4723,25 +4723,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 addq $16 * SIZE, CO2 # coffset += 16 decq I # i -- - jg .L4_11 + jg L(4_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L4_20: +L(4_20): // Test rest of M testq $15, M - jz .L4_60 // to next 3 lines of N + jz L(4_60) // to next 3 lines of N testq $8, M - jz .L4_21pre + jz L(4_21pre) ALIGN_4 /**************************************************************************/ -.L4_20_1: +L(4_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4779,7 +4779,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_20_6 + je L(4_20_6) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -4790,7 +4790,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_20_2: +L(4_20_2): KERNEL8x4_SUB KERNEL8x4_SUB @@ -4802,7 +4802,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_SUB KERNEL8x4_SUB - je .L4_20_6 + je L(4_20_6) KERNEL8x4_SUB KERNEL8x4_SUB @@ -4814,12 +4814,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_SUB KERNEL8x4_SUB - je .L4_20_6 + je L(4_20_6) - jmp .L4_20_2 + jmp L(4_20_2) ALIGN_4 -.L4_20_6: +L(4_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -4827,7 +4827,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_20_9 + je L(4_20_9) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -4839,15 +4839,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_20_7: +L(4_20_7): KERNEL8x4_SUB - jl .L4_20_7 + jl L(4_20_7) ALIGN_4 -.L4_20_9: +L(4_20_9): SAVE8x4 @@ -4875,13 +4875,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L4_21pre: +L(4_21pre): testq $4, M - jz .L4_30 + jz L(4_30) ALIGN_4 -.L4_21: +L(4_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4919,7 +4919,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_26 + je L(4_26) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -4930,7 +4930,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_22: +L(4_22): KERNEL4x4_SUB KERNEL4x4_SUB @@ -4942,7 +4942,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB KERNEL4x4_SUB - je .L4_26 + je L(4_26) KERNEL4x4_SUB KERNEL4x4_SUB @@ -4954,12 +4954,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB KERNEL4x4_SUB - je .L4_26 + je L(4_26) - jmp .L4_22 + jmp L(4_22) ALIGN_4 -.L4_26: +L(4_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4967,7 +4967,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_29 + je L(4_29) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -4979,15 +4979,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_27: +L(4_27): KERNEL4x4_SUB - jl .L4_27 + jl L(4_27) ALIGN_4 -.L4_29: +L(4_29): SAVE4x4 @@ -5012,13 +5012,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L4_30: +L(4_30): testq $2, M - jz .L4_40 + jz L(4_40) ALIGN_4 -.L4_31: +L(4_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5056,7 +5056,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_36 + je L(4_36) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -5067,7 +5067,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_32: +L(4_32): KERNEL2x4_SUB KERNEL2x4_SUB @@ -5079,7 +5079,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB KERNEL2x4_SUB - je .L4_36 + je L(4_36) KERNEL2x4_SUB KERNEL2x4_SUB @@ -5091,12 +5091,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB KERNEL2x4_SUB - je .L4_36 + je L(4_36) - jmp .L4_32 + jmp L(4_32) ALIGN_4 -.L4_36: +L(4_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -5104,7 +5104,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_39 + je L(4_39) movq %rax, BI // Index for BO leaq (,BI, 4), BI // BI = BI * 4 ; number of values @@ -5116,15 +5116,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_37: +L(4_37): KERNEL2x4_SUB - jl .L4_37 + jl L(4_37) ALIGN_4 -.L4_39: +L(4_39): SAVE2x4 @@ -5148,13 +5148,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO2 # coffset += 2 ALIGN_4 -.L4_40: +L(4_40): testq $1, M - jz .L4_60 // to next 4 lines of N + jz L(4_60) // to next 4 lines of N ALIGN_4 -.L4_41: +L(4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5190,7 +5190,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L4_46 + je L(4_46) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -5200,7 +5200,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_42: +L(4_42): KERNEL1x4_SUB KERNEL1x4_SUB @@ -5212,7 +5212,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB KERNEL1x4_SUB - je .L4_46 + je L(4_46) KERNEL1x4_SUB KERNEL1x4_SUB @@ -5224,12 +5224,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB KERNEL1x4_SUB - je .L4_46 + je L(4_46) - jmp .L4_42 + jmp L(4_42) ALIGN_4 -.L4_46: +L(4_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -5237,7 +5237,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_49 + je L(4_49) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -5248,15 +5248,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_47: +L(4_47): KERNEL1x4_SUB - jl .L4_47 + jl L(4_47) ALIGN_4 -.L4_49: +L(4_49): SAVE1x4 @@ -5283,38 +5283,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L4_60: +L(4_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif decq J // j -- - jg .L4_01 // next 4 lines of N + jg L(4_01) // next 4 lines of N /*******************************************************************************************/ -.L2_0: +L(2_0): movq Nmod6, J andq $3, J // j % 4 - je .L999 + je L(999) movq Nmod6, J andq $2, J // j % 4 - je .L1_0 + je L(1_0) -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L2_01b + jz L(2_01b) ALIGN_4 -.L2_01a: +L(2_01a): vmovsd (BO1), %xmm0 vmovsd 2*SIZE(BO1), %xmm1 @@ -5329,30 +5329,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $8*SIZE,BO1 addq $8*SIZE,BO decq %rax - jnz .L2_01a + jnz L(2_01a) -.L2_01b: +L(2_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L2_02d + jz L(2_02d) ALIGN_4 -.L2_02c: +L(2_02c): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02c + jnz L(2_02c) -.L2_02d: +L(2_02d): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -5366,11 +5366,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5407,7 +5407,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5418,7 +5418,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_12: +L(2_12): KERNEL16x2_SUB KERNEL16x2_SUB @@ -5430,7 +5430,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB KERNEL16x2_SUB - je .L2_16 + je L(2_16) KERNEL16x2_SUB KERNEL16x2_SUB @@ -5442,12 +5442,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB KERNEL16x2_SUB - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -5455,7 +5455,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5467,15 +5467,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE16x2 @@ -5497,25 +5497,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5553,7 +5553,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5564,7 +5564,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): KERNEL8x2_SUB @@ -5577,7 +5577,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB KERNEL8x2_SUB - je .L2_20_6 + je L(2_20_6) KERNEL8x2_SUB KERNEL8x2_SUB @@ -5589,12 +5589,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB KERNEL8x2_SUB - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -5602,7 +5602,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5614,15 +5614,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): SAVE8x2 @@ -5649,13 +5649,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5693,7 +5693,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -5704,7 +5704,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_22: +L(2_22): KERNEL4x2_SUB @@ -5717,7 +5717,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_26 + je L(2_26) KERNEL4x2_SUB KERNEL4x2_SUB @@ -5729,12 +5729,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -5742,7 +5742,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5754,15 +5754,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): SAVE4x2 @@ -5786,13 +5786,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5830,7 +5830,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5841,7 +5841,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_SUB KERNEL2x2_SUB @@ -5853,7 +5853,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_36 + je L(2_36) KERNEL2x2_SUB KERNEL2x2_SUB @@ -5865,12 +5865,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -5878,7 +5878,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5890,15 +5890,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): SAVE2x2 @@ -5921,13 +5921,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5963,7 +5963,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5973,7 +5973,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_SUB KERNEL1x2_SUB @@ -5985,7 +5985,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_46 + je L(2_46) KERNEL1x2_SUB KERNEL1x2_SUB @@ -5997,12 +5997,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -6010,7 +6010,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -6021,15 +6021,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): SAVE1x2 @@ -6055,7 +6055,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -6063,7 +6063,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -6071,30 +6071,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovss (BO1), %xmm0 vmovss %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -6108,11 +6108,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -6148,7 +6148,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -6158,7 +6158,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_12: +L(1_12): KERNEL16x1_SUB KERNEL16x1_SUB @@ -6170,7 +6170,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB KERNEL16x1_SUB - je .L1_16 + je L(1_16) KERNEL16x1_SUB KERNEL16x1_SUB @@ -6182,12 +6182,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB KERNEL16x1_SUB - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -6195,7 +6195,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -6206,15 +6206,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE16x1 @@ -6235,25 +6235,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -6290,7 +6290,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -6300,7 +6300,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): KERNEL8x1_SUB KERNEL8x1_SUB @@ -6312,7 +6312,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB KERNEL8x1_SUB - je .L1_20_6 + je L(1_20_6) KERNEL8x1_SUB KERNEL8x1_SUB @@ -6324,12 +6324,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB KERNEL8x1_SUB - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -6337,7 +6337,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -6348,15 +6348,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): SAVE8x1 @@ -6382,13 +6382,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -6425,7 +6425,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -6435,7 +6435,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_22: +L(1_22): KERNEL4x1_SUB KERNEL4x1_SUB @@ -6447,7 +6447,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_26 + je L(1_26) KERNEL4x1_SUB KERNEL4x1_SUB @@ -6459,12 +6459,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -6472,7 +6472,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -6483,15 +6483,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): SAVE4x1 @@ -6514,13 +6514,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -6557,7 +6557,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -6567,7 +6567,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_SUB KERNEL2x1_SUB @@ -6579,7 +6579,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_36 + je L(1_36) KERNEL2x1_SUB KERNEL2x1_SUB @@ -6591,12 +6591,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -6604,7 +6604,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -6615,15 +6615,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): SAVE2x1 @@ -6645,13 +6645,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -6686,7 +6686,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -6695,7 +6695,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_SUB KERNEL1x1_SUB @@ -6707,7 +6707,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_46 + je L(1_46) KERNEL1x1_SUB KERNEL1x1_SUB @@ -6719,12 +6719,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -6732,7 +6732,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -6742,15 +6742,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): SAVE1x1 @@ -6772,7 +6772,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/sgemm_kernel_16x4_sandy.S b/kernel/x86_64/sgemm_kernel_16x4_sandy.S index 2ee4b15548..448921a2cc 100644 --- a/kernel/x86_64/sgemm_kernel_16x4_sandy.S +++ b/kernel/x86_64/sgemm_kernel_16x4_sandy.S @@ -826,13 +826,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -861,22 +861,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv6, J cmpq $0, J - je .L2_0 + je L(2_0) ALIGN_4 /*******************************************************************************************/ -.L4_01: +L(4_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L4_01b + jz L(4_01b) ALIGN_4 -.L4_01a: +L(4_01a): prefetcht0 512(BO1) prefetchw 512(BO) @@ -893,30 +893,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 16*SIZE,BO1 addq $ 16*SIZE,BO decq %rax - jnz .L4_01a + jnz L(4_01a) -.L4_01b: +L(4_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L4_02d + jz L(4_02d) ALIGN_4 -.L4_02c: +L(4_02c): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $ 4*SIZE,BO1 addq $ 4*SIZE,BO decq %rax - jnz .L4_02c + jnz L(4_02c) -.L4_02d: +L(4_02d): movq BO1, B // next offset of B -.L4_10: +L(4_10): movq C, CO1 leaq (C, LDC, 2), CO2 leaq (C, LDC, 4), C // c += 4 * ldc @@ -931,11 +931,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L4_20 + je L(4_20) ALIGN_4 -.L4_11: +L(4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -972,7 +972,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L4_16 + je L(4_16) movq %rax, BI // Index for BO leaq (,BI,4) , BI // BI = BI * 4 ; number of values @@ -983,7 +983,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_12: +L(4_12): prefetcht0 A_PR1(AO, %rax, SIZE) prefetcht0 B_PR1(BO, BI , SIZE) @@ -1005,7 +1005,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO, %rax, SIZE) KERNEL16x4_SUB - je .L4_16 + je L(4_16) prefetcht0 A_PR1(AO, %rax, SIZE) prefetcht0 B_PR1(BO, BI , SIZE) @@ -1027,12 +1027,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO, %rax, SIZE) KERNEL16x4_SUB - je .L4_16 + je L(4_16) - jmp .L4_12 + jmp L(4_12) ALIGN_4 -.L4_16: +L(4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -1040,7 +1040,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_19 + je L(4_19) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -1052,15 +1052,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_17: +L(4_17): KERNEL16x4_SUB - jl .L4_17 + jl L(4_17) ALIGN_4 -.L4_19: +L(4_19): SAVE16x4 @@ -1083,25 +1083,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 addq $16 * SIZE, CO2 # coffset += 16 decq I # i -- - jg .L4_11 + jg L(4_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L4_20: +L(4_20): // Test rest of M testq $15, M - jz .L4_60 // to next 3 lines of N + jz L(4_60) // to next 3 lines of N testq $8, M - jz .L4_21pre + jz L(4_21pre) ALIGN_4 /**************************************************************************/ -.L4_20_1: +L(4_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1139,7 +1139,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_20_6 + je L(4_20_6) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -1150,7 +1150,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_20_2: +L(4_20_2): KERNEL8x4_SUB KERNEL8x4_SUB @@ -1162,7 +1162,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_SUB KERNEL8x4_SUB - je .L4_20_6 + je L(4_20_6) KERNEL8x4_SUB KERNEL8x4_SUB @@ -1174,12 +1174,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_SUB KERNEL8x4_SUB - je .L4_20_6 + je L(4_20_6) - jmp .L4_20_2 + jmp L(4_20_2) ALIGN_4 -.L4_20_6: +L(4_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -1187,7 +1187,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_20_9 + je L(4_20_9) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -1199,15 +1199,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_20_7: +L(4_20_7): KERNEL8x4_SUB - jl .L4_20_7 + jl L(4_20_7) ALIGN_4 -.L4_20_9: +L(4_20_9): SAVE8x4 @@ -1235,13 +1235,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L4_21pre: +L(4_21pre): testq $4, M - jz .L4_30 + jz L(4_30) ALIGN_4 -.L4_21: +L(4_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1279,7 +1279,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_26 + je L(4_26) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -1290,7 +1290,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_22: +L(4_22): KERNEL4x4_SUB KERNEL4x4_SUB @@ -1302,7 +1302,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB KERNEL4x4_SUB - je .L4_26 + je L(4_26) KERNEL4x4_SUB KERNEL4x4_SUB @@ -1314,12 +1314,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB KERNEL4x4_SUB - je .L4_26 + je L(4_26) - jmp .L4_22 + jmp L(4_22) ALIGN_4 -.L4_26: +L(4_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -1327,7 +1327,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_29 + je L(4_29) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -1339,15 +1339,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_27: +L(4_27): KERNEL4x4_SUB - jl .L4_27 + jl L(4_27) ALIGN_4 -.L4_29: +L(4_29): SAVE4x4 @@ -1372,13 +1372,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L4_30: +L(4_30): testq $2, M - jz .L4_40 + jz L(4_40) ALIGN_4 -.L4_31: +L(4_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1416,7 +1416,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_36 + je L(4_36) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -1427,7 +1427,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_32: +L(4_32): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1439,7 +1439,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB KERNEL2x4_SUB - je .L4_36 + je L(4_36) KERNEL2x4_SUB KERNEL2x4_SUB @@ -1451,12 +1451,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB KERNEL2x4_SUB - je .L4_36 + je L(4_36) - jmp .L4_32 + jmp L(4_32) ALIGN_4 -.L4_36: +L(4_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -1464,7 +1464,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_39 + je L(4_39) movq %rax, BI // Index for BO leaq (,BI, 4), BI // BI = BI * 4 ; number of values @@ -1476,15 +1476,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_37: +L(4_37): KERNEL2x4_SUB - jl .L4_37 + jl L(4_37) ALIGN_4 -.L4_39: +L(4_39): SAVE2x4 @@ -1508,13 +1508,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO2 # coffset += 2 ALIGN_4 -.L4_40: +L(4_40): testq $1, M - jz .L4_60 // to next 4 lines of N + jz L(4_60) // to next 4 lines of N ALIGN_4 -.L4_41: +L(4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1550,7 +1550,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L4_46 + je L(4_46) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -1560,7 +1560,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_42: +L(4_42): KERNEL1x4_SUB KERNEL1x4_SUB @@ -1572,7 +1572,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB KERNEL1x4_SUB - je .L4_46 + je L(4_46) KERNEL1x4_SUB KERNEL1x4_SUB @@ -1584,12 +1584,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB KERNEL1x4_SUB - je .L4_46 + je L(4_46) - jmp .L4_42 + jmp L(4_42) ALIGN_4 -.L4_46: +L(4_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -1597,7 +1597,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_49 + je L(4_49) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -1608,15 +1608,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_47: +L(4_47): KERNEL1x4_SUB - jl .L4_47 + jl L(4_47) ALIGN_4 -.L4_49: +L(4_49): SAVE1x4 @@ -1643,38 +1643,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L4_60: +L(4_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif decq J // j -- - jg .L4_01 // next 4 lines of N + jg L(4_01) // next 4 lines of N /*******************************************************************************************/ -.L2_0: +L(2_0): movq Nmod6, J andq $3, J // j % 4 - je .L999 + je L(999) movq Nmod6, J andq $2, J // j % 4 - je .L1_0 + je L(1_0) -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L2_01b + jz L(2_01b) ALIGN_4 -.L2_01a: +L(2_01a): vmovsd (BO1), %xmm0 vmovsd 2*SIZE(BO1), %xmm1 @@ -1689,30 +1689,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $8*SIZE,BO1 addq $8*SIZE,BO decq %rax - jnz .L2_01a + jnz L(2_01a) -.L2_01b: +L(2_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L2_02d + jz L(2_02d) ALIGN_4 -.L2_02c: +L(2_02c): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02c + jnz L(2_02c) -.L2_02d: +L(2_02d): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -1726,11 +1726,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1767,7 +1767,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -1778,7 +1778,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_12: +L(2_12): KERNEL16x2_SUB KERNEL16x2_SUB @@ -1790,7 +1790,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB KERNEL16x2_SUB - je .L2_16 + je L(2_16) KERNEL16x2_SUB KERNEL16x2_SUB @@ -1802,12 +1802,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB KERNEL16x2_SUB - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -1815,7 +1815,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -1827,15 +1827,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE16x2 @@ -1857,25 +1857,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1913,7 +1913,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -1924,7 +1924,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): KERNEL8x2_SUB @@ -1937,7 +1937,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB KERNEL8x2_SUB - je .L2_20_6 + je L(2_20_6) KERNEL8x2_SUB KERNEL8x2_SUB @@ -1949,12 +1949,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB KERNEL8x2_SUB - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -1962,7 +1962,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -1974,15 +1974,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): SAVE8x2 @@ -2009,13 +2009,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2053,7 +2053,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -2064,7 +2064,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_22: +L(2_22): KERNEL4x2_SUB @@ -2077,7 +2077,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_26 + je L(2_26) KERNEL4x2_SUB KERNEL4x2_SUB @@ -2089,12 +2089,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -2102,7 +2102,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2114,15 +2114,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): SAVE4x2 @@ -2146,13 +2146,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2190,7 +2190,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2201,7 +2201,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_SUB KERNEL2x2_SUB @@ -2213,7 +2213,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_36 + je L(2_36) KERNEL2x2_SUB KERNEL2x2_SUB @@ -2225,12 +2225,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -2238,7 +2238,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2250,15 +2250,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): SAVE2x2 @@ -2281,13 +2281,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2323,7 +2323,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2333,7 +2333,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_SUB KERNEL1x2_SUB @@ -2345,7 +2345,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_46 + je L(2_46) KERNEL1x2_SUB KERNEL1x2_SUB @@ -2357,12 +2357,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -2370,7 +2370,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -2381,15 +2381,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): SAVE1x2 @@ -2415,7 +2415,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -2423,7 +2423,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -2431,30 +2431,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovss (BO1), %xmm0 vmovss %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -2468,11 +2468,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2508,7 +2508,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -2518,7 +2518,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_12: +L(1_12): KERNEL16x1_SUB KERNEL16x1_SUB @@ -2530,7 +2530,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB KERNEL16x1_SUB - je .L1_16 + je L(1_16) KERNEL16x1_SUB KERNEL16x1_SUB @@ -2542,12 +2542,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB KERNEL16x1_SUB - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -2555,7 +2555,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -2566,15 +2566,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE16x1 @@ -2595,25 +2595,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2650,7 +2650,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -2660,7 +2660,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): KERNEL8x1_SUB KERNEL8x1_SUB @@ -2672,7 +2672,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB KERNEL8x1_SUB - je .L1_20_6 + je L(1_20_6) KERNEL8x1_SUB KERNEL8x1_SUB @@ -2684,12 +2684,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB KERNEL8x1_SUB - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -2697,7 +2697,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -2708,15 +2708,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): SAVE8x1 @@ -2742,13 +2742,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2785,7 +2785,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -2795,7 +2795,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_22: +L(1_22): KERNEL4x1_SUB KERNEL4x1_SUB @@ -2807,7 +2807,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_26 + je L(1_26) KERNEL4x1_SUB KERNEL4x1_SUB @@ -2819,12 +2819,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -2832,7 +2832,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -2843,15 +2843,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): SAVE4x1 @@ -2874,13 +2874,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2917,7 +2917,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -2927,7 +2927,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_SUB KERNEL2x1_SUB @@ -2939,7 +2939,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_36 + je L(1_36) KERNEL2x1_SUB KERNEL2x1_SUB @@ -2951,12 +2951,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -2964,7 +2964,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -2975,15 +2975,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): SAVE2x1 @@ -3005,13 +3005,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3046,7 +3046,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -3055,7 +3055,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_SUB KERNEL1x1_SUB @@ -3067,7 +3067,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_46 + je L(1_46) KERNEL1x1_SUB KERNEL1x1_SUB @@ -3079,12 +3079,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -3092,7 +3092,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -3102,15 +3102,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): SAVE1x1 @@ -3132,7 +3132,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/sgemm_kernel_16x4_skylakex.S b/kernel/x86_64/sgemm_kernel_16x4_skylakex.S index ac4421252d..73c8239adb 100644 --- a/kernel/x86_64/sgemm_kernel_16x4_skylakex.S +++ b/kernel/x86_64/sgemm_kernel_16x4_skylakex.S @@ -1220,13 +1220,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -1245,13 +1245,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv6, J cmpq $0, J - je .L4_00 + je L(4_00) ALIGN_4 /*******************************************************************************************/ -.L6_01: +L(6_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO @@ -1264,7 +1264,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L6_02c: +L(6_02c): vmovups (BO1), %xmm0 vmovsd (BO2), %xmm1 @@ -1274,10 +1274,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4*SIZE,BO2 addq $ 6*SIZE,BO decq %rax - jnz .L6_02c + jnz L(6_02c) -.L6_10: +L(6_10): movq C, CO1 leaq (C, LDC, 2), CO2 leaq (CO2, LDC, 1), CO2 // co2 = c + 3 * ldc @@ -1289,11 +1289,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L6_20 + je L(6_20) ALIGN_4 -.L6_11: +L(6_11): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1302,69 +1302,69 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L6_16 + je L(6_16) ALIGN_4 -.L6_12: +L(6_12): KERNEL16x6_SUB4 KERNEL16x6_SUB4 - je .L6_16 + je L(6_16) KERNEL16x6_SUB4 KERNEL16x6_SUB4 - je .L6_16 + je L(6_16) - jmp .L6_12 + jmp L(6_12) ALIGN_4 -.L6_16: +L(6_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_19 + je L(6_19) ALIGN_4 -.L6_17: +L(6_17): KERNEL16x6_SUB - jnz .L6_17 + jnz L(6_17) ALIGN_4 -.L6_19: +L(6_19): SAVE16x6 addq $16 * SIZE, CO1 # coffset += 16 addq $16 * SIZE, CO2 # coffset += 16 decq I # i -- - jg .L6_11 + jg L(6_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L6_20: +L(6_20): // Test rest of M testq $15, M - jz .L6_60 // to next 6 lines of N + jz L(6_60) // to next 6 lines of N testq $8, M - jz .L6_21pre + jz L(6_21pre) ALIGN_4 /**************************************************************************/ -.L6_20_1: +L(6_20_1): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1373,11 +1373,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_20_6 + je L(6_20_6) ALIGN_4 -.L6_20_2: +L(6_20_2): prefetcht0 A_PR1(AO) KERNEL8x6_SUB @@ -1393,7 +1393,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x6_SUB KERNEL8x6_SUB - je .L6_20_6 + je L(6_20_6) prefetcht0 A_PR1(AO) KERNEL8x6_SUB @@ -1409,28 +1409,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x6_SUB KERNEL8x6_SUB - je .L6_20_6 + je L(6_20_6) - jmp .L6_20_2 + jmp L(6_20_2) ALIGN_4 -.L6_20_6: +L(6_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_20_9 + je L(6_20_9) ALIGN_4 -.L6_20_7: +L(6_20_7): KERNEL8x6_SUB - jnz .L6_20_7 + jnz L(6_20_7) ALIGN_4 -.L6_20_9: +L(6_20_9): SAVE8x6 @@ -1442,13 +1442,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L6_21pre: +L(6_21pre): testq $4, M - jz .L6_30 + jz L(6_30) ALIGN_4 -.L6_21: +L(6_21): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1457,11 +1457,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_26 + je L(6_26) ALIGN_4 -.L6_22: +L(6_22): prefetcht0 A_PR1(AO) KERNEL4x6_SUB @@ -1475,7 +1475,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x6_SUB KERNEL4x6_SUB - je .L6_26 + je L(6_26) prefetcht0 A_PR1(AO) KERNEL4x6_SUB @@ -1489,28 +1489,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x6_SUB KERNEL4x6_SUB - je .L6_26 + je L(6_26) - jmp .L6_22 + jmp L(6_22) ALIGN_4 -.L6_26: +L(6_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_29 + je L(6_29) ALIGN_4 -.L6_27: +L(6_27): KERNEL4x6_SUB - jnz .L6_27 + jnz L(6_27) ALIGN_4 -.L6_29: +L(6_29): SAVE4x6 @@ -1519,13 +1519,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L6_30: +L(6_30): testq $2, M - jz .L6_40 + jz L(6_40) ALIGN_4 -.L6_31: +L(6_31): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1534,11 +1534,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_36 + je L(6_36) ALIGN_4 -.L6_32: +L(6_32): prefetcht0 A_PR1(AO) KERNEL2x6_SUB @@ -1551,7 +1551,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x6_SUB KERNEL2x6_SUB - je .L6_36 + je L(6_36) prefetcht0 A_PR1(AO) KERNEL2x6_SUB @@ -1564,28 +1564,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x6_SUB KERNEL2x6_SUB - je .L6_36 + je L(6_36) - jmp .L6_32 + jmp L(6_32) ALIGN_4 -.L6_36: +L(6_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_39 + je L(6_39) ALIGN_4 -.L6_37: +L(6_37): KERNEL2x6_SUB - jnz .L6_37 + jnz L(6_37) ALIGN_4 -.L6_39: +L(6_39): SAVE2x6 @@ -1593,13 +1593,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO2 # coffset += 2 ALIGN_4 -.L6_40: +L(6_40): testq $1, M - jz .L6_60 // to next 4 lines of N + jz L(6_60) // to next 4 lines of N ALIGN_4 -.L6_41: +L(6_41): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1608,11 +1608,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L6_46 + je L(6_46) ALIGN_4 -.L6_42: +L(6_42): prefetcht0 A_PR1(AO) KERNEL1x6_SUB @@ -1625,7 +1625,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x6_SUB KERNEL1x6_SUB - je .L6_46 + je L(6_46) KERNEL1x6_SUB KERNEL1x6_SUB @@ -1637,28 +1637,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x6_SUB KERNEL1x6_SUB - je .L6_46 + je L(6_46) - jmp .L6_42 + jmp L(6_42) ALIGN_4 -.L6_46: +L(6_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L6_49 + je L(6_49) ALIGN_4 -.L6_47: +L(6_47): KERNEL1x6_SUB - jnz .L6_47 + jnz L(6_47) ALIGN_4 -.L6_49: +L(6_49): SAVE1x6 @@ -1670,13 +1670,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L6_60: +L(6_60): /*******************************************************************************************/ -.L7_01: +L(7_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO @@ -1688,7 +1688,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L7_02c: +L(7_02c): vmovsd 2*SIZE(BO1), %xmm0 vmovups (BO2), %xmm1 @@ -1698,11 +1698,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4*SIZE,BO2 addq $ 6*SIZE,BO decq %rax - jnz .L7_02c + jnz L(7_02c) movq BO2, B // next offset of B -.L7_10: +L(7_10): movq C, CO1 leaq (C, LDC, 2), CO2 leaq (CO2, LDC, 1), CO2 // co2 = c + 3 * ldc @@ -1714,11 +1714,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L7_20 + je L(7_20) ALIGN_4 -.L7_11: +L(7_11): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1727,69 +1727,69 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax // K = K - ( K % 8 ) - je .L7_16 + je L(7_16) ALIGN_4 -.L7_12: +L(7_12): KERNEL16x6_SUB4 KERNEL16x6_SUB4 - je .L7_16 + je L(7_16) KERNEL16x6_SUB4 KERNEL16x6_SUB4 - je .L7_16 + je L(7_16) - jmp .L7_12 + jmp L(7_12) ALIGN_4 -.L7_16: +L(7_16): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_19 + je L(7_19) ALIGN_4 -.L7_17: +L(7_17): KERNEL16x6_SUB - jnz .L7_17 + jnz L(7_17) ALIGN_4 -.L7_19: +L(7_19): SAVE16x6 addq $16 * SIZE, CO1 # coffset += 16 addq $16 * SIZE, CO2 # coffset += 16 decq I # i -- - jg .L7_11 + jg L(7_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L7_20: +L(7_20): // Test rest of M testq $15, M - jz .L7_60 // to next 6 lines of N + jz L(7_60) // to next 6 lines of N testq $8, M - jz .L7_21pre + jz L(7_21pre) ALIGN_4 /**************************************************************************/ -.L7_20_1: +L(7_20_1): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1798,11 +1798,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_20_6 + je L(7_20_6) ALIGN_4 -.L7_20_2: +L(7_20_2): prefetcht0 A_PR1(AO) KERNEL8x6_SUB @@ -1818,7 +1818,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x6_SUB KERNEL8x6_SUB - je .L7_20_6 + je L(7_20_6) prefetcht0 A_PR1(AO) KERNEL8x6_SUB @@ -1834,28 +1834,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x6_SUB KERNEL8x6_SUB - je .L7_20_6 + je L(7_20_6) - jmp .L7_20_2 + jmp L(7_20_2) ALIGN_4 -.L7_20_6: +L(7_20_6): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_20_9 + je L(7_20_9) ALIGN_4 -.L7_20_7: +L(7_20_7): KERNEL8x6_SUB - jnz .L7_20_7 + jnz L(7_20_7) ALIGN_4 -.L7_20_9: +L(7_20_9): SAVE8x6 @@ -1867,13 +1867,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L7_21pre: +L(7_21pre): testq $4, M - jz .L7_30 + jz L(7_30) ALIGN_4 -.L7_21: +L(7_21): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1882,11 +1882,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_26 + je L(7_26) ALIGN_4 -.L7_22: +L(7_22): prefetcht0 A_PR1(AO) KERNEL4x6_SUB @@ -1900,7 +1900,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x6_SUB KERNEL4x6_SUB - je .L7_26 + je L(7_26) prefetcht0 A_PR1(AO) KERNEL4x6_SUB @@ -1914,28 +1914,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x6_SUB KERNEL4x6_SUB - je .L7_26 + je L(7_26) - jmp .L7_22 + jmp L(7_22) ALIGN_4 -.L7_26: +L(7_26): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_29 + je L(7_29) ALIGN_4 -.L7_27: +L(7_27): KERNEL4x6_SUB - jnz .L7_27 + jnz L(7_27) ALIGN_4 -.L7_29: +L(7_29): SAVE4x6 @@ -1944,13 +1944,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L7_30: +L(7_30): testq $2, M - jz .L7_40 + jz L(7_40) ALIGN_4 -.L7_31: +L(7_31): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -1959,11 +1959,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_36 + je L(7_36) ALIGN_4 -.L7_32: +L(7_32): prefetcht0 A_PR1(AO) KERNEL2x6_SUB @@ -1976,7 +1976,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x6_SUB KERNEL2x6_SUB - je .L7_36 + je L(7_36) prefetcht0 A_PR1(AO) KERNEL2x6_SUB @@ -1989,28 +1989,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x6_SUB KERNEL2x6_SUB - je .L7_36 + je L(7_36) - jmp .L7_32 + jmp L(7_32) ALIGN_4 -.L7_36: +L(7_36): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_39 + je L(7_39) ALIGN_4 -.L7_37: +L(7_37): KERNEL2x6_SUB - jnz .L7_37 + jnz L(7_37) ALIGN_4 -.L7_39: +L(7_39): SAVE2x6 @@ -2018,13 +2018,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO2 # coffset += 2 ALIGN_4 -.L7_40: +L(7_40): testq $1, M - jz .L7_60 // to next 4 lines of N + jz L(7_60) // to next 4 lines of N ALIGN_4 -.L7_41: +L(7_41): leaq BUFFER1, BO // first buffer to BO addq $4 * SIZE, BO @@ -2033,11 +2033,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $-8, %rax - je .L7_46 + je L(7_46) ALIGN_4 -.L7_42: +L(7_42): prefetcht0 A_PR1(AO) KERNEL1x6_SUB @@ -2050,7 +2050,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x6_SUB KERNEL1x6_SUB - je .L7_46 + je L(7_46) KERNEL1x6_SUB KERNEL1x6_SUB @@ -2062,28 +2062,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x6_SUB KERNEL1x6_SUB - je .L7_46 + je L(7_46) - jmp .L7_42 + jmp L(7_42) ALIGN_4 -.L7_46: +L(7_46): movq K, %rax andq $7, %rax # if (k & 1) - je .L7_49 + je L(7_49) ALIGN_4 -.L7_47: +L(7_47): KERNEL1x6_SUB - jnz .L7_47 + jnz L(7_47) ALIGN_4 -.L7_49: +L(7_49): SAVE1x6 @@ -2095,35 +2095,35 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L7_60: +L(7_60): decq J // j -- - jg .L6_01 // next 12 lines of N + jg L(6_01) // next 12 lines of N /*******************************************************************************************/ -.L4_00: +L(4_00): movq Nmod6, J sarq $2, J // j = j / 4 cmpq $ 0, J - je .L2_00 + je L(2_00) ALIGN_4 -.L4_01: +L(4_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L4_01b + jz L(4_01b) ALIGN_4 -.L4_01a: +L(4_01a): prefetcht0 512(BO1) prefetchw 512(BO) @@ -2140,30 +2140,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 16*SIZE,BO1 addq $ 16*SIZE,BO decq %rax - jnz .L4_01a + jnz L(4_01a) -.L4_01b: +L(4_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L4_02d + jz L(4_02d) ALIGN_4 -.L4_02c: +L(4_02c): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $ 4*SIZE,BO1 addq $ 4*SIZE,BO decq %rax - jnz .L4_02c + jnz L(4_02c) -.L4_02d: +L(4_02d): movq BO1, B // next offset of B -.L4_10: +L(4_10): movq C, CO1 leaq (C, LDC, 2), CO2 leaq (C, LDC, 4), C // c += 4 * ldc @@ -2178,11 +2178,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L4_20 + je L(4_20) ALIGN_4 -.L4_11: +L(4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2219,7 +2219,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L4_16 + je L(4_16) movq %rax, BI // Index for BO leaq (,BI,4) , BI // BI = BI * 4 ; number of values @@ -2230,7 +2230,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_12: +L(4_12): prefetcht0 A_PR1(AO, %rax, SIZE) prefetcht0 B_PR1(BO, BI , SIZE) @@ -2252,7 +2252,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO, %rax, SIZE) KERNEL16x4_SUB - je .L4_16 + je L(4_16) prefetcht0 A_PR1(AO, %rax, SIZE) prefetcht0 B_PR1(BO, BI , SIZE) @@ -2274,12 +2274,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO, %rax, SIZE) KERNEL16x4_SUB - je .L4_16 + je L(4_16) - jmp .L4_12 + jmp L(4_12) ALIGN_4 -.L4_16: +L(4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -2287,7 +2287,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_19 + je L(4_19) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2299,15 +2299,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_17: +L(4_17): KERNEL16x4_SUB - jl .L4_17 + jl L(4_17) ALIGN_4 -.L4_19: +L(4_19): SAVE16x4 @@ -2330,25 +2330,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 addq $16 * SIZE, CO2 # coffset += 16 decq I # i -- - jg .L4_11 + jg L(4_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L4_20: +L(4_20): // Test rest of M testq $15, M - jz .L4_60 // to next 3 lines of N + jz L(4_60) // to next 3 lines of N testq $8, M - jz .L4_21pre + jz L(4_21pre) ALIGN_4 /**************************************************************************/ -.L4_20_1: +L(4_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2386,7 +2386,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_20_6 + je L(4_20_6) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2397,7 +2397,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_20_2: +L(4_20_2): KERNEL8x4_SUB KERNEL8x4_SUB @@ -2409,7 +2409,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_SUB KERNEL8x4_SUB - je .L4_20_6 + je L(4_20_6) KERNEL8x4_SUB KERNEL8x4_SUB @@ -2421,12 +2421,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_SUB KERNEL8x4_SUB - je .L4_20_6 + je L(4_20_6) - jmp .L4_20_2 + jmp L(4_20_2) ALIGN_4 -.L4_20_6: +L(4_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -2434,7 +2434,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_20_9 + je L(4_20_9) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2446,15 +2446,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_20_7: +L(4_20_7): KERNEL8x4_SUB - jl .L4_20_7 + jl L(4_20_7) ALIGN_4 -.L4_20_9: +L(4_20_9): SAVE8x4 @@ -2482,13 +2482,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L4_21pre: +L(4_21pre): testq $4, M - jz .L4_30 + jz L(4_30) ALIGN_4 -.L4_21: +L(4_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2526,7 +2526,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_26 + je L(4_26) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2537,7 +2537,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_22: +L(4_22): KERNEL4x4_SUB KERNEL4x4_SUB @@ -2549,7 +2549,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB KERNEL4x4_SUB - je .L4_26 + je L(4_26) KERNEL4x4_SUB KERNEL4x4_SUB @@ -2561,12 +2561,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB KERNEL4x4_SUB - je .L4_26 + je L(4_26) - jmp .L4_22 + jmp L(4_22) ALIGN_4 -.L4_26: +L(4_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -2574,7 +2574,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_29 + je L(4_29) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2586,15 +2586,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_27: +L(4_27): KERNEL4x4_SUB - jl .L4_27 + jl L(4_27) ALIGN_4 -.L4_29: +L(4_29): SAVE4x4 @@ -2619,13 +2619,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L4_30: +L(4_30): testq $2, M - jz .L4_40 + jz L(4_40) ALIGN_4 -.L4_31: +L(4_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2663,7 +2663,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_36 + je L(4_36) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2674,7 +2674,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_32: +L(4_32): KERNEL2x4_SUB KERNEL2x4_SUB @@ -2686,7 +2686,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB KERNEL2x4_SUB - je .L4_36 + je L(4_36) KERNEL2x4_SUB KERNEL2x4_SUB @@ -2698,12 +2698,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB KERNEL2x4_SUB - je .L4_36 + je L(4_36) - jmp .L4_32 + jmp L(4_32) ALIGN_4 -.L4_36: +L(4_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -2711,7 +2711,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_39 + je L(4_39) movq %rax, BI // Index for BO leaq (,BI, 4), BI // BI = BI * 4 ; number of values @@ -2723,15 +2723,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_37: +L(4_37): KERNEL2x4_SUB - jl .L4_37 + jl L(4_37) ALIGN_4 -.L4_39: +L(4_39): SAVE2x4 @@ -2755,13 +2755,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO2 # coffset += 2 ALIGN_4 -.L4_40: +L(4_40): testq $1, M - jz .L4_60 // to next 4 lines of N + jz L(4_60) // to next 4 lines of N ALIGN_4 -.L4_41: +L(4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2797,7 +2797,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L4_46 + je L(4_46) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2807,7 +2807,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_42: +L(4_42): KERNEL1x4_SUB KERNEL1x4_SUB @@ -2819,7 +2819,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB KERNEL1x4_SUB - je .L4_46 + je L(4_46) KERNEL1x4_SUB KERNEL1x4_SUB @@ -2831,12 +2831,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB KERNEL1x4_SUB - je .L4_46 + je L(4_46) - jmp .L4_42 + jmp L(4_42) ALIGN_4 -.L4_46: +L(4_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -2844,7 +2844,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_49 + je L(4_49) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -2855,15 +2855,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_47: +L(4_47): KERNEL1x4_SUB - jl .L4_47 + jl L(4_47) ALIGN_4 -.L4_49: +L(4_49): SAVE1x4 @@ -2890,38 +2890,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L4_60: +L(4_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif decq J // j -- - jg .L4_01 // next 4 lines of N + jg L(4_01) // next 4 lines of N /*******************************************************************************************/ -.L2_00: +L(2_00): movq Nmod6, J andq $3, J // j % 4 - je .L999 + je L(999) movq Nmod6, J andq $2, J // j % 4 - je .L1_0 + je L(1_0) -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L2_01b + jz L(2_01b) ALIGN_4 -.L2_01a: +L(2_01a): vmovsd (BO1), %xmm0 vmovsd 2*SIZE(BO1), %xmm1 @@ -2936,30 +2936,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $8*SIZE,BO1 addq $8*SIZE,BO decq %rax - jnz .L2_01a + jnz L(2_01a) -.L2_01b: +L(2_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L2_02d + jz L(2_02d) ALIGN_4 -.L2_02c: +L(2_02c): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02c + jnz L(2_02c) -.L2_02d: +L(2_02d): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -2973,11 +2973,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3014,7 +3014,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3025,7 +3025,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_12: +L(2_12): KERNEL16x2_SUB KERNEL16x2_SUB @@ -3037,7 +3037,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB KERNEL16x2_SUB - je .L2_16 + je L(2_16) KERNEL16x2_SUB KERNEL16x2_SUB @@ -3049,12 +3049,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB KERNEL16x2_SUB - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3062,7 +3062,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3074,15 +3074,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE16x2 @@ -3104,25 +3104,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3160,7 +3160,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3171,7 +3171,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): KERNEL8x2_SUB @@ -3184,7 +3184,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB KERNEL8x2_SUB - je .L2_20_6 + je L(2_20_6) KERNEL8x2_SUB KERNEL8x2_SUB @@ -3196,12 +3196,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB KERNEL8x2_SUB - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -3209,7 +3209,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3221,15 +3221,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): SAVE8x2 @@ -3256,13 +3256,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3300,7 +3300,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -3311,7 +3311,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_22: +L(2_22): KERNEL4x2_SUB @@ -3324,7 +3324,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_26 + je L(2_26) KERNEL4x2_SUB KERNEL4x2_SUB @@ -3336,12 +3336,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -3349,7 +3349,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3361,15 +3361,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): SAVE4x2 @@ -3393,13 +3393,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3437,7 +3437,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3448,7 +3448,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_SUB KERNEL2x2_SUB @@ -3460,7 +3460,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_36 + je L(2_36) KERNEL2x2_SUB KERNEL2x2_SUB @@ -3472,12 +3472,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -3485,7 +3485,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3497,15 +3497,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): SAVE2x2 @@ -3528,13 +3528,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3570,7 +3570,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3580,7 +3580,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_SUB KERNEL1x2_SUB @@ -3592,7 +3592,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_46 + je L(2_46) KERNEL1x2_SUB KERNEL1x2_SUB @@ -3604,12 +3604,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -3617,7 +3617,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -3628,15 +3628,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): SAVE1x2 @@ -3662,7 +3662,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -3670,7 +3670,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -3678,30 +3678,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovss (BO1), %xmm0 vmovss %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -3715,11 +3715,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3755,7 +3755,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -3765,7 +3765,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_12: +L(1_12): KERNEL16x1_SUB KERNEL16x1_SUB @@ -3777,7 +3777,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB KERNEL16x1_SUB - je .L1_16 + je L(1_16) KERNEL16x1_SUB KERNEL16x1_SUB @@ -3789,12 +3789,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB KERNEL16x1_SUB - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3802,7 +3802,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -3813,15 +3813,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE16x1 @@ -3842,25 +3842,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -3897,7 +3897,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -3907,7 +3907,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): KERNEL8x1_SUB KERNEL8x1_SUB @@ -3919,7 +3919,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB KERNEL8x1_SUB - je .L1_20_6 + je L(1_20_6) KERNEL8x1_SUB KERNEL8x1_SUB @@ -3931,12 +3931,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB KERNEL8x1_SUB - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -3944,7 +3944,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -3955,15 +3955,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): SAVE8x1 @@ -3989,13 +3989,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4032,7 +4032,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -4042,7 +4042,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_22: +L(1_22): KERNEL4x1_SUB KERNEL4x1_SUB @@ -4054,7 +4054,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_26 + je L(1_26) KERNEL4x1_SUB KERNEL4x1_SUB @@ -4066,12 +4066,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4079,7 +4079,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -4090,15 +4090,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): SAVE4x1 @@ -4121,13 +4121,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4164,7 +4164,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -4174,7 +4174,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_SUB KERNEL2x1_SUB @@ -4186,7 +4186,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_36 + je L(1_36) KERNEL2x1_SUB KERNEL2x1_SUB @@ -4198,12 +4198,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -4211,7 +4211,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -4222,15 +4222,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): SAVE2x1 @@ -4252,13 +4252,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4293,7 +4293,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -4302,7 +4302,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_SUB KERNEL1x1_SUB @@ -4314,7 +4314,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_46 + je L(1_46) KERNEL1x1_SUB KERNEL1x1_SUB @@ -4326,12 +4326,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -4339,7 +4339,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -4349,15 +4349,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): SAVE1x1 @@ -4379,7 +4379,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp @@ -4471,13 +4471,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -4506,22 +4506,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Ndiv6, J cmpq $0, J - je .L2_0 + je L(2_0) ALIGN_4 /*******************************************************************************************/ -.L4_01: +L(4_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L4_01b + jz L(4_01b) ALIGN_4 -.L4_01a: +L(4_01a): prefetcht0 512(BO1) prefetchw 512(BO) @@ -4538,30 +4538,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 16*SIZE,BO1 addq $ 16*SIZE,BO decq %rax - jnz .L4_01a + jnz L(4_01a) -.L4_01b: +L(4_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L4_02d + jz L(4_02d) ALIGN_4 -.L4_02c: +L(4_02c): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $ 4*SIZE,BO1 addq $ 4*SIZE,BO decq %rax - jnz .L4_02c + jnz L(4_02c) -.L4_02d: +L(4_02d): movq BO1, B // next offset of B -.L4_10: +L(4_10): movq C, CO1 leaq (C, LDC, 2), CO2 leaq (C, LDC, 4), C // c += 4 * ldc @@ -4576,11 +4576,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L4_20 + je L(4_20) ALIGN_4 -.L4_11: +L(4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4617,7 +4617,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L4_16 + je L(4_16) movq %rax, BI // Index for BO leaq (,BI,4) , BI // BI = BI * 4 ; number of values @@ -4628,7 +4628,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_12: +L(4_12): prefetcht0 A_PR1(AO, %rax, SIZE) prefetcht0 B_PR1(BO, BI , SIZE) @@ -4650,7 +4650,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO, %rax, SIZE) KERNEL16x4_SUB - je .L4_16 + je L(4_16) prefetcht0 A_PR1(AO, %rax, SIZE) prefetcht0 B_PR1(BO, BI , SIZE) @@ -4672,12 +4672,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO, %rax, SIZE) KERNEL16x4_SUB - je .L4_16 + je L(4_16) - jmp .L4_12 + jmp L(4_12) ALIGN_4 -.L4_16: +L(4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -4685,7 +4685,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_19 + je L(4_19) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -4697,15 +4697,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_17: +L(4_17): KERNEL16x4_SUB - jl .L4_17 + jl L(4_17) ALIGN_4 -.L4_19: +L(4_19): SAVE16x4 @@ -4728,25 +4728,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 addq $16 * SIZE, CO2 # coffset += 16 decq I # i -- - jg .L4_11 + jg L(4_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L4_20: +L(4_20): // Test rest of M testq $15, M - jz .L4_60 // to next 3 lines of N + jz L(4_60) // to next 3 lines of N testq $8, M - jz .L4_21pre + jz L(4_21pre) ALIGN_4 /**************************************************************************/ -.L4_20_1: +L(4_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4784,7 +4784,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_20_6 + je L(4_20_6) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -4795,7 +4795,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_20_2: +L(4_20_2): KERNEL8x4_SUB KERNEL8x4_SUB @@ -4807,7 +4807,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_SUB KERNEL8x4_SUB - je .L4_20_6 + je L(4_20_6) KERNEL8x4_SUB KERNEL8x4_SUB @@ -4819,12 +4819,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_SUB KERNEL8x4_SUB - je .L4_20_6 + je L(4_20_6) - jmp .L4_20_2 + jmp L(4_20_2) ALIGN_4 -.L4_20_6: +L(4_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -4832,7 +4832,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_20_9 + je L(4_20_9) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -4844,15 +4844,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_20_7: +L(4_20_7): KERNEL8x4_SUB - jl .L4_20_7 + jl L(4_20_7) ALIGN_4 -.L4_20_9: +L(4_20_9): SAVE8x4 @@ -4880,13 +4880,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L4_21pre: +L(4_21pre): testq $4, M - jz .L4_30 + jz L(4_30) ALIGN_4 -.L4_21: +L(4_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -4924,7 +4924,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_26 + je L(4_26) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -4935,7 +4935,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_22: +L(4_22): KERNEL4x4_SUB KERNEL4x4_SUB @@ -4947,7 +4947,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB KERNEL4x4_SUB - je .L4_26 + je L(4_26) KERNEL4x4_SUB KERNEL4x4_SUB @@ -4959,12 +4959,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB KERNEL4x4_SUB - je .L4_26 + je L(4_26) - jmp .L4_22 + jmp L(4_22) ALIGN_4 -.L4_26: +L(4_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -4972,7 +4972,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_29 + je L(4_29) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -4984,15 +4984,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_27: +L(4_27): KERNEL4x4_SUB - jl .L4_27 + jl L(4_27) ALIGN_4 -.L4_29: +L(4_29): SAVE4x4 @@ -5017,13 +5017,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L4_30: +L(4_30): testq $2, M - jz .L4_40 + jz L(4_40) ALIGN_4 -.L4_31: +L(4_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5061,7 +5061,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L4_36 + je L(4_36) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -5072,7 +5072,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_32: +L(4_32): KERNEL2x4_SUB KERNEL2x4_SUB @@ -5084,7 +5084,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB KERNEL2x4_SUB - je .L4_36 + je L(4_36) KERNEL2x4_SUB KERNEL2x4_SUB @@ -5096,12 +5096,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB KERNEL2x4_SUB - je .L4_36 + je L(4_36) - jmp .L4_32 + jmp L(4_32) ALIGN_4 -.L4_36: +L(4_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -5109,7 +5109,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_39 + je L(4_39) movq %rax, BI // Index for BO leaq (,BI, 4), BI // BI = BI * 4 ; number of values @@ -5121,15 +5121,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_37: +L(4_37): KERNEL2x4_SUB - jl .L4_37 + jl L(4_37) ALIGN_4 -.L4_39: +L(4_39): SAVE2x4 @@ -5153,13 +5153,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO2 # coffset += 2 ALIGN_4 -.L4_40: +L(4_40): testq $1, M - jz .L4_60 // to next 4 lines of N + jz L(4_60) // to next 4 lines of N ALIGN_4 -.L4_41: +L(4_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5195,7 +5195,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L4_46 + je L(4_46) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -5205,7 +5205,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_42: +L(4_42): KERNEL1x4_SUB KERNEL1x4_SUB @@ -5217,7 +5217,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB KERNEL1x4_SUB - je .L4_46 + je L(4_46) KERNEL1x4_SUB KERNEL1x4_SUB @@ -5229,12 +5229,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB KERNEL1x4_SUB - je .L4_46 + je L(4_46) - jmp .L4_42 + jmp L(4_42) ALIGN_4 -.L4_46: +L(4_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -5242,7 +5242,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L4_49 + je L(4_49) movq %rax, BI // Index for BO leaq (,BI,4), BI // BI = BI * 4 ; number of values @@ -5253,15 +5253,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L4_47: +L(4_47): KERNEL1x4_SUB - jl .L4_47 + jl L(4_47) ALIGN_4 -.L4_49: +L(4_49): SAVE1x4 @@ -5288,38 +5288,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L4_60: +L(4_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif decq J // j -- - jg .L4_01 // next 4 lines of N + jg L(4_01) // next 4 lines of N /*******************************************************************************************/ -.L2_0: +L(2_0): movq Nmod6, J andq $3, J // j % 4 - je .L999 + je L(999) movq Nmod6, J andq $2, J // j % 4 - je .L1_0 + je L(1_0) -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax sarq $2, %rax // K / 4 - jz .L2_01b + jz L(2_01b) ALIGN_4 -.L2_01a: +L(2_01a): vmovsd (BO1), %xmm0 vmovsd 2*SIZE(BO1), %xmm1 @@ -5334,30 +5334,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $8*SIZE,BO1 addq $8*SIZE,BO decq %rax - jnz .L2_01a + jnz L(2_01a) -.L2_01b: +L(2_01b): movq K, %rax andq $3, %rax // K % 4 - jz .L2_02d + jz L(2_02d) ALIGN_4 -.L2_02c: +L(2_02c): vmovsd (BO1), %xmm0 vmovsd %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L2_02c + jnz L(2_02c) -.L2_02d: +L(2_02d): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -5371,11 +5371,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L2_20 + je L(2_20) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5412,7 +5412,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5423,7 +5423,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_12: +L(2_12): KERNEL16x2_SUB KERNEL16x2_SUB @@ -5435,7 +5435,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB KERNEL16x2_SUB - je .L2_16 + je L(2_16) KERNEL16x2_SUB KERNEL16x2_SUB @@ -5447,12 +5447,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB KERNEL16x2_SUB - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -5460,7 +5460,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5472,15 +5472,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL16x2_SUB - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): SAVE16x2 @@ -5502,25 +5502,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_20: +L(2_20): // Test rest of M testq $15, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N testq $8, M - jz .L2_21pre + jz L(2_21pre) ALIGN_4 /**************************************************************************/ -.L2_20_1: +L(2_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5558,7 +5558,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_20_6 + je L(2_20_6) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5569,7 +5569,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_2: +L(2_20_2): KERNEL8x2_SUB @@ -5582,7 +5582,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB KERNEL8x2_SUB - je .L2_20_6 + je L(2_20_6) KERNEL8x2_SUB KERNEL8x2_SUB @@ -5594,12 +5594,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB KERNEL8x2_SUB - je .L2_20_6 + je L(2_20_6) - jmp .L2_20_2 + jmp L(2_20_2) ALIGN_4 -.L2_20_6: +L(2_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -5607,7 +5607,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_20_9 + je L(2_20_9) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5619,15 +5619,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_20_7: +L(2_20_7): KERNEL8x2_SUB - jl .L2_20_7 + jl L(2_20_7) ALIGN_4 -.L2_20_9: +L(2_20_9): SAVE8x2 @@ -5654,13 +5654,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L2_21pre: +L(2_21pre): testq $4, M - jz .L2_30 + jz L(2_30) ALIGN_4 -.L2_21: +L(2_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5698,7 +5698,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_26 + je L(2_26) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 1 ; number of values @@ -5709,7 +5709,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_22: +L(2_22): KERNEL4x2_SUB @@ -5722,7 +5722,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_26 + je L(2_26) KERNEL4x2_SUB KERNEL4x2_SUB @@ -5734,12 +5734,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB KERNEL4x2_SUB - je .L2_26 + je L(2_26) - jmp .L2_22 + jmp L(2_22) ALIGN_4 -.L2_26: +L(2_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -5747,7 +5747,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_29 + je L(2_29) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5759,15 +5759,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_27: +L(2_27): KERNEL4x2_SUB - jl .L2_27 + jl L(2_27) ALIGN_4 -.L2_29: +L(2_29): SAVE4x2 @@ -5791,13 +5791,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L2_30: +L(2_30): testq $2, M - jz .L2_40 + jz L(2_40) ALIGN_4 -.L2_31: +L(2_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5835,7 +5835,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L2_36 + je L(2_36) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5846,7 +5846,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_32: +L(2_32): KERNEL2x2_SUB KERNEL2x2_SUB @@ -5858,7 +5858,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_36 + je L(2_36) KERNEL2x2_SUB KERNEL2x2_SUB @@ -5870,12 +5870,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_36 + je L(2_36) - jmp .L2_32 + jmp L(2_32) ALIGN_4 -.L2_36: +L(2_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -5883,7 +5883,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_39 + je L(2_39) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5895,15 +5895,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_37: +L(2_37): KERNEL2x2_SUB - jl .L2_37 + jl L(2_37) ALIGN_4 -.L2_39: +L(2_39): SAVE2x2 @@ -5926,13 +5926,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -5968,7 +5968,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -5978,7 +5978,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_42: +L(2_42): KERNEL1x2_SUB KERNEL1x2_SUB @@ -5990,7 +5990,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_46 + je L(2_46) KERNEL1x2_SUB KERNEL1x2_SUB @@ -6002,12 +6002,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -6015,7 +6015,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq (BI,BI,1), BI // BI = BI * 2 ; number of values @@ -6026,15 +6026,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): SAVE1x2 @@ -6060,7 +6060,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -6068,7 +6068,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -6076,30 +6076,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovss (BO1), %xmm0 vmovss %xmm0, (BO) addq $1*SIZE,BO1 addq $1*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -6113,11 +6113,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $4, I // i = (m >> 4) - je .L1_20 + je L(1_20) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -6153,7 +6153,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO salq $4, %rax // rax = rax * 16 ; number of values @@ -6163,7 +6163,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_12: +L(1_12): KERNEL16x1_SUB KERNEL16x1_SUB @@ -6175,7 +6175,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB KERNEL16x1_SUB - je .L1_16 + je L(1_16) KERNEL16x1_SUB KERNEL16x1_SUB @@ -6187,12 +6187,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB KERNEL16x1_SUB - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -6200,7 +6200,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO @@ -6211,15 +6211,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL16x1_SUB - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): SAVE16x1 @@ -6240,25 +6240,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $16 * SIZE, CO1 # coffset += 16 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_20: +L(1_20): // Test rest of M testq $15, M - jz .L999 + jz L(999) testq $8, M - jz .L1_21pre + jz L(1_21pre) ALIGN_4 /**************************************************************************/ -.L1_20_1: +L(1_20_1): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -6295,7 +6295,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_20_6 + je L(1_20_6) movq %rax, BI // Index for BO salq $3, %rax // rax = rax * 8 ; number of values @@ -6305,7 +6305,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_2: +L(1_20_2): KERNEL8x1_SUB KERNEL8x1_SUB @@ -6317,7 +6317,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB KERNEL8x1_SUB - je .L1_20_6 + je L(1_20_6) KERNEL8x1_SUB KERNEL8x1_SUB @@ -6329,12 +6329,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB KERNEL8x1_SUB - je .L1_20_6 + je L(1_20_6) - jmp .L1_20_2 + jmp L(1_20_2) ALIGN_4 -.L1_20_6: +L(1_20_6): #ifndef TRMMKERNEL movq K, %rax #else @@ -6342,7 +6342,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_20_9 + je L(1_20_9) movq %rax, BI // Index for BO @@ -6353,15 +6353,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_20_7: +L(1_20_7): KERNEL8x1_SUB - jl .L1_20_7 + jl L(1_20_7) ALIGN_4 -.L1_20_9: +L(1_20_9): SAVE8x1 @@ -6387,13 +6387,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /**************************************************************************/ -.L1_21pre: +L(1_21pre): testq $4, M - jz .L1_30 + jz L(1_30) ALIGN_4 -.L1_21: +L(1_21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -6430,7 +6430,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_26 + je L(1_26) movq %rax, BI // Index for BO salq $2, %rax // rax = rax * 4 ; number of values @@ -6440,7 +6440,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_22: +L(1_22): KERNEL4x1_SUB KERNEL4x1_SUB @@ -6452,7 +6452,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_26 + je L(1_26) KERNEL4x1_SUB KERNEL4x1_SUB @@ -6464,12 +6464,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_26 + je L(1_26) - jmp .L1_22 + jmp L(1_22) ALIGN_4 -.L1_26: +L(1_26): #ifndef TRMMKERNEL movq K, %rax #else @@ -6477,7 +6477,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_29 + je L(1_29) movq %rax, BI // Index for BO @@ -6488,15 +6488,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_27: +L(1_27): KERNEL4x1_SUB - jl .L1_27 + jl L(1_27) ALIGN_4 -.L1_29: +L(1_29): SAVE4x1 @@ -6519,13 +6519,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L1_30: +L(1_30): testq $2, M - jz .L1_40 + jz L(1_40) ALIGN_4 -.L1_31: +L(1_31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -6562,7 +6562,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax - je .L1_36 + je L(1_36) movq %rax, BI // Index for BO salq $1, %rax // rax = rax *2 ; number of values @@ -6572,7 +6572,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_32: +L(1_32): KERNEL2x1_SUB KERNEL2x1_SUB @@ -6584,7 +6584,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_36 + je L(1_36) KERNEL2x1_SUB KERNEL2x1_SUB @@ -6596,12 +6596,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_36 + je L(1_36) - jmp .L1_32 + jmp L(1_32) ALIGN_4 -.L1_36: +L(1_36): #ifndef TRMMKERNEL movq K, %rax #else @@ -6609,7 +6609,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_39 + je L(1_39) movq %rax, BI // Index for BO @@ -6620,15 +6620,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_37: +L(1_37): KERNEL2x1_SUB - jl .L1_37 + jl L(1_37) ALIGN_4 -.L1_39: +L(1_39): SAVE2x1 @@ -6650,13 +6650,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $2 * SIZE, CO1 # coffset += 2 ALIGN_4 -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -6691,7 +6691,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $-8, %rax - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq (AO, %rax, SIZE), AO @@ -6700,7 +6700,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_42: +L(1_42): KERNEL1x1_SUB KERNEL1x1_SUB @@ -6712,7 +6712,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_46 + je L(1_46) KERNEL1x1_SUB KERNEL1x1_SUB @@ -6724,12 +6724,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -6737,7 +6737,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO @@ -6747,15 +6747,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): SAVE1x1 @@ -6777,7 +6777,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ALIGN_4 -.L999: +L(999): movq SP, %rsp movq (%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/sgemm_kernel_8x4_bulldozer.S b/kernel/x86_64/sgemm_kernel_8x4_bulldozer.S index 2194fd6c1b..2b3128562c 100644 --- a/kernel/x86_64/sgemm_kernel_8x4_bulldozer.S +++ b/kernel/x86_64/sgemm_kernel_8x4_bulldozer.S @@ -404,9 +404,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L50 + jle L(50) -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -417,10 +417,10 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): prefetcht0 192(B) prefetcht0 256(B) @@ -478,17 +478,17 @@ addq $64 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movaps 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -504,10 +504,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): movq C, CO1 leaq (C, LDC, 1), CO2 movq A, AO @@ -515,10 +515,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -575,10 +575,10 @@ leaq (BO, %rax, 8), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_3 -.L12: +L(12): KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -588,7 +588,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -598,7 +598,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -608,7 +608,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -618,7 +618,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -628,7 +628,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -638,7 +638,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -648,7 +648,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -658,10 +658,10 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) BRANCH - jl .L12 + jl L(12) ALIGN_4 -.L15: +L(15): movaps ALPHA, %xmm7 @@ -671,7 +671,7 @@ movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_3 @@ -684,14 +684,14 @@ addq $64 * SIZE, BO ALIGN_3 -.L16: +L(16): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L18 + je L(18) leaq (, %rax, 8), %rax leaq (AO, %rax, 4), AO @@ -699,7 +699,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulps %xmm1, %xmm0 mulps -28 * SIZE(AO, %rax, 4), %xmm1 addps %xmm0, %xmm8 @@ -726,10 +726,10 @@ movaps %xmm0, %xmm2 addq $SIZE * 2, %rax - jl .L17 + jl L(17) ALIGN_4 -.L18: +L(18): #ifndef TRMMKERNEL vfmaddps 0 * SIZE(CO1),%xmm7, %xmm8, %xmm8 @@ -785,12 +785,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -834,10 +834,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -945,10 +945,10 @@ addq $ 32 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -957,10 +957,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -977,10 +977,10 @@ addq $ 4 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): mulps %xmm15, %xmm0 mulps %xmm15, %xmm1 mulps %xmm15, %xmm2 @@ -1026,9 +1026,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $2, M - je .L40 + je L(40) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1072,10 +1072,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movsd 4 * SIZE(BO), %xmm9 @@ -1191,10 +1191,10 @@ addq $ 16 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1203,10 +1203,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movsd 4 * SIZE(BO), %xmm9 @@ -1224,10 +1224,10 @@ addq $ 2 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): mulps %xmm15, %xmm0 mulps %xmm15, %xmm1 mulps %xmm15, %xmm2 @@ -1267,9 +1267,9 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L40: +L(40): testq $1, M - je .L49 + je L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1314,10 +1314,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): mulss %xmm8, %xmm9 addss %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -1433,10 +1433,10 @@ addq $ 8 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1445,10 +1445,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -1466,10 +1466,10 @@ addq $ 1 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): mulss %xmm15, %xmm0 mulss %xmm15, %xmm1 mulss %xmm15, %xmm2 @@ -1507,19 +1507,19 @@ #endif ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C # c += 4 * ldc decq J # j -- - jg .L01 + jg L(01) -.L50: +L(50): testq $2, N - je .L100 + je L(100) -.L51: +L(51): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1530,10 +1530,10 @@ movq K, %rax sarq $2, %rax - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 @@ -1563,17 +1563,17 @@ addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): movq K, %rax andq $3, %rax BRANCH - jle .L60 + jle L(60) ALIGN_4 -.L54: +L(54): movsd 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -1590,20 +1590,20 @@ addq $ 2 * SIZE, B addq $ 8 * SIZE, BO decq %rax - jne .L54 + jne L(54) ALIGN_4 -.L60: +L(60): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L70 + jle L(70) ALIGN_4 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1651,10 +1651,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -1762,10 +1762,10 @@ addq $64 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1774,10 +1774,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -1794,10 +1794,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -1844,12 +1844,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L61 + jg L(61) ALIGN_4 -.L70: +L(70): testq $4, M - je .L80 + je L(80) #if !defined(TRMMKERNEL) || \ @@ -1894,10 +1894,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 @@ -1958,10 +1958,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -1970,10 +1970,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -1984,10 +1984,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2026,9 +2026,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L80: +L(80): testq $2, M - je .L90 + je L(90) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2072,10 +2072,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L85 + je L(85) ALIGN_4 -.L82: +L(82): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movsd 4 * SIZE(BO), %xmm9 @@ -2143,10 +2143,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L85: +L(85): #ifndef TRMMKERNEL movq K, %rax #else @@ -2155,10 +2155,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movsd 4 * SIZE(BO), %xmm9 @@ -2170,10 +2170,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movsd 0 * SIZE(CO2), %xmm10 @@ -2210,9 +2210,9 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L90: +L(90): testq $1, M - je .L99 + je L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2256,10 +2256,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -2327,10 +2327,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2339,10 +2339,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -2354,10 +2354,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): #ifndef TRMMKERNEL movss 0 * SIZE(CO1), %xmm8 movss 0 * SIZE(CO2), %xmm10 @@ -2390,7 +2390,7 @@ #endif ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -2398,11 +2398,11 @@ ALIGN_4 -.L100: +L(100): testq $1, N - je .L999 + je L(999) -.L101: +L(101): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -2413,11 +2413,11 @@ movq K, %rax sarq $3, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movups 0 * SIZE(B), %xmm3 movups 4 * SIZE(B), %xmm7 @@ -2447,17 +2447,17 @@ addq $32 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): movq K, %rax andq $7, %rax BRANCH - jle .L110 + jle L(110) ALIGN_4 -.L104: +L(104): movss 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -2467,19 +2467,19 @@ addq $ 1 * SIZE, B addq $ 4 * SIZE, BO decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L110: +L(110): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L120 + jle L(120) ALIGN_4 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2526,10 +2526,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulps %xmm9, %xmm8 mulps -28 * SIZE(AO), %xmm9 @@ -2590,10 +2590,10 @@ addq $64 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2602,10 +2602,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulps %xmm9, %xmm8 mulps -28 * SIZE(AO), %xmm9 addps %xmm8, %xmm0 @@ -2616,10 +2616,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2652,12 +2652,12 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L111 + jg L(111) ALIGN_4 -.L120: +L(120): testq $4, M - je .L130 + je L(130) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2699,10 +2699,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L125 + je L(125) ALIGN_4 -.L122: +L(122): mulps %xmm8, %xmm9 movaps -28 * SIZE(AO), %xmm8 mulps 4 * SIZE(BO), %xmm8 @@ -2734,10 +2734,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L122 + jne L(122) ALIGN_4 -.L125: +L(125): #ifndef TRMMKERNEL movq K, %rax #else @@ -2746,10 +2746,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): mulps %xmm8, %xmm9 movaps -28 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -2758,10 +2758,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -2794,9 +2794,9 @@ addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L130: +L(130): testq $2, M - je .L140 + je L(140) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2838,10 +2838,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L135 + je L(135) ALIGN_4 -.L132: +L(132): mulps %xmm8, %xmm9 movsd -30 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -2884,10 +2884,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L132 + jne L(132) ALIGN_4 -.L135: +L(135): #ifndef TRMMKERNEL movq K, %rax #else @@ -2896,10 +2896,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): mulps %xmm8, %xmm9 movsd -30 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -2908,10 +2908,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): addps %xmm1, %xmm0 mulps %xmm15, %xmm0 @@ -2938,9 +2938,9 @@ addq $2 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L140: +L(140): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2982,10 +2982,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L145 + je L(145) ALIGN_4 -.L142: +L(142): mulss %xmm8, %xmm9 movss -31 * SIZE(AO), %xmm8 mulss 4 * SIZE(BO), %xmm8 @@ -3016,10 +3016,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L142 + jne L(142) ALIGN_4 -.L145: +L(145): #ifndef TRMMKERNEL movq K, %rax #else @@ -3028,10 +3028,10 @@ movss ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): mulss %xmm8, %xmm9 movss -31 * SIZE(AO), %xmm8 addss %xmm9, %xmm0 @@ -3040,10 +3040,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): addss %xmm1, %xmm0 addss %xmm3, %xmm2 addss %xmm2, %xmm0 @@ -3057,7 +3057,7 @@ movss %xmm0, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/sgemm_kernel_8x8_sandy.S b/kernel/x86_64/sgemm_kernel_8x8_sandy.S index 2843efbaeb..2fc6a724ec 100644 --- a/kernel/x86_64/sgemm_kernel_8x8_sandy.S +++ b/kernel/x86_64/sgemm_kernel_8x8_sandy.S @@ -255,9 +255,9 @@ movq %r11, kk #endif MOVQ bn,j; SARQ $3,j; -JLE .L0_loopE; +JLE L(0_loopE); ALIGN_4; -.L0_bodyB:; +L(0_bodyB):; #if defined(TRMMKERNEL) && defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk; @@ -271,9 +271,9 @@ LEAQ (bb, k, 1), prebb; MOVQ ba,ptrba; MOVQ bm,i; SARQ $3,i; -JLE .L1_loopE; +JLE L(1_loopE); ALIGN_4; -.L1_bodyB:; +L(1_bodyB):; #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -324,9 +324,9 @@ ADDQ $8, %rax; MOVQ %rax, kkk; #endif SARQ $2,k; -JLE .L2_loopE; +JLE L(2_loopE); ALIGN_4; -.L2_bodyB:; +L(2_bodyB):; # Computing kernel #### Unroll times 1 #### @@ -452,19 +452,19 @@ MUL_SY yvec1, yvec4, yvec6; MUL_SY yvec1, yvec5, yvec7; ADD_SY yvec10, yvec6, yvec10; ADD_SY yvec8, yvec7, yvec8; -.L2_bodyE:; +L(2_bodyE):; DECQ k; -JG .L2_bodyB; +JG L(2_bodyB); ALIGN_4 -.L2_loopE:; +L(2_loopE):; #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L3_loopE; +JLE L(3_loopE); ALIGN_4 -.L3_loobB: +L(3_loobB): #### Unroll times 1 #### MUL_SY yvec0, yvec2, yvec6; SHUF_SY $0x03, yvec2, yvec2, yvec4; @@ -526,15 +526,15 @@ MUL_SY yvec1, yvec4, yvec6; MUL_SY yvec1, yvec5, yvec7; ADD_SY yvec10, yvec6, yvec10; ADD_SY yvec8, yvec7, yvec8; -.L3_loopE: +L(3_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L4_loopE; +JLE L(4_loopE); ALIGN_4 -.L4_loopB:; +L(4_loopB):; #### Unroll times 1 #### MUL_SY yvec0, yvec2, yvec6; SHUF_SY $0x03, yvec2, yvec2, yvec4; @@ -564,7 +564,7 @@ MUL_SY yvec0, yvec5, yvec7; ADD_SY yvec10, yvec6, yvec10; ADD_SY yvec8, yvec7, yvec8; -.L4_loopE:; +L(4_loopE):; #### Load Alpha #### BROAD_SY MEMALPHA,yvec7; MUL_SY yvec7,yvec15,yvec15; @@ -591,7 +591,7 @@ REVS_SY $0xe4,yvec7,yvec8,yvec8; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L4_loopEx; +JNE L(4_loopEx); ALIGN_4 LEAQ (ldc,ldc,2),%rax; EXTRA_SY $1,yvec15,xvec7; @@ -648,12 +648,12 @@ ADDQ $8, kk #endif ADDQ $8*SIZE,C0; ADDQ $8*SIZE,C1; -.L1_bodyE:; +L(1_bodyE):; DECQ i; -JG .L1_bodyB; -JMP .L1_loopE; +JG L(1_bodyB); +JMP L(1_loopE); ALIGN_4; -.L4_loopEx: +L(4_loopEx): LEAQ (ldc,ldc,2),%rax; EXTRA_SY $1, yvec15, xvec7; #ifndef TRMMKERNEL @@ -795,13 +795,13 @@ ADDQ $8, kk ADDQ $8*SIZE, C0; ADDQ $8*SIZE, C1; DECQ i; -JG .L1_bodyB; +JG L(1_bodyB); ALIGN_4 -.L1_loopE:; +L(1_loopE):; TEST $4, bm; -JLE .L5_loopE; +JLE L(5_loopE); ALIGN_4 -.L5_bodyB: +L(5_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -839,9 +839,9 @@ ADDQ $8, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L8_loopE; +JLE L(8_loopE); ALIGN_4 -.L8_bodyB: +L(8_bodyB): #### Unroll time 1 #### SHUF_SX $0x4e, xvec2, xvec4; @@ -965,17 +965,17 @@ ODUP_SX 0*SIZE(ptrbb), xvec3; MUL_SX xvec1, xvec5, xvec5; ADD_SX xvec5, xvec8, xvec8; DECQ k; -JG .L8_bodyB; +JG L(8_bodyB); ALIGN_4 -.L8_loopE: +L(8_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L9_loopE; +JLE L(9_loopE); ALIGN_4 -.L9_bodyB: +L(9_bodyB): #### Unroll time 1 #### SHUF_SX $0x4e, xvec2, xvec4; MUL_SX xvec0, xvec2, xvec2; @@ -1038,15 +1038,15 @@ ODUP_SX 0*SIZE(ptrbb), xvec3; MUL_SX xvec1, xvec5, xvec5; ADD_SX xvec5, xvec8, xvec8; -.L9_loopE: +L(9_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L10_loopE; +JLE L(10_loopE); ALIGN_4 -.L10_bodyB: +L(10_bodyB): #### Unroll time 1 #### SHUF_SX $0x4e, xvec2, xvec4; MUL_SX xvec0, xvec2, xvec2; @@ -1076,7 +1076,7 @@ ADD_SX xvec4, xvec9, xvec9; MUL_SX xvec0, xvec5, xvec5; ADD_SX xvec5, xvec8, xvec8; -.L10_loopE: +L(10_loopE): #### Multiply Alpha #### BROAD_SX MEMALPHA, xvec7; MUL_SX xvec7, xvec15, xvec15; @@ -1104,7 +1104,7 @@ REVS_SX $0xe4, xvec7, xvec8, xvec8; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L10_loopEx; +JNE L(10_loopEx); ALIGN_4 LEAQ (ldc,ldc,2),%rax; #ifndef TRMMKERNEL @@ -1137,9 +1137,9 @@ ADDQ $4, kk #endif ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -JMP .L5_loopE; +JMP L(5_loopE); ALIGN_4 -.L10_loopEx: +L(10_loopEx): LEAQ (ldc,ldc,2),%rax; #ifndef TRMMKERNEL LDL_SX 0*SIZE(C0), xvec7, xvec7; @@ -1195,11 +1195,11 @@ ADDQ $4, kk #endif ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -.L5_loopE: +L(5_loopE): TEST $2, bm; -JLE .L6_loopE; +JLE L(6_loopE); ALIGN_4 -.L6_bodyB: +L(6_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -1231,9 +1231,9 @@ ADDQ $8, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L11_loopE; +JLE L(11_loopE); ALIGN_4 -.L11_bodyB: +L(11_bodyB): #### Computing kernel LD_SX 0*SIZE(ptrba), xvec0; # a1, a2, a3, a4 SHUF_SX $0x44, xvec0, xvec1; # a1, a2, a1, a2 @@ -1300,17 +1300,17 @@ ADD_SX xvec5, xvec12, xvec12; ADDQ $8*SIZE, ptrba; ADDQ $32*SIZE, ptrbb; DECQ k; -JG .L11_bodyB; +JG L(11_bodyB); ALIGN_4 -.L11_loopE: +L(11_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L12_loopE; +JLE L(12_loopE); ALIGN_4 -.L12_bodyB: +L(12_bodyB): LD_SX 0*SIZE(ptrba), xvec0; # a1, a2, a3, a4 SHUF_SX $0x44, xvec0, xvec1; # a1, a2, a1, a2 EDUP_SX 0*SIZE(ptrbb), xvec2; @@ -1344,15 +1344,15 @@ ADD_SX xvec5, xvec12, xvec12; ADDQ $4*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; -.L12_loopE: +L(12_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L13_loopE; +JLE L(13_loopE); ALIGN_4 -.L13_bodyB: +L(13_bodyB): LD_SX 0*SIZE(ptrba), xvec0; # a1, a2, a3, a4 SHUF_SX $0x44, xvec0, xvec1; # a1, a2, a1, a2 EDUP_SX 0*SIZE(ptrbb), xvec2; @@ -1370,7 +1370,7 @@ MUL_SX xvec1, xvec5, xvec5; ADD_SX xvec5, xvec12, xvec12; ADDQ $2*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L13_loopE: +L(13_loopE): LEAQ (ldc,ldc,2),%rax; #### Multiply Alpha #### BROAD_SX MEMALPHA, xvec7; @@ -1413,11 +1413,11 @@ ADDQ $2, kk ADDQ $2*SIZE, C0; ADDQ $2*SIZE, C1; #### Writing Back #### -.L6_loopE: +L(6_loopE): TEST $1, bm; -JLE .L7_loopE; +JLE L(7_loopE); ALIGN_4 -.L7_bodyB: +L(7_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -1447,9 +1447,9 @@ ADDQ $8, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L14_loopE; +JLE L(14_loopE); ALIGN_4 -.L14_bodyB: +L(14_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; MUL_SX xvec0, xvec2, xvec2; @@ -1485,17 +1485,17 @@ ADD_SX xvec5, xvec14, xvec14; ADDQ $4*SIZE, ptrba; ADDQ $32*SIZE, ptrbb; DECQ k; -JG .L14_bodyB; +JG L(14_bodyB); ALIGN_4 -.L14_loopE: +L(14_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L15_loopE; +JLE L(15_loopE); ALIGN_4 -.L15_bodyB: +L(15_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; MUL_SX xvec0, xvec2, xvec2; @@ -1514,15 +1514,15 @@ ADD_SX xvec5, xvec14, xvec14; ADDQ $2*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; -.L15_loopE: +L(15_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L16_loopE; +JLE L(16_loopE); ALIGN_4 -.L16_bodyB: +L(16_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; MUL_SX xvec0, xvec2, xvec2; @@ -1533,7 +1533,7 @@ ADD_SX xvec3, xvec14, xvec14; ADDQ $1, ptrba; ADDQ $4, ptrbb; -.L16_loopE: +L(16_loopE): BROAD_SX MEMALPHA, xvec7; MUL_SX xvec7, xvec15, xvec15; MUL_SX xvec7, xvec14, xvec14; @@ -1583,7 +1583,7 @@ ADDQ $1, kk ADDQ $1*SIZE, C0; ADDQ $1*SIZE, C1; #### Writing Back #### -.L7_loopE: +L(7_loopE): #if defined(TRMMKERNEL)&&!defined(LEFT) ADDQ $8, kk #endif @@ -1591,15 +1591,15 @@ MOVQ bk,k; SALQ $5,k; ADDQ k,bb; LEAQ (C,ldc,8),C; -.L0_bodyE:; +L(0_bodyE):; DECQ j; -JG .L0_bodyB; +JG L(0_bodyB); ALIGN_4; -.L0_loopE:; +L(0_loopE):; TEST $4, bn; # Rn = 4 -JLE .L20_loopE; +JLE L(20_loopE); ALIGN_4; -.L20_bodyB: +L(20_bodyB): #if defined(TRMMKERNEL) && defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk; @@ -1610,9 +1610,9 @@ LEAQ (C, ldc, 2), C1; MOVQ ba, ptrba; MOVQ bm, i; SARQ $3, i; -JLE .L21_loopE; +JLE L(21_loopE); ALIGN_4 -.L21_bodyB: +L(21_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -1650,9 +1650,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2,k; -JLE .L211_loopE; +JLE L(211_loopE); ALIGN_4 -.L211_bodyB: +L(211_bodyB): #### Unroll time 1 #### ODUP_SX 0*SIZE(ptrbb), xvec3; SHUF_SX $0x4e, xvec2, xvec4; @@ -1782,17 +1782,17 @@ MUL_SX xvec1, xvec7, xvec7; ADD_SX xvec7, xvec8, xvec8; LD_SX 4*SIZE(ptrba), xvec1; DECQ k; -JG .L211_bodyB; +JG L(211_bodyB); ALIGN_4 -.L211_loopE: +L(211_loopE): #ifndef TRMMKERNEL TEST $2, bk #else TEST $2, kkk; #endif -JLE .L212_loopE; +JLE L(212_loopE); ALIGN_4 -.L212_bodyB: +L(212_bodyB): #### Unroll time 1 #### ODUP_SX 0*SIZE(ptrbb), xvec3; SHUF_SX $0x4e, xvec2, xvec4; @@ -1858,15 +1858,15 @@ MUL_SX xvec1, xvec7, xvec7; ADD_SX xvec7, xvec8, xvec8; LD_SX 4*SIZE(ptrba), xvec1; -.L212_loopE: +L(212_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L213_loopE; +JLE L(213_loopE); ALIGN_4 -.L213_bodyB: +L(213_bodyB): ODUP_SX 0*SIZE(ptrbb), xvec3; SHUF_SX $0x4e, xvec2, xvec4; MOV_SX xvec2, xvec6; @@ -1897,7 +1897,7 @@ ADD_SX xvec5, xvec9, xvec9; MUL_SX xvec1, xvec7, xvec7; ADD_SX xvec7, xvec8, xvec8; -.L213_loopE: +L(213_loopE): #### Multiply Alpha #### BROAD_SX MEMALPHA, xvec7; MUL_SX xvec7, xvec15, xvec15; @@ -1964,13 +1964,13 @@ ADDQ $8, kk ADDQ $8*SIZE, C0; ADDQ $8*SIZE, C1; DECQ i; -JG .L21_bodyB; +JG L(21_bodyB); ALIGN_4 -.L21_loopE: +L(21_loopE): TEST $4, bm; -JLE .L22_loopE; +JLE L(22_loopE); ALIGN_4 -.L22_bodyB: +L(22_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -2001,9 +2001,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L221_loopE; +JLE L(221_loopE); ALIGN_4 -.L221_bodyB: +L(221_bodyB): LD_SX 0*SIZE(ptrba), xvec0; EDUP_SX 0*SIZE(ptrbb), xvec2; ODUP_SX 0*SIZE(ptrbb), xvec3; @@ -2071,17 +2071,17 @@ ADDQ $16*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; DECQ k; -JG .L221_bodyB; +JG L(221_bodyB); ALIGN_4 -.L221_loopE: +L(221_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L222_loopE; +JLE L(222_loopE); ALIGN_4 -.L222_bodyB: +L(222_bodyB): LD_SX 0*SIZE(ptrba), xvec0; EDUP_SX 0*SIZE(ptrbb), xvec2; ODUP_SX 0*SIZE(ptrbb), xvec3; @@ -2115,15 +2115,15 @@ MUL_SX xvec1, xvec5, xvec5; ADD_SX xvec5, xvec12, xvec12; ADDQ $8*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L222_loopE: +L(222_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L223_loopE; +JLE L(223_loopE); ALIGN_4 -.L223_bodyB: +L(223_bodyB): LD_SX 0*SIZE(ptrba), xvec0; EDUP_SX 0*SIZE(ptrbb), xvec2; ODUP_SX 0*SIZE(ptrbb), xvec3; @@ -2141,7 +2141,7 @@ MUL_SX xvec0, xvec5, xvec5; ADD_SX xvec5, xvec12, xvec12; ADDQ $4*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L223_loopE: +L(223_loopE): #### Multiply Alpha #### BROAD_SX MEMALPHA, xvec7; MUL_SX xvec7, xvec15, xvec15; @@ -2183,11 +2183,11 @@ ADDQ $4, kk #endif ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -.L22_loopE: +L(22_loopE): TEST $2, bm; -JLE .L23_loopE; +JLE L(23_loopE); ALIGN_4 -.L23_bodyB: +L(23_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -2216,9 +2216,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L231_loopE; +JLE L(231_loopE); ALIGN_4 -.L231_bodyB: +L(231_bodyB): LD_SX 0*SIZE(ptrba), xvec0; EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; @@ -2256,17 +2256,17 @@ ADD_SX xvec7, xvec14, xvec14; ADDQ $8*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; DECQ k; -JG .L231_bodyB; +JG L(231_bodyB); ALIGN_4 -.L231_loopE: +L(231_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L232_loopE; +JLE L(232_loopE); ALIGN_4 -.L232_bodyB: +L(232_bodyB): LD_SX 0*SIZE(ptrba), xvec0; EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; @@ -2286,15 +2286,15 @@ ADD_SX xvec7, xvec14, xvec14; ADDQ $4*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L232_loopE: +L(232_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L233_loopE; +JLE L(233_loopE); ALIGN_4 -.L233_bodyB: +L(233_bodyB): LD_SX 0*SIZE(ptrba), xvec0; EDUP_SX 0*SIZE(ptrbb), xvec4; ODUP_SX 0*SIZE(ptrbb), xvec5; @@ -2306,7 +2306,7 @@ ADD_SX xvec5, xvec14, xvec14; ADDQ $2*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L233_loopE: +L(233_loopE): #### Multiply Alpha #### BROAD_SY MEMALPHA, yvec7; MUL_SY xvec7, xvec15, xvec15; @@ -2336,11 +2336,11 @@ ADDQ $2, kk #endif ADDQ $2*SIZE, C0; ADDQ $2*SIZE, C1; -.L23_loopE: +L(23_loopE): TEST $1, bm; -JLE .L24_loopE; +JLE L(24_loopE); ALIGN_4 -.L24_bodyB: +L(24_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -2368,9 +2368,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L241_loopE; +JLE L(241_loopE); ALIGN_4 -.L241_bodyB: +L(241_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec1; MUL_SX xvec0, xvec1, xvec1; @@ -2393,17 +2393,17 @@ ADD_SX xvec7, xvec15, xvec15; ADDQ $4*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; DECQ k; -JG .L241_bodyB; +JG L(241_bodyB); ALIGN_4 -.L241_loopE: +L(241_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L242_loopE; +JLE L(242_loopE); ALIGN_4 -.L242_bodyB: +L(242_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec1; MUL_SX xvec0, xvec1, xvec1; @@ -2416,22 +2416,22 @@ ADD_SX xvec3, xvec15, xvec15; ADDQ $2*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L242_loopE: +L(242_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L243_loopE; +JLE L(243_loopE); ALIGN_4; -.L243_bodyB: +L(243_bodyB): BROAD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec1; MUL_SX xvec0, xvec1, xvec1; ADD_SX xvec1, xvec15, xvec15; ADDQ $1*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L243_loopE: +L(243_loopE): #### Multiply Alpha #### BROAD_SX MEMALPHA, xvec7; MUL_SX xvec7, xvec15, xvec15; @@ -2463,7 +2463,7 @@ ADDQ $1, kk #endif ADDQ $1*SIZE, C0; ADDQ $1*SIZE, C1; -.L24_loopE: +L(24_loopE): #if defined(TRMMKERNEL)&&!defined(LEFT) ADDQ $4, kk #endif @@ -2471,11 +2471,11 @@ MOVQ bk, k; SALQ $4, k; ADDQ k, bb; LEAQ (C, ldc, 4), C; -.L20_loopE: +L(20_loopE): TEST $2, bn; -JLE .L30_loopE; +JLE L(30_loopE); ALIGN_4 -.L30_bodyB: +L(30_bodyB): #if defined(TRMMKERNEL) && defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk @@ -2485,9 +2485,9 @@ LEAQ (C, ldc, 1), C1; MOVQ ba, ptrba; MOVQ bm, i; SARQ $3, i; -JLE .L31_loopE; +JLE L(31_loopE); ALIGN_4 -.L31_bodyB: +L(31_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -2518,9 +2518,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L311_loopE; +JLE L(311_loopE); ALIGN_4 -.L311_bodyB: +L(311_bodyB): LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0x50, xvec2, xvec3; LD_SX 0*SIZE(ptrba), xvec0; @@ -2594,17 +2594,17 @@ ADD_SX xvec6, xvec12, xvec12; ADDQ $32*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L311_bodyB; +JG L(311_bodyB); ALIGN_4 -.L311_loopE: +L(311_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L312_loopE; +JLE L(312_loopE); ALIGN_4 -.L312_bodyB: +L(312_bodyB): LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0x50, xvec2, xvec3; LD_SX 0*SIZE(ptrba), xvec0; @@ -2642,15 +2642,15 @@ ADD_SX xvec6, xvec12, xvec12; ADDQ $16*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L312_loopE: +L(312_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L313_loopE; +JLE L(313_loopE); ALIGN_4 -.L313_bodyB: +L(313_bodyB): LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0x50, xvec2, xvec3; LD_SX 0*SIZE(ptrba), xvec0; @@ -2671,7 +2671,7 @@ ADD_SX xvec6, xvec12, xvec12; ADDQ $8*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L313_loopE: +L(313_loopE): BROAD_SX MEMALPHA, xvec7; MUL_SX xvec7, xvec15, xvec15; MUL_SX xvec7, xvec14, xvec14; @@ -2713,13 +2713,13 @@ ADDQ $8, kk ADDQ $8*SIZE, C0; ADDQ $8*SIZE, C1; DECQ i; -JG .L31_bodyB; +JG L(31_bodyB); ALIGN_4 -.L31_loopE: +L(31_loopE): TEST $4, bm; -JLE .L32_loopE; +JLE L(32_loopE); ALIGN_4 -.L32_bodyB: +L(32_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -2748,9 +2748,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L321_loopE; +JLE L(321_loopE); ALIGN_4 -.L321_bodyB: +L(321_bodyB): LD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0x50, xvec2, xvec3; @@ -2788,17 +2788,17 @@ ADD_SX xvec6, xvec14, xvec14; ADDQ $16*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L321_bodyB; +JG L(321_bodyB); ALIGN_4 -.L321_loopE: +L(321_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L322_loopE; +JLE L(322_loopE); ALIGN_4 -.L322_bodyB: +L(322_bodyB): LD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0x50, xvec2, xvec3; @@ -2818,15 +2818,15 @@ ADD_SX xvec6, xvec14, xvec14; ADDQ $8*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L322_loopE: +L(322_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L323_loopE; +JLE L(323_loopE); ALIGN_4 -.L323_bodyB: +L(323_bodyB): LD_SX 0*SIZE(ptrba), xvec0; LD_SX 0*SIZE(ptrbb), xvec2; SHUF_SX $0x50, xvec2, xvec3; @@ -2838,7 +2838,7 @@ ADD_SX xvec4, xvec14, xvec14; ADDQ $4*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L323_loopE: +L(323_loopE): BROAD_SX MEMALPHA, xvec7; MUL_SX xvec7, xvec15, xvec15; MUL_SX xvec7, xvec14, xvec14; @@ -2867,11 +2867,11 @@ ADDQ $4, kk #endif ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -.L32_loopE: +L(32_loopE): TEST $2, bm; -JLE .L33_loopE; +JLE L(33_loopE); ALIGN_4 -.L33_bodyB: +L(33_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -2902,9 +2902,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L331_loopE; +JLE L(331_loopE); ALIGN_4 -.L331_bodyB: +L(331_bodyB): LD_SX 0*SIZE(ptrba), xvec0; # a0, a1, a2, a3 EDUP_SX 0*SIZE(ptrbb), xvec2; # b0, b0, b2, b2 ODUP_SX 0*SIZE(ptrbb), xvec3; # b1, b1, b3, b3 @@ -2925,17 +2925,17 @@ ADD_SX xvec3, xvec14, xvec14; ADDQ $8*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L331_bodyB; +JG L(331_bodyB); ALIGN_4 -.L331_loopE: +L(331_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L332_loopE; +JLE L(332_loopE); ALIGN_4 -.L332_bodyB: +L(332_bodyB): LD_SX 0*SIZE(ptrba), xvec0; # a0, a1, a2, a3 EDUP_SX 0*SIZE(ptrbb), xvec2; # b0, b0, b2, b2 ODUP_SX 0*SIZE(ptrbb), xvec3; # b1, b1, b3, b3 @@ -2948,15 +2948,15 @@ ADD_SX xvec3, xvec14, xvec14; ADDQ $4*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L332_loopE: +L(332_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L333_loopE; +JLE L(333_loopE); ALIGN_4 -.L333_bodyB: +L(333_bodyB): movss 0*SIZE(ptrba), xvec0; movss 1*SIZE(ptrba), xvec1; movss 0*SIZE(ptrbb), xvec2; @@ -2979,7 +2979,7 @@ ADD_SX xvec7, xvec14, xvec14 ADDQ $2*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L333_loopE: +L(333_loopE): BROAD_SX MEMALPHA, xvec7; MUL_SX xvec7, xvec15, xvec15; MUL_SX xvec7, xvec14, xvec14; @@ -3011,11 +3011,11 @@ ADDQ $2, kk; ADDQ $2*SIZE, C0; ADDQ $2*SIZE, C1; #### Writing Back #### -.L33_loopE: +L(33_loopE): TEST $1, bm; -JLE .L34_loopE; +JLE L(34_loopE); ALIGN_4 -.L34_bodyB: +L(34_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -3044,9 +3044,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L341_loopE; +JLE L(341_loopE); ALIGN_4 -.L341_bodyB: +L(341_bodyB): movss 0*SIZE(ptrba), xvec0; movss 0*SIZE(ptrbb), xvec1; mulss xvec0, xvec1; @@ -3086,17 +3086,17 @@ addss xvec2, xvec14; addq $4*SIZE, ptrba; addq $8*SIZE, ptrbb; decq k; -jg .L341_bodyB; +jg L(341_bodyB); ALIGN_4 -.L341_loopE: +L(341_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L342_loopE; +JLE L(342_loopE); ALIGN_4 -.L342_bodyB: +L(342_bodyB): movss 0*SIZE(ptrba), xvec0; movss 0*SIZE(ptrbb), xvec1; mulss xvec0, xvec1; @@ -3116,15 +3116,15 @@ mulss xvec0, xvec2; addss xvec2, xvec14; addq $2*SIZE, ptrba; addq $4*SIZE, ptrbb; -.L342_loopE: +L(342_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L343_loopE; +JLE L(343_loopE); ALIGN_4 -.L343_bodyB: +L(343_bodyB): movss 0*SIZE(ptrba), xvec0; movss 0*SIZE(ptrbb), xvec1; mulss xvec0, xvec1; @@ -3136,7 +3136,7 @@ addss xvec2, xvec14; addq $1*SIZE, ptrba; addq $2*SIZE, ptrbb -.L343_loopE: +L(343_loopE): #### Writing back #### movss MEMALPHA, xvec7; mulss xvec7, xvec15; @@ -3161,7 +3161,7 @@ ADDQ $1, kk; #endif addq $1*SIZE, C0; addq $1*SIZE, C1; -.L34_loopE: +L(34_loopE): #if defined(TRMMKERNEL) && !defined(LEFT) ADDQ $2, kk; #endif @@ -3169,11 +3169,11 @@ MOVQ bk, k; SALQ $3, k; ADDQ k, bb; LEAQ (C, ldc, 2), C; -.L30_loopE: +L(30_loopE): TEST $1, bn; -JLE .L40_loopE; +JLE L(40_loopE); ALIGN_4 -.L40_bodyB: +L(40_bodyB): #if defined(TRMMKERNEL)&&defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk; @@ -3182,9 +3182,9 @@ MOVQ C, C0; MOVQ ba, ptrba; MOVQ bm, i; SARQ $3, i; -JLE .L41_loopE; +JLE L(41_loopE); ALIGN_4 -.L41_bodyB: +L(41_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -3212,9 +3212,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L411_loopE; +JLE L(411_loopE); ALIGN_4 -.L411_bodyB: +L(411_bodyB): LD_SY 0*SIZE(ptrba), yvec0; BROAD_SY 0*SIZE(ptrbb), yvec1; MUL_SY yvec0, yvec1, yvec2; @@ -3238,17 +3238,17 @@ ADD_SY yvec2, yvec15, yvec15; ADDQ $32*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; DECQ k; -JG .L411_bodyB; +JG L(411_bodyB); ALIGN_4 -.L411_loopE: +L(411_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L412_loopE; +JLE L(412_loopE); ALIGN_4 -.L412_bodyB: +L(412_bodyB): LD_SY 0*SIZE(ptrba), yvec0; BROAD_SY 0*SIZE(ptrbb), yvec1; MUL_SY yvec0, yvec1, yvec2; @@ -3261,22 +3261,22 @@ ADD_SY yvec2, yvec15, yvec15; ADDQ $16*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L412_loopE: +L(412_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L413_loopE; +JLE L(413_loopE); ALIGN_4 -.L413_bodyB: +L(413_bodyB): LD_SY 0*SIZE(ptrba), yvec0; BROAD_SY 0*SIZE(ptrbb), yvec1; MUL_SY yvec0, yvec1, yvec2; ADD_SY yvec2, yvec15, yvec15; ADDQ $8*SIZE, ptrba; ADDQ $1*SIZE, ptrbb; -.L413_loopE: +L(413_loopE): #### Writing #### BROAD_SY MEMALPHA, yvec7; MUL_SY yvec7, yvec15, yvec15; @@ -3311,13 +3311,13 @@ ADDQ $8, kk; #endif ADDQ $8*SIZE, C0; DECQ i; -JG .L41_bodyB; +JG L(41_bodyB); ALIGN_4 -.L41_loopE: +L(41_loopE): TEST $4, bm; -JLE .L42_loopE; +JLE L(42_loopE); ALIGN_4 -.L42_bodyB: +L(42_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -3344,9 +3344,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk #endif SARQ $2, k; -JLE .L421_loopE; +JLE L(421_loopE); ALIGN_4 -.L421_bodyB: +L(421_bodyB): LD_SX 0*SIZE(ptrba), xvec0; BROAD_SX 0*SIZE(ptrbb), xvec1; MUL_SX xvec0, xvec1, xvec1; @@ -3369,17 +3369,17 @@ ADD_SX xvec1, xvec15, xvec15; ADDQ $16*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; DECQ k; -JG .L421_bodyB; +JG L(421_bodyB); ALIGN_4 -.L421_loopE: +L(421_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L422_loopE; +JLE L(422_loopE); ALIGN_4 -.L422_bodyB: +L(422_bodyB): LD_SX 0*SIZE(ptrba), xvec0; BROAD_SX 0*SIZE(ptrbb), xvec1; MUL_SX xvec0, xvec1, xvec1; @@ -3392,15 +3392,15 @@ ADD_SX xvec1, xvec15, xvec15; ADDQ $8*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L422_loopE: +L(422_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L423_loopE; +JLE L(423_loopE); ALIGN_4 -.L423_bodyB: +L(423_bodyB): LD_SX 0*SIZE(ptrba), xvec0; BROAD_SX 0*SIZE(ptrbb), xvec1; MUL_SX xvec0, xvec1, xvec1; @@ -3408,7 +3408,7 @@ ADD_SX xvec1, xvec15, xvec15; ADDQ $4*SIZE, ptrba; ADDQ $1*SIZE, ptrbb; -.L423_loopE: +L(423_loopE): #### Writing back #### BROAD_SX MEMALPHA, xvec7; MUL_SX xvec7, xvec15, xvec15; @@ -3431,11 +3431,11 @@ ADDQ $4, kk #endif ADDQ $4*SIZE, C0; -.L42_loopE: +L(42_loopE): TEST $2, bm; -JLE .L43_loopE; +JLE L(43_loopE); ALIGN_4 -.L43_bodyB: +L(43_bodyB): #if !defined(TRMMKERNEL) || (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -3463,9 +3463,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L431_loopE; +JLE L(431_loopE); ALIGN_4 -.L431_bodyB: +L(431_bodyB): vmovss 0*SIZE(ptrba), xvec0; vmovss 1*SIZE(ptrba), xvec1; vmovss 0*SIZE(ptrbb), xvec2; @@ -3500,17 +3500,17 @@ vaddss xvec4, xvec14, xvec14; addq $8*SIZE, ptrba; addq $4*SIZE, ptrbb; decq k; -JG .L431_bodyB; +JG L(431_bodyB); ALIGN_4 -.L431_loopE: +L(431_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L432_loopE; +JLE L(432_loopE); ALIGN_4 -.L432_bodyB: +L(432_bodyB): vmovss 0*SIZE(ptrba), xvec0; vmovss 1*SIZE(ptrba), xvec1; vmovss 0*SIZE(ptrbb), xvec2; @@ -3529,15 +3529,15 @@ vaddss xvec4, xvec14, xvec14; addq $4*SIZE, ptrba; addq $2*SIZE, ptrbb; -.L432_loopE: +L(432_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L433_loopE; +JLE L(433_loopE); ALIGN_4 -.L433_bodyB: +L(433_bodyB): vmovss 0*SIZE(ptrba), xvec0; vmovss 1*SIZE(ptrba), xvec1; vmovss 0*SIZE(ptrbb), xvec2; @@ -3548,7 +3548,7 @@ vaddss xvec1, xvec14, xvec14; addq $2*SIZE, ptrba; addq $1*SIZE, ptrbb; -.L433_loopE: +L(433_loopE): #### Writing Back #### vmovss MEMALPHA, xvec7; vmulss xvec7, xvec15, xvec15; @@ -3572,11 +3572,11 @@ addq $2, kk #endif addq $2*SIZE, C0; -.L43_loopE: +L(43_loopE): TEST $1, bm; -JLE .L44_loopE; +JLE L(44_loopE); ALIGN_4 -.L44_bodyB: +L(44_bodyB): #if !defined(TRMMKERNEL) || (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) MOVQ bb, ptrbb; #else @@ -3603,9 +3603,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L441_loopE; +JLE L(441_loopE); ALIGN_4 -.L441_bodyB: +L(441_bodyB): vmovss 0*SIZE(ptrba), xvec0; vmovss 0*SIZE(ptrbb), xvec1; vmulss xvec0, xvec1, xvec1; @@ -3628,17 +3628,17 @@ vaddss xvec1, xvec15, xvec15; addq $4*SIZE, ptrba; addq $4*SIZE, ptrbb; decq k; -JG .L441_bodyB; +JG L(441_bodyB); ALIGN_4 -.L441_loopE: +L(441_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L442_loopE; +JLE L(442_loopE); ALIGN_4 -.L442_bodyB: +L(442_bodyB): vmovss 0*SIZE(ptrba), xvec0; vmovss 0*SIZE(ptrbb), xvec1; vmulss xvec0, xvec1, xvec1; @@ -3651,15 +3651,15 @@ vaddss xvec1, xvec15, xvec15; addq $2*SIZE, ptrba; addq $2*SIZE, ptrbb; -.L442_loopE: +L(442_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L443_loopE; +JLE L(443_loopE); ALIGN_4 -.L443_bodyB: +L(443_bodyB): vmovss 0*SIZE(ptrba), xvec0; vmovss 0*SIZE(ptrbb), xvec1; vmulss xvec0, xvec1, xvec1; @@ -3667,7 +3667,7 @@ vaddss xvec1, xvec15, xvec15; addq $1*SIZE, ptrba; addq $1*SIZE, ptrbb; -.L443_loopE: +L(443_loopE): #### Writing Back #### vmovss MEMALPHA, xvec7; vmulss xvec7, xvec15, xvec15; @@ -3687,13 +3687,13 @@ addq $1, kk #endif addq $1*SIZE, C0; -.L44_loopE: +L(44_loopE): MOV bk, k; SALQ $2, k; ADDQ k, bb; ADDQ ldc, C; -.L40_loopE: +L(40_loopE): movq 0(%rsp), %rbx; movq 8(%rsp), %rbp; movq 16(%rsp), %r12; diff --git a/kernel/x86_64/sgemv_n.S b/kernel/x86_64/sgemv_n.S index 8f64fe5e2d..95d92cb757 100644 --- a/kernel/x86_64/sgemv_n.S +++ b/kernel/x86_64/sgemv_n.S @@ -160,19 +160,19 @@ movq LDA,LDAX movq X,XX movq STACK_Y, Y -.L0t: +L(0t): xorq I,I addq $1,I salq $22,I subq I,MMM movq I,M - jge .L00t + jge L(00t) movq MMM,M addq I,M - jle .L999x + jle L(999x) -.L00t: +L(00t): movq AA,A movq NN,N movq LDAX,LDA @@ -191,9 +191,9 @@ #ifdef ALIGNED_ACCESS movq M, MM testq $4 * SIZE - 1, A - je .L0X + je L(0X) cmpq $3, M - jle .L0X + jle L(0X) movq A, MM sarq $BASE_SHIFT, MM @@ -201,13 +201,13 @@ subq $4, MM addq M, MM -.L0X: +L(0X): #endif testq N, N # if n <= 0 goto END - jle .L999 + jle L(999) testq M, M # if n <= 0 goto END - jle .L999 + jle L(999) subq $-32 * SIZE, A @@ -224,33 +224,33 @@ sarq $4, %rax ALIGN_3 -.L01: +L(01): movaps %xmm0, 0 * SIZE(Y1) movaps %xmm0, 4 * SIZE(Y1) movaps %xmm0, 8 * SIZE(Y1) movaps %xmm0, 12 * SIZE(Y1) addq $16 * SIZE, Y1 decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L10: +L(10): #ifdef ALIGNED_ACCESS movq A, %rax andq $4 * SIZE - 1, %rax addq %rax, BUFFER testq $4 * SIZE - 1, LDA - jne .L100 + jne L(100) #endif #if GEMV_UNROLL >= 8 cmpq $8, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $8, N leaq 32 * SIZE(BUFFER), Y1 @@ -297,10 +297,10 @@ #ifdef ALIGNED_ACCESS cmpq $3, M - jle .L17 + jle L(17) testq $SIZE, A1 - je .L1X + je L(1X) movss -32 * SIZE(A1), %xmm4 movss -32 * SIZE(A1, LDA, 1), %xmm5 @@ -338,9 +338,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L1X: +L(1X): testq $2 * SIZE, A1 - je .L1XX + je L(1XX) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA, 1), %xmm5 @@ -378,11 +378,11 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L1XX: +L(1XX): #endif movq MM, I sarq $4, I - jle .L15 + jle L(15) MOVUPS_A1 (-32 * SIZE, A1, %xmm4) MOVUPS_A1 (-28 * SIZE, A1, %xmm5) @@ -395,10 +395,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L14 + jle L(14) ALIGN_3 -.L13: +L(13): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -555,10 +555,10 @@ subq $1, I BRANCH - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): mulps %xmm8, %xmm4 addps %xmm4, %xmm0 MOVUPS_A2 (-32 * SIZE, A1, LDA, 1, %xmm4) @@ -669,9 +669,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L15: +L(15): testq $8, MM - je .L16 + je L(16) MOVUPS_A1 (-32 * SIZE, A1, %xmm4) MOVUPS_A1 (-28 * SIZE, A1, %xmm5) @@ -741,9 +741,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L16: +L(16): testq $4, MM - je .L17 + je L(17) MOVUPS_A1 (-32 * SIZE, A1, %xmm4) MOVUPS_A2 (-32 * SIZE, A1, LDA, 1, %xmm5) @@ -781,9 +781,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L17: +L(17): testq $2, MM - je .L18 + je L(18) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA, 1), %xmm5 @@ -821,9 +821,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L18: +L(18): testq $1, MM - je .L19 + je L(19) movss -32 * SIZE(A1), %xmm4 movss -32 * SIZE(A1, LDA, 1), %xmm5 @@ -857,21 +857,21 @@ movss %xmm0, -32 * SIZE(Y1) ALIGN_3 -.L19: +L(19): cmpq $8, N - jge .L11 + jge L(11) ALIGN_3 -.L20: +L(20): #endif cmpq $4, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 4 ALIGN_3 -.L21: +L(21): #endif subq $4, N @@ -903,10 +903,10 @@ #ifdef ALIGNED_ACCESS cmpq $3, M - jle .L27 + jle L(27) testq $SIZE, A1 - je .L2X + je L(2X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(A1, LDA), %xmm1 @@ -931,9 +931,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L2X: +L(2X): testq $2 * SIZE, A1 - je .L2XX + je L(2XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A1, LDA), %xmm1 @@ -958,12 +958,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L2XX: +L(2XX): #endif movq MM, I sarq $4, I - jle .L25 + jle L(25) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) MOVUPS_A1 (-28 * SIZE, A1, %xmm9) @@ -981,10 +981,10 @@ MOVUPS_A2(-20 * SIZE, A1, LDA, 1, %xmm7) decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -1073,10 +1073,10 @@ subq $1, I BRANCH - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): mulps %xmm12, %xmm8 addps %xmm8, %xmm0 MOVUPS_A1(-32 * SIZE, A2, %xmm8) @@ -1130,9 +1130,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L25: +L(25): testq $8, MM - je .L26 + je L(26) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) MOVUPS_A1 (-28 * SIZE, A1, %xmm9) @@ -1176,9 +1176,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L26: +L(26): testq $4, MM - je .L27 + je L(27) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) @@ -1206,9 +1206,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L27: +L(27): testq $2, MM - je .L28 + je L(28) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A1, LDA), %xmm1 @@ -1233,12 +1233,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L28: +L(28): testq $1, MM #if GEMV_UNROLL == 4 - je .L29 + je L(29) #else - je .L30 + je L(30) #endif movss -32 * SIZE(Y1), %xmm8 @@ -1261,18 +1261,18 @@ ALIGN_3 #if GEMV_UNROLL == 4 -.L29: +L(29): cmpq $4, N - jge .L21 + jge L(21) #endif ALIGN_3 -.L30: +L(30): testq N, N - jle .L990 + jle L(990) cmpq $3, N - jne .L40 + jne L(40) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -1297,10 +1297,10 @@ #ifdef ALIGNED_ACCESS cmpq $3, M - jle .L37 + jle L(37) testq $SIZE, A1 - je .L3X + je L(3X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(A1, LDA), %xmm1 @@ -1322,9 +1322,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L3X: +L(3X): testq $2 * SIZE, A1 - je .L3XX + je L(3XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A1, LDA), %xmm1 @@ -1346,12 +1346,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L3XX: +L(3XX): #endif movq MM, I sarq $4, I - jle .L35 + jle L(35) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) MOVUPS_A1 (-28 * SIZE, A1, %xmm9) @@ -1369,10 +1369,10 @@ MOVUPS_A2(-20 * SIZE, A1, LDA, 1, %xmm7) decq I - jle .L34 + jle L(34) ALIGN_3 -.L33: +L(33): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 3 - 128 + PREOFFSET(A1) #endif @@ -1444,10 +1444,10 @@ subq $1, I BRANCH - jg .L33 + jg L(33) ALIGN_3 -.L34: +L(34): mulps %xmm12, %xmm8 addps %xmm8, %xmm0 MOVUPS_A1(-32 * SIZE, A2, %xmm8) @@ -1488,9 +1488,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L35: +L(35): testq $8, MM - je .L36 + je L(36) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) MOVUPS_A1 (-28 * SIZE, A1, %xmm9) @@ -1528,9 +1528,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L36: +L(36): testq $4, MM - je .L37 + je L(37) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) @@ -1555,9 +1555,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L37: +L(37): testq $2, MM - je .L38 + je L(38) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A1, LDA), %xmm1 @@ -1579,9 +1579,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L38: +L(38): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm8 @@ -1597,12 +1597,12 @@ addss %xmm2, %xmm8 movss %xmm8, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_3 -.L40: +L(40): cmpq $2, N - jne .L50 + jne L(50) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -1624,10 +1624,10 @@ #ifdef ALIGNED_ACCESS cmpq $3, M - jle .L47 + jle L(47) testq $SIZE, A1 - je .L4X + je L(4X) movss -32 * SIZE(Y1), %xmm8 @@ -1646,9 +1646,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L4X: +L(4X): testq $2 * SIZE, A1 - je .L4XX + je L(4XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A2), %xmm1 @@ -1667,12 +1667,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L4XX: +L(4XX): #endif movq MM, I sarq $4, I - jle .L45 + jle L(45) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) MOVUPS_A1 (-28 * SIZE, A1, %xmm9) @@ -1690,10 +1690,10 @@ MOVUPS_A1(-20 * SIZE, A2, %xmm7) decq I - jle .L44 + jle L(44) ALIGN_3 -.L43: +L(43): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -1748,10 +1748,10 @@ subq $1, I BRANCH - jg .L43 + jg L(43) ALIGN_3 -.L44: +L(44): mulps %xmm12, %xmm8 addps %xmm8, %xmm0 mulps %xmm12, %xmm9 @@ -1779,9 +1779,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L45: +L(45): testq $8, MM - je .L46 + je L(46) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) MOVUPS_A1 (-28 * SIZE, A1, %xmm9) @@ -1808,9 +1808,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L46: +L(46): testq $4, MM - je .L47 + je L(47) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) @@ -1828,9 +1828,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L47: +L(47): testq $2, MM - je .L48 + je L(48) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A2), %xmm1 @@ -1849,9 +1849,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L48: +L(48): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm8 @@ -1864,12 +1864,12 @@ addss %xmm1, %xmm8 movss %xmm8, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_3 -.L50: +L(50): cmpq $1, N - jne .L990 + jne L(990) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -1881,10 +1881,10 @@ #ifdef ALIGNED_ACCESS cmpq $3, M - jle .L57 + jle L(57) testq $SIZE, A1 - je .L5X + je L(5X) movss -32 * SIZE(Y1), %xmm8 movss -32 * SIZE(A1), %xmm0 @@ -1899,9 +1899,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L5X: +L(5X): testq $2 * SIZE, A1 - je .L5XX + je L(5XX) movsd -32 * SIZE(Y1), %xmm8 movsd -32 * SIZE(A1), %xmm0 @@ -1916,12 +1916,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L5XX: +L(5XX): #endif movq MM, I sarq $4, I - jle .L55 + jle L(55) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) MOVUPS_A1 (-28 * SIZE, A1, %xmm9) @@ -1934,10 +1934,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L54 + jle L(54) ALIGN_3 -.L53: +L(53): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -1974,10 +1974,10 @@ subq $1, I BRANCH - jg .L53 + jg L(53) ALIGN_3 -.L54: +L(54): mulps %xmm12, %xmm8 addps %xmm8, %xmm0 MOVUPS_YS1(-32 * SIZE, Y1, %xmm0) @@ -1995,9 +1995,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L55: +L(55): testq $8, MM - je .L56 + je L(56) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) MOVUPS_A1 (-28 * SIZE, A1, %xmm9) @@ -2016,9 +2016,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L56: +L(56): testq $4, MM - je .L57 + je L(57) MOVUPS_A1 (-32 * SIZE, A1, %xmm8) MOVUPS_YL1(-32 * SIZE, Y1, %xmm0) @@ -2031,9 +2031,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L57: +L(57): testq $2, MM - je .L58 + je L(58) movsd -32 * SIZE(Y1), %xmm8 movsd -32 * SIZE(A1), %xmm0 @@ -2047,9 +2047,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L58: +L(58): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm8 movss -32 * SIZE(A1), %xmm0 @@ -2060,18 +2060,18 @@ movss %xmm8, -32 * SIZE(Y1) #ifdef ALIGNED_ACCESS - jmp .L990 + jmp L(990) ALIGN_3 -.L100: +L(100): testq $2 * SIZE - 1, LDA - jne .L200 + jne L(200) cmpq $4, N - jl .L110 + jl L(110) ALIGN_3 -.L101: +L(101): subq $4, N leaq 32 * SIZE(BUFFER), Y1 @@ -2101,10 +2101,10 @@ shufps $0, %xmm15, %xmm15 cmpq $3, M - jle .L107 + jle L(107) testq $SIZE, A1 - je .L10X + je L(10X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(A1, LDA), %xmm1 @@ -2129,9 +2129,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L10X: +L(10X): testq $2 * SIZE, A1 - je .L10XX + je L(10XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A1, LDA), %xmm1 @@ -2156,13 +2156,13 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L10XX: +L(10XX): movhps -32 * SIZE(A1, LDA), %xmm8 movhps -32 * SIZE(A2, LDA), %xmm9 movq MM, I sarq $4, I - jle .L105 + jle L(105) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -2174,10 +2174,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L104 + jle L(104) ALIGN_3 -.L103: +L(103): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -2274,10 +2274,10 @@ subq $1, I BRANCH - jg .L103 + jg L(103) ALIGN_3 -.L104: +L(104): mulps %xmm12, %xmm4 addps %xmm4, %xmm0 movaps -20 * SIZE(A1), %xmm7 @@ -2345,9 +2345,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L105: +L(105): testq $8, MM - je .L106 + je L(106) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -2395,9 +2395,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L106: +L(106): testq $4, MM - je .L107 + je L(107) movaps -32 * SIZE(A1), %xmm4 movaps -30 * SIZE(A1, LDA), %xmm5 @@ -2424,9 +2424,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L107: +L(107): testq $2, MM - je .L108 + je L(108) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA), %xmm5 @@ -2451,9 +2451,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L108: +L(108): testq $1, MM - je .L109 + je L(109) movss -32 * SIZE(Y1), %xmm0 @@ -2474,17 +2474,17 @@ movss %xmm0, -32 * SIZE(Y1) ALIGN_3 -.L109: +L(109): cmpq $4, N - jge .L101 + jge L(101) ALIGN_3 -.L110: +L(110): testq N, N - jle .L990 + jle L(990) cmpq $3, N - jne .L120 + jne L(120) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -2509,10 +2509,10 @@ shufps $0, %xmm14, %xmm14 cmpq $3, M - jle .L117 + jle L(117) testq $SIZE, A1 - je .L11X + je L(11X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(A1, LDA), %xmm1 @@ -2534,9 +2534,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L11X: +L(11X): testq $2 * SIZE, A1 - je .L11XX + je L(11XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A1, LDA), %xmm1 @@ -2558,13 +2558,13 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L11XX: +L(11XX): movhps -32 * SIZE(A1, LDA), %xmm8 movhps -32 * SIZE(A2, LDA), %xmm9 movq MM, I sarq $4, I - jle .L115 + jle L(115) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -2576,10 +2576,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L114 + jle L(114) ALIGN_3 -.L113: +L(113): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 3 - 128 + PREOFFSET(A1) #endif @@ -2655,10 +2655,10 @@ subq $1, I BRANCH - jg .L113 + jg L(113) ALIGN_3 -.L114: +L(114): mulps %xmm12, %xmm4 addps %xmm4, %xmm0 movaps -20 * SIZE(A1), %xmm7 @@ -2709,9 +2709,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L115: +L(115): testq $8, MM - je .L116 + je L(116) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -2749,9 +2749,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L116: +L(116): testq $4, MM - je .L117 + je L(117) movaps -32 * SIZE(A1), %xmm4 movaps -30 * SIZE(A1, LDA), %xmm5 @@ -2774,9 +2774,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L117: +L(117): testq $2, MM - je .L118 + je L(118) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA), %xmm5 @@ -2798,9 +2798,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L118: +L(118): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm0 @@ -2816,12 +2816,12 @@ addss %xmm6, %xmm0 movss %xmm0, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_3 -.L120: +L(120): cmpq $2, N - jl .L130 + jl L(130) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -2842,10 +2842,10 @@ shufps $0, %xmm13, %xmm13 cmpq $3, M - jle .L127 + jle L(127) testq $SIZE, A1 - je .L12X + je L(12X) movss -32 * SIZE(Y1), %xmm8 @@ -2864,9 +2864,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L12X: +L(12X): testq $2 * SIZE, A1 - je .L12XX + je L(12XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A2), %xmm1 @@ -2885,12 +2885,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L12XX: +L(12XX): movhps -32 * SIZE(A2), %xmm8 movq MM, I sarq $4, I - jle .L125 + jle L(125) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -2902,10 +2902,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L124 + jle L(124) ALIGN_3 -.L123: +L(123): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -2964,10 +2964,10 @@ subq $1, I BRANCH - jg .L123 + jg L(123) ALIGN_3 -.L124: +L(124): mulps %xmm12, %xmm4 addps %xmm4, %xmm0 movaps -20 * SIZE(A1), %xmm7 @@ -3005,9 +3005,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L125: +L(125): testq $8, MM - je .L126 + je L(126) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -3038,9 +3038,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L126: +L(126): testq $4, MM - je .L127 + je L(127) movaps -32 * SIZE(A1), %xmm4 movaps -30 * SIZE(A2), %xmm5 @@ -3060,9 +3060,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L127: +L(127): testq $2, MM - je .L128 + je L(128) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A2), %xmm1 @@ -3081,9 +3081,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L128: +L(128): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm8 @@ -3096,12 +3096,12 @@ addss %xmm1, %xmm8 movss %xmm8, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_3 -.L130: +L(130): cmpq $1, N - jne .L990 + jne L(990) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -3112,10 +3112,10 @@ shufps $0, %xmm12, %xmm12 cmpq $3, M - jle .L137 + jle L(137) testq $SIZE, A1 - je .L13X + je L(13X) movss -32 * SIZE(Y1), %xmm8 movss -32 * SIZE(A1), %xmm0 @@ -3130,9 +3130,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L13X: +L(13X): testq $2 * SIZE, A1 - je .L13XX + je L(13XX) movsd -32 * SIZE(Y1), %xmm8 movsd -32 * SIZE(A1), %xmm0 @@ -3147,10 +3147,10 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L13XX: +L(13XX): movq MM, I sarq $4, I - jle .L135 + jle L(135) movaps -32 * SIZE(A1), %xmm8 movaps -28 * SIZE(A1), %xmm9 @@ -3163,10 +3163,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L134 + jle L(134) ALIGN_3 -.L133: +L(133): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -3203,10 +3203,10 @@ subq $1, I BRANCH - jg .L133 + jg L(133) ALIGN_3 -.L134: +L(134): mulps %xmm12, %xmm8 addps %xmm8, %xmm0 MOVUPS_YS1(-32 * SIZE, Y1, %xmm0) @@ -3224,9 +3224,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L135: +L(135): testq $8, MM - je .L136 + je L(136) movaps -32 * SIZE(A1), %xmm8 movaps -28 * SIZE(A1), %xmm9 @@ -3245,9 +3245,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L136: +L(136): testq $4, MM - je .L137 + je L(137) movaps -32 * SIZE(A1), %xmm8 MOVUPS_YL1(-32 * SIZE, Y1, %xmm0) @@ -3260,9 +3260,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L137: +L(137): testq $2, MM - je .L138 + je L(138) movsd -32 * SIZE(Y1), %xmm8 movsd -32 * SIZE(A1), %xmm0 @@ -3276,9 +3276,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L138: +L(138): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm8 movss -32 * SIZE(A1), %xmm0 @@ -3287,18 +3287,18 @@ addss %xmm0, %xmm8 movss %xmm8, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_3 -.L200: +L(200): testq $2 * SIZE, LDA - jne .L300 + jne L(300) cmpq $4, N - jl .L210 + jl L(210) ALIGN_3 -.L201: +L(201): subq $4, N leaq 32 * SIZE(BUFFER), Y1 @@ -3328,10 +3328,10 @@ shufps $0, %xmm15, %xmm15 cmpq $3, M - jle .L207 + jle L(207) testq $SIZE, A1 - je .L20X + je L(20X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(A1, LDA), %xmm1 @@ -3356,9 +3356,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L20X: +L(20X): testq $2 * SIZE, A1 - je .L20XX + je L(20XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A1, LDA), %xmm1 @@ -3383,14 +3383,14 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L20XX: +L(20XX): movaps -33 * SIZE(A1, LDA), %xmm8 movaps -34 * SIZE(A2), %xmm9 movaps -35 * SIZE(A2, LDA), %xmm10 movq MM, I sarq $4, I - jle .L205 + jle L(205) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -3402,10 +3402,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L204 + jle L(204) ALIGN_3 -.L203: +L(203): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -3514,10 +3514,10 @@ subq $1, I BRANCH - jg .L203 + jg L(203) ALIGN_3 -.L204: +L(204): mulps %xmm12, %xmm4 addps %xmm4, %xmm0 movaps -20 * SIZE(A1), %xmm7 @@ -3599,9 +3599,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L205: +L(205): testq $8, MM - je .L206 + je L(206) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -3657,9 +3657,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L206: +L(206): testq $4, MM - je .L207 + je L(207) movaps -32 * SIZE(A1), %xmm4 movaps -29 * SIZE(A1, LDA), %xmm5 @@ -3690,9 +3690,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L207: +L(207): testq $2, MM - je .L208 + je L(208) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA), %xmm5 @@ -3717,9 +3717,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L208: +L(208): testq $1, MM - je .L209 + je L(209) movss -32 * SIZE(Y1), %xmm0 @@ -3740,14 +3740,14 @@ movss %xmm0, -32 * SIZE(Y1) ALIGN_3 -.L209: +L(209): cmpq $4, N - jge .L201 + jge L(201) ALIGN_3 -.L210: +L(210): cmpq $3, N - jne .L220 + jne L(220) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -3772,10 +3772,10 @@ shufps $0, %xmm14, %xmm14 cmpq $3, M - jle .L217 + jle L(217) testq $SIZE, A1 - je .L21X + je L(21X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(A1, LDA), %xmm1 @@ -3797,9 +3797,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L21X: +L(21X): testq $2 * SIZE, A1 - je .L21XX + je L(21XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A1, LDA), %xmm1 @@ -3821,14 +3821,14 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L21XX: +L(21XX): movaps -33 * SIZE(A1, LDA), %xmm8 movaps -34 * SIZE(A2), %xmm9 movaps -35 * SIZE(A2, LDA), %xmm10 movq MM, I sarq $4, I - jle .L215 + jle L(215) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -3840,10 +3840,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L214 + jle L(214) ALIGN_3 -.L213: +L(213): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 3 - 128 + PREOFFSET(A1) #endif @@ -3927,10 +3927,10 @@ subq $1, I BRANCH - jg .L213 + jg L(213) ALIGN_3 -.L214: +L(214): mulps %xmm12, %xmm4 addps %xmm4, %xmm0 movaps -20 * SIZE(A1), %xmm7 @@ -3990,9 +3990,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L215: +L(215): testq $8, MM - je .L216 + je L(216) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -4035,9 +4035,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L216: +L(216): testq $4, MM - je .L217 + je L(217) movaps -32 * SIZE(A1), %xmm4 movaps -29 * SIZE(A1, LDA), %xmm5 @@ -4064,9 +4064,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L217: +L(217): testq $2, MM - je .L218 + je L(218) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA), %xmm5 @@ -4088,9 +4088,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L218: +L(218): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm0 @@ -4106,15 +4106,15 @@ addss %xmm6, %xmm0 movss %xmm0, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_4 -.L220: +L(220): testq N, N - jle .L990 + jle L(990) cmpq $2, N - jne .L230 + jne L(230) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -4135,10 +4135,10 @@ shufps $0, %xmm13, %xmm13 cmpq $3, M - jle .L227 + jle L(227) testq $SIZE, A1 - je .L22X + je L(22X) movss -32 * SIZE(Y1), %xmm9 @@ -4157,9 +4157,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L22X: +L(22X): testq $2 * SIZE, A1 - je .L22XX + je L(22XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A2), %xmm1 @@ -4178,12 +4178,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L22XX: +L(22XX): movaps -33 * SIZE(A1, LDA), %xmm8 movq MM, I sarq $4, I - jle .L225 + jle L(225) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -4195,10 +4195,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L224 + jle L(224) ALIGN_3 -.L223: +L(223): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -4262,10 +4262,10 @@ subq $1, I BRANCH - jg .L223 + jg L(223) ALIGN_3 -.L224: +L(224): mulps %xmm12, %xmm4 addps %xmm4, %xmm0 movaps -20 * SIZE(A1), %xmm7 @@ -4308,9 +4308,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L225: +L(225): testq $8, MM - je .L226 + je L(226) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -4344,9 +4344,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L226: +L(226): testq $4, MM - je .L227 + je L(227) movaps -32 * SIZE(A1), %xmm4 movaps -29 * SIZE(A2), %xmm5 @@ -4367,9 +4367,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L227: +L(227): testq $2, MM - je .L228 + je L(228) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A2), %xmm1 @@ -4388,9 +4388,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L228: +L(228): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm9 @@ -4403,12 +4403,12 @@ addss %xmm1, %xmm9 movss %xmm9, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_3 -.L230: +L(230): cmpq $1, N - jne .L990 + jne L(990) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -4419,10 +4419,10 @@ shufps $0, %xmm12, %xmm12 cmpq $3, M - jle .L237 + jle L(237) testq $SIZE, A1 - je .L23X + je L(23X) movss -32 * SIZE(Y1), %xmm8 movss -32 * SIZE(A1), %xmm0 @@ -4436,9 +4436,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L23X: +L(23X): testq $2 * SIZE, A1 - je .L23XX + je L(23XX) movsd -32 * SIZE(Y1), %xmm8 movsd -32 * SIZE(A1), %xmm0 @@ -4452,13 +4452,13 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L23XX: +L(23XX): testq $2 * SIZE, A1 - jne .L230 + jne L(230) movq MM, I sarq $4, I - jle .L235 + jle L(235) movaps -32 * SIZE(A1), %xmm8 movaps -28 * SIZE(A1), %xmm9 @@ -4471,10 +4471,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L234 + jle L(234) ALIGN_3 -.L233: +L(233): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -4511,10 +4511,10 @@ subq $1, I BRANCH - jg .L233 + jg L(233) ALIGN_3 -.L234: +L(234): mulps %xmm12, %xmm8 addps %xmm8, %xmm0 MOVUPS_YS1(-32 * SIZE, Y1, %xmm0) @@ -4532,9 +4532,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L235: +L(235): testq $8, MM - je .L236 + je L(236) movaps -32 * SIZE(A1), %xmm8 movaps -28 * SIZE(A1), %xmm9 @@ -4553,9 +4553,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L236: +L(236): testq $4, MM - je .L237 + je L(237) movaps -32 * SIZE(A1), %xmm8 MOVUPS_YL1(-32 * SIZE, Y1, %xmm0) @@ -4568,9 +4568,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L237: +L(237): testq $2, MM - je .L238 + je L(238) movsd -32 * SIZE(Y1), %xmm8 movsd -32 * SIZE(A1), %xmm0 @@ -4584,9 +4584,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L238: +L(238): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm8 movss -32 * SIZE(A1), %xmm0 @@ -4595,15 +4595,15 @@ addss %xmm0, %xmm8 movss %xmm8, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_4 -.L300: +L(300): cmpq $4, N - jl .L310 + jl L(310) ALIGN_3 -.L301: +L(301): subq $4, N leaq 32 * SIZE(BUFFER), Y1 @@ -4633,10 +4633,10 @@ shufps $0, %xmm15, %xmm15 cmpq $3, M - jle .L307 + jle L(307) testq $SIZE, A1 - je .L30X + je L(30X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(A1, LDA), %xmm1 @@ -4661,9 +4661,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L30X: +L(30X): testq $2 * SIZE, A1 - je .L30XX + je L(30XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A1, LDA), %xmm1 @@ -4688,14 +4688,14 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L30XX: +L(30XX): movaps -35 * SIZE(A1, LDA), %xmm8 movaps -34 * SIZE(A2), %xmm9 movaps -33 * SIZE(A2, LDA), %xmm10 movq MM, I sarq $4, I - jle .L305 + jle L(305) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -4707,10 +4707,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L304 + jle L(304) ALIGN_3 -.L303: +L(303): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -4819,10 +4819,10 @@ subq $1, I BRANCH - jg .L303 + jg L(303) ALIGN_3 -.L304: +L(304): mulps %xmm12, %xmm4 addps %xmm4, %xmm0 movaps -20 * SIZE(A1), %xmm7 @@ -4904,9 +4904,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L305: +L(305): testq $8, MM - je .L306 + je L(306) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -4961,9 +4961,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L306: +L(306): testq $4, MM - je .L307 + je L(307) movaps -32 * SIZE(A1), %xmm4 movaps -31 * SIZE(A1, LDA), %xmm5 @@ -4994,9 +4994,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L307: +L(307): testq $2, MM - je .L308 + je L(308) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA), %xmm5 @@ -5021,9 +5021,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L308: +L(308): testq $1, MM - je .L309 + je L(309) movss -32 * SIZE(Y1), %xmm0 @@ -5044,14 +5044,14 @@ movss %xmm0, -32 * SIZE(Y1) ALIGN_3 -.L309: +L(309): cmpq $4, N - jge .L301 + jge L(301) ALIGN_3 -.L310: +L(310): cmpq $3, N - jne .L320 + jne L(320) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -5076,10 +5076,10 @@ shufps $0, %xmm14, %xmm14 cmpq $3, M - jle .L317 + jle L(317) testq $SIZE, A1 - je .L31X + je L(31X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(A1, LDA), %xmm1 @@ -5101,9 +5101,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L31X: +L(31X): testq $2 * SIZE, A1 - je .L31XX + je L(31XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A1, LDA), %xmm1 @@ -5125,14 +5125,14 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L31XX: +L(31XX): movaps -35 * SIZE(A1, LDA), %xmm8 movaps -34 * SIZE(A2), %xmm9 movaps -33 * SIZE(A2, LDA), %xmm10 movq MM, I sarq $4, I - jle .L315 + jle L(315) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -5144,10 +5144,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L314 + jle L(314) ALIGN_3 -.L313: +L(313): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 3 - 128 + PREOFFSET(A1) #endif @@ -5231,10 +5231,10 @@ subq $1, I BRANCH - jg .L313 + jg L(313) ALIGN_3 -.L314: +L(314): mulps %xmm12, %xmm4 addps %xmm4, %xmm0 movaps -20 * SIZE(A1), %xmm7 @@ -5295,9 +5295,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L315: +L(315): testq $8, MM - je .L316 + je L(316) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -5340,9 +5340,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L316: +L(316): testq $4, MM - je .L317 + je L(317) movaps -32 * SIZE(A1), %xmm4 movaps -31 * SIZE(A1, LDA), %xmm5 @@ -5368,9 +5368,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L317: +L(317): testq $2, MM - je .L318 + je L(318) movsd -32 * SIZE(A1), %xmm4 movsd -32 * SIZE(A1, LDA), %xmm5 @@ -5392,9 +5392,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L318: +L(318): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm0 @@ -5410,12 +5410,12 @@ addss %xmm6, %xmm0 movss %xmm0, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_3 -.L320: +L(320): cmpq $2, N - jne .L330 + jne L(330) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -5435,10 +5435,10 @@ shufps $0, %xmm13, %xmm13 cmpq $3, M - jle .L327 + jle L(327) testq $SIZE, A1 - je .L32X + je L(32X) movss -32 * SIZE(Y1), %xmm9 @@ -5457,9 +5457,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L32X: +L(32X): testq $2 * SIZE, A1 - je .L32XX + je L(32XX) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A2), %xmm1 @@ -5478,12 +5478,12 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L32XX: +L(32XX): movaps -35 * SIZE(A1, LDA), %xmm8 movq MM, I sarq $4, I - jle .L325 + jle L(325) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -5495,10 +5495,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L324 + jle L(324) ALIGN_3 -.L323: +L(323): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -5561,10 +5561,10 @@ subq $1, I BRANCH - jg .L323 + jg L(323) ALIGN_3 -.L324: +L(324): mulps %xmm12, %xmm4 addps %xmm4, %xmm0 movaps -20 * SIZE(A1), %xmm7 @@ -5608,9 +5608,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L325: +L(325): testq $8, MM - je .L326 + je L(326) movaps -32 * SIZE(A1), %xmm4 movaps -28 * SIZE(A1), %xmm5 @@ -5643,9 +5643,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L326: +L(326): testq $4, MM - je .L327 + je L(327) movaps -32 * SIZE(A1), %xmm4 movaps -31 * SIZE(A2), %xmm5 @@ -5666,9 +5666,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L327: +L(327): testq $2, MM - je .L328 + je L(328) movsd -32 * SIZE(A1), %xmm0 movsd -32 * SIZE(A2), %xmm1 @@ -5687,9 +5687,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L328: +L(328): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm8 @@ -5702,12 +5702,12 @@ addss %xmm1, %xmm8 movss %xmm8, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) ALIGN_3 -.L330: +L(330): cmpq $1, N - jne .L990 + jne L(990) leaq 32 * SIZE(BUFFER), Y1 movq A, A1 @@ -5718,10 +5718,10 @@ shufps $0, %xmm12, %xmm12 cmpq $3, M - jle .L337 + jle L(337) testq $SIZE, A1 - je .L33X + je L(33X) movss -32 * SIZE(Y1), %xmm8 movss -32 * SIZE(A1), %xmm0 @@ -5735,9 +5735,9 @@ addq $1 * SIZE, Y1 ALIGN_3 -.L33X: +L(33X): testq $2 * SIZE, A1 - je .L33XX + je L(33XX) movsd -32 * SIZE(Y1), %xmm8 movsd -32 * SIZE(A1), %xmm0 @@ -5751,10 +5751,10 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L33XX: +L(33XX): movq MM, I sarq $4, I - jle .L335 + jle L(335) movaps -32 * SIZE(A1), %xmm8 movaps -28 * SIZE(A1), %xmm9 @@ -5767,10 +5767,10 @@ MOVUPS_YL1(-20 * SIZE, Y1, %xmm3) decq I - jle .L334 + jle L(334) ALIGN_3 -.L333: +L(333): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -5807,10 +5807,10 @@ subq $1, I BRANCH - jg .L333 + jg L(333) ALIGN_3 -.L334: +L(334): mulps %xmm12, %xmm8 addps %xmm8, %xmm0 MOVUPS_YS1(-32 * SIZE, Y1, %xmm0) @@ -5828,9 +5828,9 @@ subq $-16 * SIZE, Y1 ALIGN_3 -.L335: +L(335): testq $8, MM - je .L336 + je L(336) movaps -32 * SIZE(A1), %xmm8 movaps -28 * SIZE(A1), %xmm9 @@ -5849,9 +5849,9 @@ addq $8 * SIZE, Y1 ALIGN_3 -.L336: +L(336): testq $4, MM - je .L337 + je L(337) movaps -32 * SIZE(A1), %xmm8 MOVUPS_YL1(-32 * SIZE, Y1, %xmm0) @@ -5864,9 +5864,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L337: +L(337): testq $2, MM - je .L338 + je L(338) movsd -32 * SIZE(Y1), %xmm8 movsd -32 * SIZE(A1), %xmm0 @@ -5880,9 +5880,9 @@ addq $2 * SIZE, Y1 ALIGN_3 -.L338: +L(338): testq $1, MM - je .L990 + je L(990) movss -32 * SIZE(Y1), %xmm8 movss -32 * SIZE(A1), %xmm0 @@ -5891,19 +5891,19 @@ addss %xmm0, %xmm8 movss %xmm8, -32 * SIZE(Y1) - jmp .L990 + jmp L(990) #endif ALIGN_4 -.L990: +L(990): movq Y, Y1 movq M, %rax sarq $3, %rax - jle .L994 + jle L(994) ALIGN_3 -.L992: +L(992): movsd 0 * SIZE(BUFFER), %xmm0 movhps 2 * SIZE(BUFFER), %xmm0 movsd 4 * SIZE(BUFFER), %xmm4 @@ -5953,15 +5953,15 @@ addq $8 * SIZE, BUFFER decq %rax - jg .L992 + jg L(992) ALIGN_3 -.L994: +L(994): testq $7, M - jle .L999 + jle L(999) testq $4, M - jle .L995 + jle L(995) movsd 0 * SIZE(BUFFER), %xmm0 movhps 2 * SIZE(BUFFER), %xmm0 @@ -5991,9 +5991,9 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L995: +L(995): testq $2, M - jle .L996 + jle L(996) movsd (BUFFER), %xmm0 @@ -6012,9 +6012,9 @@ addq $2 * SIZE, BUFFER ALIGN_3 -.L996: +L(996): testq $1, M - jle .L999 + jle L(999) movss (BUFFER), %xmm0 @@ -6023,13 +6023,13 @@ movss %xmm0, (Y1) ALIGN_3 -.L999: +L(999): leaq (,M,SIZE),%rax addq %rax,AA - jmp .L0t + jmp L(0t) ALIGN_4 -.L999x: +L(999x): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/sgemv_t.S b/kernel/x86_64/sgemv_t.S index 33e2fa86cc..013806ef8a 100644 --- a/kernel/x86_64/sgemv_t.S +++ b/kernel/x86_64/sgemv_t.S @@ -154,20 +154,20 @@ #endif -.L0t: +L(0t): xorq M,M addq $1,M salq $22,M subq M,MMM - jge .L00t + jge L(00t) ALIGN_4 movq MMM,%rax addq M,%rax - jle .L999x + jle L(999x) movq %rax,M -.L00t: +L(00t): movq LDAX,LDA movq NN,N movq AA,A @@ -185,9 +185,9 @@ #ifdef ALIGNED_ACCESS movq M, MM testq $4 * SIZE - 1, A - je .L0X + je L(0X) cmpq $3, M - jle .L0X + jle L(0X) movq A, MM sarq $BASE_SHIFT, MM @@ -195,13 +195,13 @@ subq $4, MM addq M, MM -.L0X: +L(0X): #endif testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) ALIGN_4 subq $-32 * SIZE, A @@ -216,10 +216,10 @@ movq M, I sarq $3, I - jle .L05 + jle L(05) ALIGN_4 -.L02: +L(02): movss (X), %xmm0 addq INCX, X movss (X), %xmm1 @@ -251,39 +251,39 @@ addq $8 * SIZE, X1 decq I - jg .L02 + jg L(02) ALIGN_4 -.L05: +L(05): movq M, I andq $7, I - jle .L10 + jle L(10) ALIGN_2 -.L06: +L(06): movss (X), %xmm0 addq INCX, X movss %xmm0, 0 * SIZE(X1) addq $SIZE, X1 decq I - jg .L06 + jg L(06) ALIGN_4 -.L10: +L(10): movq Y, Y1 #ifdef ALIGNED_ACCESS testq $4 * SIZE - 1, LDA - jne .L100 + jne L(100) #endif #if GEMV_UNROLL >= 8 cmpq $8, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $8, N leaq 32 * SIZE(BUFFER), X1 @@ -303,10 +303,10 @@ #ifdef ALIGNED_ACCESS cmpq $3, M - jle .L17 + jle L(17) testq $SIZE, A1 - je .L1X + je L(1X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -339,9 +339,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L1X: +L(1X): testq $2 * SIZE, A1 - je .L1XX + je L(1XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -387,7 +387,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L1XX: +L(1XX): #endif MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) @@ -399,7 +399,7 @@ movq MM, I sarq $4, I - jle .L15 + jle L(15) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-32 * SIZE, A1, LDA, 1, %xmm1) @@ -407,10 +407,10 @@ MOVUPS_A2 (-32 * SIZE, A1, LDA3, 1, %xmm3) decq I - jle .L13 + jle L(13) ALIGN_4 -.L12: +L(12): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -561,10 +561,10 @@ addq $16 * SIZE, X1 decq I - jg .L12 + jg L(12) ALIGN_4 -.L13: +L(13): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-32 * SIZE, A2, %xmm0) @@ -670,9 +670,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L15: +L(15): testq $8, MM - jle .L16 + jle L(16) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-32 * SIZE, A1, LDA, 1, %xmm1) @@ -731,9 +731,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L16: +L(16): testq $4, MM - jle .L17 + jle L(17) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-32 * SIZE, A1, LDA, 1, %xmm1) @@ -766,9 +766,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L17: +L(17): testq $2, MM - jle .L18 + jle L(18) #ifdef movsd xorps %xmm0, %xmm0 @@ -816,9 +816,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L18: +L(18): testq $1, MM - jle .L19 + jle L(19) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -847,7 +847,7 @@ addss %xmm3, %xmm15 ALIGN_4 -.L19: +L(19): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm11, %xmm10 @@ -958,19 +958,19 @@ addq INCY, Y1 cmpq $8, N - jge .L11 + jge L(11) ALIGN_4 -.L20: +L(20): #endif cmpq $4, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 4 ALIGN_3 -.L21: +L(21): #endif subq $4, N @@ -987,10 +987,10 @@ #ifdef ALIGNED_ACCESS cmpq $3, M - jle .L27 + jle L(27) testq $SIZE, A1 - je .L2X + je L(2X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -1011,9 +1011,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L2X: +L(2X): testq $2 * SIZE, A1 - je .L2XX + je L(2XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -1047,7 +1047,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L2XX: +L(2XX): #endif MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) @@ -1059,7 +1059,7 @@ movq MM, I sarq $4, I - jle .L25 + jle L(25) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-32 * SIZE, A1, LDA, 1, %xmm1) @@ -1072,10 +1072,10 @@ MOVUPS_A2 (-28 * SIZE, A2, LDA, 1, %xmm15) decq I - jle .L23 + jle L(23) ALIGN_4 -.L22: +L(22): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -1158,10 +1158,10 @@ addq $16 * SIZE, X1 decq I - jg .L22 + jg L(22) ALIGN_4 -.L23: +L(23): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-24 * SIZE, A1, %xmm0) @@ -1215,9 +1215,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L25: +L(25): testq $8, MM - jle .L26 + jle L(26) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -1251,9 +1251,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L26: +L(26): testq $4, MM - jle .L27 + jle L(27) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -1273,9 +1273,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L27: +L(27): testq $2, MM - jle .L28 + jle L(28) #ifdef movsd xorps %xmm0, %xmm0 @@ -1312,9 +1312,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L28: +L(28): testq $1, MM - jle .L29 + jle L(29) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -1331,7 +1331,7 @@ addss %xmm3, %xmm11 ALIGN_4 -.L29: +L(29): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm11, %xmm10 @@ -1391,13 +1391,13 @@ #if GEMV_UNROLL == 4 cmpq $4, N - jge .L21 + jge L(21) #endif ALIGN_4 -.L30: +L(30): cmpq $3, N - jne .L40 + jne L(40) leaq 32 * SIZE(BUFFER), X1 @@ -1411,10 +1411,10 @@ #ifdef ALIGNED_ACCESS cmpq $3, M - jle .L37 + jle L(37) testq $SIZE, A1 - je .L3X + je L(3X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -1435,9 +1435,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L3X: +L(3X): testq $2 * SIZE, A1 - je .L3XX + je L(3XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -1471,7 +1471,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L3XX: +L(3XX): #endif MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) @@ -1483,7 +1483,7 @@ movq MM, I sarq $4, I - jle .L35 + jle L(35) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-32 * SIZE, A1, LDA, 1, %xmm1) @@ -1494,10 +1494,10 @@ MOVUPS_A1 (-28 * SIZE, A2, %xmm14) decq I - jle .L33 + jle L(33) ALIGN_4 -.L32: +L(32): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 3 - 128 + PREOFFSET(A1) #endif @@ -1563,10 +1563,10 @@ addq $16 * SIZE, X1 decq I - jg .L32 + jg L(32) ALIGN_4 -.L33: +L(33): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-24 * SIZE, A1, %xmm0) @@ -1610,9 +1610,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L35: +L(35): testq $8, MM - jle .L36 + jle L(36) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -1640,9 +1640,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L36: +L(36): testq $4, MM - jle .L37 + jle L(37) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -1659,9 +1659,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L37: +L(37): testq $2, MM - jle .L38 + jle L(38) #ifdef movsd xorps %xmm0, %xmm0 @@ -1694,9 +1694,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L38: +L(38): testq $1, MM - jle .L39 + jle L(39) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -1710,7 +1710,7 @@ addss %xmm2, %xmm10 ALIGN_4 -.L39: +L(39): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm11, %xmm10 @@ -1760,12 +1760,12 @@ addq INCY, Y1 movss %xmm10, (Y1) addq INCY, Y1 - jmp .L999 + jmp L(999) ALIGN_4 -.L40: +L(40): cmpq $2, N - jne .L50 + jne L(50) leaq 32 * SIZE(BUFFER), X1 @@ -1778,10 +1778,10 @@ #ifdef ALIGNED_ACCESS cmpq $3, M - jle .L47 + jle L(47) testq $SIZE, A1 - je .L4X + je L(4X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -1796,9 +1796,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L4X: +L(4X): testq $2 * SIZE, A1 - je .L4XX + je L(4XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -1820,7 +1820,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L4XX: +L(4XX): #endif MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) @@ -1828,7 +1828,7 @@ movq MM, I sarq $4, I - jle .L45 + jle L(45) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-32 * SIZE, A2, %xmm1) @@ -1836,10 +1836,10 @@ MOVUPS_A1 (-28 * SIZE, A2, %xmm13) decq I - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -1889,10 +1889,10 @@ addq $16 * SIZE, X1 decq I - jg .L42 + jg L(42) ALIGN_4 -.L43: +L(43): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-24 * SIZE, A1, %xmm0) @@ -1926,9 +1926,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L45: +L(45): testq $8, MM - jle .L46 + jle L(46) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -1950,9 +1950,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L46: +L(46): testq $4, MM - jle .L47 + jle L(47) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -1966,9 +1966,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L47: +L(47): testq $2, MM - jle .L48 + jle L(48) #ifdef movsd xorps %xmm0, %xmm0 @@ -1993,9 +1993,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L48: +L(48): testq $1, MM - jle .L49 + jle L(49) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -2006,7 +2006,7 @@ addss %xmm1, %xmm9 ALIGN_4 -.L49: +L(49): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm8, %xmm8 @@ -2034,12 +2034,12 @@ addq INCY, Y1 movss %xmm9, (Y1) addq INCY, Y1 - jmp .L999 + jmp L(999) ALIGN_4 -.L50: +L(50): cmpq $1, N - jne .L999 + jne L(999) leaq 32 * SIZE(BUFFER), X1 @@ -2050,10 +2050,10 @@ #ifdef ALIGNED_ACCESS cmpq $3, M - jle .L57 + jle L(57) testq $SIZE, A1 - je .L5X + je L(5X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -2064,9 +2064,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L5X: +L(5X): testq $2 * SIZE, A1 - je .L5XX + je L(5XX) #ifdef movsd @@ -2083,7 +2083,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L5XX: +L(5XX): #endif MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) @@ -2091,16 +2091,16 @@ movq MM, I sarq $4, I - jle .L55 + jle L(55) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-28 * SIZE, A1, %xmm12) decq I - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -2133,10 +2133,10 @@ addq $16 * SIZE, X1 decq I - jg .L52 + jg L(52) ALIGN_4 -.L53: +L(53): mulps %xmm4, %xmm0 MOVUPS_XL1 (-24 * SIZE, X1, %xmm4) addps %xmm0, %xmm8 @@ -2159,9 +2159,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L55: +L(55): testq $8, MM - jle .L56 + jle L(56) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -2176,9 +2176,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L56: +L(56): testq $4, MM - jle .L57 + jle L(57) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -2188,9 +2188,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L57: +L(57): testq $2, MM - jle .L58 + jle L(58) #ifdef movsd xorps %xmm0, %xmm0 @@ -2208,9 +2208,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L58: +L(58): testq $1, MM - jle .L59 + jle L(59) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -2218,7 +2218,7 @@ addss %xmm0, %xmm8 ALIGN_4 -.L59: +L(59): addps %xmm9, %xmm8 #ifdef HAVE_SSE3 @@ -2240,18 +2240,18 @@ movss %xmm8, (Y1) #ifdef ALIGNED_ACCESS - jmp .L999 + jmp L(999) ALIGN_4 -.L100: +L(100): testq $2 * SIZE - 1, LDA - jne .L200 + jne L(200) cmpq $4, N - jl .L110 + jl L(110) ALIGN_3 -.L101: +L(101): subq $4, N leaq 32 * SIZE(BUFFER), X1 @@ -2266,10 +2266,10 @@ xorps %xmm11, %xmm11 cmpq $3, M - jle .L107 + jle L(107) testq $SIZE, A1 - je .L10X + je L(10X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -2290,9 +2290,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L10X: +L(10X): testq $2 * SIZE, A1 - je .L10XX + je L(10XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -2326,7 +2326,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L10XX: +L(10XX): MOVUPS_A2 (-34 * SIZE, A1, LDA, 1, %xmm12) MOVUPS_A2 (-34 * SIZE, A2, LDA, 1, %xmm13) @@ -2339,7 +2339,7 @@ movq MM, I sarq $4, I - jle .L105 + jle L(105) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-30 * SIZE, A1, LDA, 1, %xmm1) @@ -2347,10 +2347,10 @@ MOVUPS_A2 (-30 * SIZE, A2, LDA, 1, %xmm3) decq I - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -2441,10 +2441,10 @@ addq $16 * SIZE, X1 decq I - jg .L102 + jg L(102) ALIGN_4 -.L103: +L(103): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-28 * SIZE, A1, %xmm0) @@ -2510,9 +2510,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L105: +L(105): testq $8, MM - jle .L106 + jle L(106) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-30 * SIZE, A1, LDA, 1, %xmm1) @@ -2551,9 +2551,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L106: +L(106): testq $4, MM - jle .L107 + jle L(107) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -2576,9 +2576,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L107: +L(107): testq $2, MM - jle .L108 + jle L(108) #ifdef movsd xorps %xmm0, %xmm0 @@ -2615,9 +2615,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L108: +L(108): testq $1, MM - jle .L109 + jle L(109) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -2634,7 +2634,7 @@ addss %xmm3, %xmm11 ALIGN_4 -.L109: +L(109): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm11, %xmm10 @@ -2693,12 +2693,12 @@ addq INCY, Y1 cmpq $4, N - jge .L101 + jge L(101) ALIGN_4 -.L110: +L(110): cmpq $3, N - jne .L120 + jne L(120) leaq 32 * SIZE(BUFFER), X1 @@ -2711,10 +2711,10 @@ xorps %xmm10, %xmm10 cmpq $3, M - jle .L117 + jle L(117) testq $SIZE, A1 - je .L11X + je L(11X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -2732,9 +2732,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L11X: +L(11X): testq $2 * SIZE, A1 - je .L11XX + je L(11XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -2762,7 +2762,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L11XX: +L(11XX): MOVUPS_A2 (-34 * SIZE, A1, LDA, 1, %xmm12) MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) @@ -2770,17 +2770,17 @@ movq MM, I sarq $4, I - jle .L115 + jle L(115) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-30 * SIZE, A1, LDA, 1, %xmm1) MOVUPS_A1 (-32 * SIZE, A2, %xmm2) decq I - jle .L113 + jle L(113) ALIGN_4 -.L112: +L(112): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 3 - 128 + PREOFFSET(A1) #endif @@ -2850,10 +2850,10 @@ addq $16 * SIZE, X1 decq I - jg .L112 + jg L(112) ALIGN_4 -.L113: +L(113): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-28 * SIZE, A1, %xmm0) @@ -2904,9 +2904,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L115: +L(115): testq $8, MM - jle .L116 + jle L(116) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-30 * SIZE, A1, LDA, 1, %xmm1) @@ -2937,9 +2937,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L116: +L(116): testq $4, MM - jle .L117 + jle L(117) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -2958,9 +2958,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L117: +L(117): testq $2, MM - jle .L118 + jle L(118) #ifdef movsd xorps %xmm0, %xmm0 @@ -2990,9 +2990,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L118: +L(118): testq $1, MM - jle .L119 + jle L(119) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -3006,7 +3006,7 @@ addss %xmm2, %xmm10 ALIGN_4 -.L119: +L(119): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm11, %xmm10 @@ -3054,12 +3054,12 @@ movss %xmm9, (Y1) addq INCY, Y1 movss %xmm10, (Y1) - jmp .L999 + jmp L(999) ALIGN_4 -.L120: +L(120): cmpq $2, N - jne .L130 + jne L(130) leaq 32 * SIZE(BUFFER), X1 @@ -3071,10 +3071,10 @@ xorps %xmm9, %xmm9 cmpq $3, M - jle .L127 + jle L(127) testq $SIZE, A1 - je .L12X + je L(12X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -3089,9 +3089,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L12X: +L(12X): testq $2 * SIZE, A1 - je .L12XX + je L(12XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -3113,7 +3113,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L12XX: +L(12XX): MOVUPS_A1 (-34 * SIZE, A2, %xmm12) MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) @@ -3121,16 +3121,16 @@ movq MM, I sarq $4, I - jle .L125 + jle L(125) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-30 * SIZE, A2, %xmm1) decq I - jle .L123 + jle L(123) ALIGN_4 -.L122: +L(122): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -3184,10 +3184,10 @@ addq $16 * SIZE, X1 decq I - jg .L122 + jg L(122) ALIGN_4 -.L123: +L(123): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-28 * SIZE, A1, %xmm0) @@ -3227,9 +3227,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L125: +L(125): testq $8, MM - jle .L126 + jle L(126) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-30 * SIZE, A2, %xmm1) @@ -3254,9 +3254,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L126: +L(126): testq $4, MM - jle .L127 + jle L(127) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -3271,9 +3271,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L127: +L(127): testq $2, MM - jle .L128 + jle L(128) #ifdef movsd xorps %xmm0, %xmm0 @@ -3298,9 +3298,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L128: +L(128): testq $1, MM - jle .L129 + jle L(129) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -3311,7 +3311,7 @@ addss %xmm1, %xmm9 ALIGN_4 -.L129: +L(129): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm8, %xmm8 @@ -3339,12 +3339,12 @@ addq INCY, Y1 movss %xmm9, (Y1) addq INCY, Y1 - jmp .L999 + jmp L(999) ALIGN_4 -.L130: +L(130): cmpq $1, N - jne .L999 + jne L(999) leaq 32 * SIZE(BUFFER), X1 @@ -3354,10 +3354,10 @@ xorps %xmm9, %xmm9 cmpq $3, M - jle .L137 + jle L(137) testq $SIZE, A1 - je .L13X + je L(13X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -3368,9 +3368,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L13X: +L(13X): testq $2 * SIZE, A1 - je .L13XX + je L(13XX) #ifdef movsd @@ -3387,23 +3387,23 @@ addq $2 * SIZE, X1 ALIGN_3 -.L13XX: +L(13XX): MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) MOVUPS_XL1 (-28 * SIZE, X1, %xmm5) movq MM, I sarq $4, I - jle .L135 + jle L(135) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-28 * SIZE, A1, %xmm12) decq I - jle .L133 + jle L(133) ALIGN_4 -.L132: +L(132): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -3436,10 +3436,10 @@ addq $16 * SIZE, X1 decq I - jg .L132 + jg L(132) ALIGN_4 -.L133: +L(133): mulps %xmm4, %xmm0 MOVUPS_XL1 (-24 * SIZE, X1, %xmm4) addps %xmm0, %xmm8 @@ -3462,9 +3462,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L135: +L(135): testq $8, MM - jle .L136 + jle L(136) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -3479,9 +3479,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L136: +L(136): testq $4, MM - jle .L137 + jle L(137) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -3491,9 +3491,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L137: +L(137): testq $2, MM - jle .L138 + jle L(138) #ifdef movsd xorps %xmm0, %xmm0 @@ -3511,9 +3511,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L138: +L(138): testq $1, MM - jle .L139 + jle L(139) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -3521,7 +3521,7 @@ addss %xmm0, %xmm8 ALIGN_4 -.L139: +L(139): addps %xmm9, %xmm8 #ifdef HAVE_SSE3 @@ -3541,18 +3541,18 @@ addss (Y), %xmm8 movss %xmm8, (Y1) - jmp .L999 + jmp L(999) ALIGN_4 -.L200: +L(200): testq $2 * SIZE, LDA - jne .L300 + jne L(300) cmpq $4, N - jl .L210 + jl L(210) ALIGN_3 -.L201: +L(201): subq $4, N leaq 32 * SIZE(BUFFER), X1 @@ -3567,10 +3567,10 @@ xorps %xmm11, %xmm11 cmpq $3, M - jle .L207 + jle L(207) testq $SIZE, A1 - je .L20X + je L(20X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -3591,9 +3591,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L20X: +L(20X): testq $2 * SIZE, A1 - je .L20XX + je L(20XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -3627,7 +3627,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L20XX: +L(20XX): movaps -33 * SIZE(A1, LDA), %xmm12 movaps -34 * SIZE(A2), %xmm13 movaps -35 * SIZE(A2, LDA), %xmm14 @@ -3641,7 +3641,7 @@ movq MM, I sarq $4, I - jle .L205 + jle L(205) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-29 * SIZE, A1, LDA, 1, %xmm1) @@ -3649,10 +3649,10 @@ MOVUPS_A2 (-31 * SIZE, A2, LDA, 1, %xmm3) decq I - jle .L203 + jle L(203) ALIGN_4 -.L202: +L(202): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -3758,10 +3758,10 @@ addq $16 * SIZE, X1 decq I - jg .L202 + jg L(202) ALIGN_4 -.L203: +L(203): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-28 * SIZE, A1, %xmm0) @@ -3843,9 +3843,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L205: +L(205): testq $8, MM - jle .L206 + jle L(206) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-29 * SIZE, A1, LDA, 1, %xmm1) @@ -3892,9 +3892,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L206: +L(206): testq $4, MM - jle .L207 + jle L(207) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-29 * SIZE, A1, LDA, 1, %xmm1) @@ -3921,9 +3921,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L207: +L(207): testq $2, MM - jle .L208 + jle L(208) #ifdef movsd xorps %xmm0, %xmm0 @@ -3960,9 +3960,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L208: +L(208): testq $1, MM - jle .L209 + jle L(209) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -3979,7 +3979,7 @@ addss %xmm3, %xmm11 ALIGN_4 -.L209: +L(209): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm11, %xmm10 @@ -4038,12 +4038,12 @@ addq INCY, Y1 cmpq $4, N - jge .L201 + jge L(201) ALIGN_4 -.L210: +L(210): cmpq $3, N - jne .L220 + jne L(220) leaq 32 * SIZE(BUFFER), X1 @@ -4056,10 +4056,10 @@ xorps %xmm10, %xmm10 cmpq $3, M - jle .L217 + jle L(217) testq $SIZE, A1 - je .L21X + je L(21X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -4077,9 +4077,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L21X: +L(21X): testq $2 * SIZE, A1 - je .L21XX + je L(21XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -4107,7 +4107,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L21XX: +L(21XX): movaps -33 * SIZE(A1, LDA), %xmm12 movaps -34 * SIZE(A2), %xmm13 @@ -4120,17 +4120,17 @@ movq MM, I sarq $4, I - jle .L215 + jle L(215) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-29 * SIZE, A1, LDA, 1, %xmm1) MOVUPS_A1 (-30 * SIZE, A2, %xmm2) decq I - jle .L213 + jle L(213) ALIGN_4 -.L212: +L(212): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 3 - 128 + PREOFFSET(A1) #endif @@ -4210,10 +4210,10 @@ addq $16 * SIZE, X1 decq I - jg .L212 + jg L(212) ALIGN_4 -.L213: +L(213): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-28 * SIZE, A1, %xmm0) @@ -4295,9 +4295,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L215: +L(215): testq $8, MM - jle .L216 + jle L(216) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-29 * SIZE, A1, LDA, 1, %xmm1) @@ -4334,9 +4334,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L216: +L(216): testq $4, MM - jle .L217 + jle L(217) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-29 * SIZE, A1, LDA, 1, %xmm1) @@ -4358,9 +4358,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L217: +L(217): testq $2, MM - jle .L218 + jle L(218) #ifdef movsd xorps %xmm0, %xmm0 @@ -4390,9 +4390,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L218: +L(218): testq $1, MM - jle .L219 + jle L(219) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -4406,7 +4406,7 @@ addss %xmm2, %xmm10 ALIGN_4 -.L219: +L(219): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm11, %xmm10 @@ -4454,15 +4454,15 @@ movss %xmm9, (Y1) addq INCY, Y1 movss %xmm10, (Y1) - jmp .L999 + jmp L(999) ALIGN_4 -.L220: +L(220): testq N, N - jle .L999 + jle L(999) cmpq $2, N - jne .L230 + jne L(230) leaq 32 * SIZE(BUFFER), X1 @@ -4474,10 +4474,10 @@ xorps %xmm9, %xmm9 cmpq $3, M - jle .L227 + jle L(227) testq $SIZE, A1 - je .L22X + je L(22X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -4492,9 +4492,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L22X: +L(22X): testq $2 * SIZE, A1 - je .L22XX + je L(22XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -4516,7 +4516,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L22XX: +L(22XX): movaps -33 * SIZE(A2), %xmm12 MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) @@ -4524,16 +4524,16 @@ movq MM, I sarq $4, I - jle .L225 + jle L(225) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-29 * SIZE, A2, %xmm1) decq I - jle .L223 + jle L(223) ALIGN_4 -.L222: +L(222): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -4591,10 +4591,10 @@ addq $16 * SIZE, X1 decq I - jg .L222 + jg L(222) ALIGN_4 -.L223: +L(223): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-28 * SIZE, A1, %xmm2) @@ -4638,9 +4638,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L225: +L(225): testq $8, MM - jle .L226 + jle L(226) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-29 * SIZE, A2, %xmm1) @@ -4667,9 +4667,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L226: +L(226): testq $4, MM - jle .L227 + jle L(227) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-29 * SIZE, A2, %xmm1) @@ -4687,9 +4687,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L227: +L(227): testq $2, MM - jle .L228 + jle L(228) #ifdef movsd xorps %xmm0, %xmm0 @@ -4714,9 +4714,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L228: +L(228): testq $1, MM - jle .L229 + jle L(229) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -4727,7 +4727,7 @@ addss %xmm1, %xmm9 ALIGN_4 -.L229: +L(229): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm8, %xmm8 @@ -4755,12 +4755,12 @@ addq INCY, Y1 movss %xmm9, (Y1) addq INCY, Y1 - jmp .L999 + jmp L(999) ALIGN_4 -.L230: +L(230): cmpq $1, N - jne .L999 + jne L(999) leaq 32 * SIZE(BUFFER), X1 @@ -4770,10 +4770,10 @@ xorps %xmm9, %xmm9 cmpq $3, M - jle .L237 + jle L(237) testq $SIZE, A1 - je .L23X + je L(23X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -4784,9 +4784,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L23X: +L(23X): testq $2 * SIZE, A1 - je .L23XX + je L(23XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -4802,23 +4802,23 @@ addq $2 * SIZE, X1 ALIGN_3 -.L23XX: +L(23XX): MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) MOVUPS_XL1 (-28 * SIZE, X1, %xmm5) movq MM, I sarq $4, I - jle .L235 + jle L(235) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-28 * SIZE, A1, %xmm12) decq I - jle .L233 + jle L(233) ALIGN_4 -.L232: +L(232): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -4851,10 +4851,10 @@ addq $16 * SIZE, X1 decq I - jg .L232 + jg L(232) ALIGN_4 -.L233: +L(233): mulps %xmm4, %xmm0 MOVUPS_XL1 (-24 * SIZE, X1, %xmm4) addps %xmm0, %xmm8 @@ -4877,9 +4877,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L235: +L(235): testq $8, MM - jle .L236 + jle L(236) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -4894,9 +4894,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L236: +L(236): testq $4, MM - jle .L237 + jle L(237) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -4906,9 +4906,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L237: +L(237): testq $2, MM - jle .L238 + jle L(238) #ifdef movsd xorps %xmm0, %xmm0 @@ -4926,9 +4926,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L238: +L(238): testq $1, MM - jle .L239 + jle L(239) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -4936,7 +4936,7 @@ addss %xmm0, %xmm8 ALIGN_4 -.L239: +L(239): addps %xmm9, %xmm8 #ifdef HAVE_SSE3 @@ -4956,15 +4956,15 @@ addss (Y), %xmm8 movss %xmm8, (Y1) - jmp .L999 + jmp L(999) ALIGN_4 -.L300: +L(300): cmpq $4, N - jl .L310 + jl L(310) ALIGN_3 -.L301: +L(301): subq $4, N leaq 32 * SIZE(BUFFER), X1 @@ -4979,10 +4979,10 @@ xorps %xmm11, %xmm11 cmpq $3, M - jle .L307 + jle L(307) testq $SIZE, A1 - je .L30X + je L(30X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -5003,9 +5003,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L30X: +L(30X): testq $2 * SIZE, A1 - je .L30XX + je L(30XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -5039,7 +5039,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L30XX: +L(30XX): movaps -35 * SIZE(A1, LDA), %xmm12 movaps -34 * SIZE(A2), %xmm13 movaps -33 * SIZE(A2, LDA), %xmm14 @@ -5053,7 +5053,7 @@ movq MM, I sarq $4, I - jle .L305 + jle L(305) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-31 * SIZE, A1, LDA, 1, %xmm1) @@ -5061,10 +5061,10 @@ MOVUPS_A2 (-29 * SIZE, A2, LDA, 1, %xmm3) decq I - jle .L303 + jle L(303) ALIGN_4 -.L302: +L(302): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -5170,10 +5170,10 @@ addq $16 * SIZE, X1 decq I - jg .L302 + jg L(302) ALIGN_4 -.L303: +L(303): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-28 * SIZE, A1, %xmm0) @@ -5255,9 +5255,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L305: +L(305): testq $8, MM - jle .L306 + jle L(306) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-31 * SIZE, A1, LDA, 1, %xmm1) @@ -5304,9 +5304,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L306: +L(306): testq $4, MM - jle .L307 + jle L(307) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-31 * SIZE, A1, LDA, 1, %xmm1) @@ -5333,9 +5333,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L307: +L(307): testq $2, MM - jle .L308 + jle L(308) #ifdef movsd xorps %xmm0, %xmm0 @@ -5372,9 +5372,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L308: +L(308): testq $1, MM - jle .L309 + jle L(309) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -5391,7 +5391,7 @@ addss %xmm3, %xmm11 ALIGN_4 -.L309: +L(309): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm11, %xmm10 @@ -5450,15 +5450,15 @@ addq INCY, Y1 cmpq $4, N - jge .L301 + jge L(301) ALIGN_4 -.L310: +L(310): testq N, N - jle .L999 + jle L(999) cmpq $3, N - jne .L320 + jne L(320) leaq 32 * SIZE(BUFFER), X1 @@ -5471,10 +5471,10 @@ xorps %xmm10, %xmm10 cmpq $3, M - jle .L317 + jle L(317) testq $SIZE, A1 - je .L31X + je L(31X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -5492,9 +5492,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L31X: +L(31X): testq $2 * SIZE, A1 - je .L31XX + je L(31XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -5522,7 +5522,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L31XX: +L(31XX): movaps -35 * SIZE(A1, LDA), %xmm12 movaps -34 * SIZE(A2), %xmm13 @@ -5531,17 +5531,17 @@ movq MM, I sarq $4, I - jle .L315 + jle L(315) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-31 * SIZE, A1, LDA, 1, %xmm1) MOVUPS_A1 (-30 * SIZE, A2, %xmm2) decq I - jle .L313 + jle L(313) ALIGN_4 -.L312: +L(312): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 3 - 128 + PREOFFSET(A1) #endif @@ -5620,10 +5620,10 @@ addq $16 * SIZE, X1 decq I - jg .L312 + jg L(312) ALIGN_4 -.L313: +L(313): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-28 * SIZE, A1, %xmm0) @@ -5705,9 +5705,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L315: +L(315): testq $8, MM - jle .L316 + jle L(316) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-31 * SIZE, A1, LDA, 1, %xmm1) @@ -5744,9 +5744,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L316: +L(316): testq $4, MM - jle .L317 + jle L(317) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A2 (-31 * SIZE, A1, LDA, 1, %xmm1) @@ -5768,9 +5768,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L317: +L(317): testq $2, MM - jle .L318 + jle L(318) #ifdef movsd xorps %xmm0, %xmm0 @@ -5800,9 +5800,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L318: +L(318): testq $1, MM - jle .L319 + jle L(319) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -5816,7 +5816,7 @@ addss %xmm2, %xmm10 ALIGN_4 -.L319: +L(319): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm11, %xmm10 @@ -5864,12 +5864,12 @@ movss %xmm9, (Y1) addq INCY, Y1 movss %xmm10, (Y1) - jmp .L999 + jmp L(999) ALIGN_3 -.L320: +L(320): cmpq $2, N - jne .L330 + jne L(330) leaq 32 * SIZE(BUFFER), X1 @@ -5880,10 +5880,10 @@ xorps %xmm9, %xmm9 cmpq $3, M - jle .L327 + jle L(327) testq $SIZE, A1 - je .L32X + je L(32X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -5898,9 +5898,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L32X: +L(32X): testq $2 * SIZE, A1 - je .L32XX + je L(32XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -5922,7 +5922,7 @@ addq $2 * SIZE, X1 ALIGN_3 -.L32XX: +L(32XX): movaps -35 * SIZE(A2), %xmm12 MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) @@ -5930,16 +5930,16 @@ movq MM, I sarq $4, I - jle .L325 + jle L(325) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-31 * SIZE, A2, %xmm1) decq I - jle .L323 + jle L(323) ALIGN_4 -.L322: +L(322): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -5997,10 +5997,10 @@ addq $16 * SIZE, X1 decq I - jg .L322 + jg L(322) ALIGN_4 -.L323: +L(323): mulps %xmm4, %xmm0 addps %xmm0, %xmm8 MOVUPS_A1 (-28 * SIZE, A1, %xmm0) @@ -6044,9 +6044,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L325: +L(325): testq $8, MM - jle .L326 + jle L(326) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-31 * SIZE, A2, %xmm1) @@ -6073,9 +6073,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L326: +L(326): testq $4, MM - jle .L327 + jle L(327) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-31 * SIZE, A2, %xmm1) @@ -6092,9 +6092,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L327: +L(327): testq $2, MM - jle .L328 + jle L(328) #ifdef movsd xorps %xmm0, %xmm0 @@ -6119,9 +6119,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L328: +L(328): testq $1, MM - jle .L329 + jle L(329) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -6132,7 +6132,7 @@ addss %xmm1, %xmm9 ALIGN_4 -.L329: +L(329): #ifdef HAVE_SSE3 haddps %xmm9, %xmm8 haddps %xmm8, %xmm8 @@ -6160,12 +6160,12 @@ addq INCY, Y1 movss %xmm9, (Y1) addq INCY, Y1 - jmp .L999 + jmp L(999) ALIGN_4 -.L330: +L(330): cmpq $1, N - jne .L999 + jne L(999) leaq 32 * SIZE(BUFFER), X1 @@ -6175,10 +6175,10 @@ xorps %xmm9, %xmm9 cmpq $3, M - jle .L337 + jle L(337) testq $SIZE, A1 - je .L33X + je L(33X) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -6189,9 +6189,9 @@ addq $1 * SIZE, X1 ALIGN_3 -.L33X: +L(33X): testq $2 * SIZE, A1 - je .L33XX + je L(33XX) #ifdef movsd xorps %xmm0, %xmm0 @@ -6207,23 +6207,23 @@ addq $2 * SIZE, X1 ALIGN_3 -.L33XX: +L(33XX): MOVUPS_XL1 (-32 * SIZE, X1, %xmm4) MOVUPS_XL1 (-28 * SIZE, X1, %xmm5) movq MM, I sarq $4, I - jle .L335 + jle L(335) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) MOVUPS_A1 (-28 * SIZE, A1, %xmm12) decq I - jle .L333 + jle L(333) ALIGN_4 -.L332: +L(332): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 8 - 128 + PREOFFSET(A1) #endif @@ -6256,10 +6256,10 @@ addq $16 * SIZE, X1 decq I - jg .L332 + jg L(332) ALIGN_4 -.L333: +L(333): mulps %xmm4, %xmm0 MOVUPS_XL1 (-24 * SIZE, X1, %xmm4) addps %xmm0, %xmm8 @@ -6282,9 +6282,9 @@ addq $16 * SIZE, X1 ALIGN_4 -.L335: +L(335): testq $8, MM - jle .L336 + jle L(336) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -6299,9 +6299,9 @@ addq $8 * SIZE, X1 ALIGN_4 -.L336: +L(336): testq $4, MM - jle .L337 + jle L(337) MOVUPS_A1 (-32 * SIZE, A1, %xmm0) mulps %xmm4, %xmm0 @@ -6311,9 +6311,9 @@ addq $4 * SIZE, X1 ALIGN_4 -.L337: +L(337): testq $2, MM - jle .L338 + jle L(338) #ifdef movsd xorps %xmm0, %xmm0 @@ -6331,9 +6331,9 @@ addq $2 * SIZE, X1 ALIGN_4 -.L338: +L(338): testq $1, MM - jle .L339 + jle L(339) movss -32 * SIZE(A1), %xmm0 movss -32 * SIZE(X1), %xmm4 @@ -6341,7 +6341,7 @@ addss %xmm0, %xmm8 ALIGN_4 -.L339: +L(339): addps %xmm9, %xmm8 #ifdef HAVE_SSE3 @@ -6362,17 +6362,17 @@ addss (Y), %xmm8 movss %xmm8, (Y1) - jmp .L999 + jmp L(999) #endif ALIGN_4 -.L999: +L(999): leaq (,M,SIZE),%rax addq %rax,AA - jmp .L0t + jmp L(0t) ALIGN_4 -.L999x: +L(999x): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/sum.S b/kernel/x86_64/sum.S index 9f2cdc1ecd..792a38cf2e 100644 --- a/kernel/x86_64/sum.S +++ b/kernel/x86_64/sum.S @@ -56,9 +56,9 @@ fldz testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) salq $BASE_SHIFT, INCX @@ -66,14 +66,14 @@ fldz fldz cmpq $SIZE, INCX - jne .L40 + jne L(40) movq M, I sarq $3, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -101,30 +101,30 @@ faddp %st, %st(1) decq I - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): andq $7, M - jle .L998 + jle L(998) ALIGN_4 -.L21: +L(21): FLD (X) faddp %st,%st(1) addq $1 * SIZE, X decq M - jg .L21 - jmp .L998 + jg L(21) + jmp L(998) ALIGN_4 -.L40: +L(40): movq M, I sarq $3, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): FLD (X) addq INCX, X FLD (X) @@ -154,30 +154,30 @@ faddp %st, %st(1) decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): andq $7, M - jle .L998 + jle L(998) ALIGN_4 -.L61: +L(61): FLD (X) addq INCX, X faddp %st,%st(1) decq M - jg .L61 + jg L(61) ALIGN_4 -.L998: +L(998): faddp %st,%st(2) faddp %st,%st(1) faddp %st,%st(1) ALIGN_4 -.L999: +L(999): ret EPILOGUE diff --git a/kernel/x86_64/swap.S b/kernel/x86_64/swap.S index 9529724abe..a81d852c37 100644 --- a/kernel/x86_64/swap.S +++ b/kernel/x86_64/swap.S @@ -82,16 +82,16 @@ salq $BASE_SHIFT, INCY cmpq $SIZE, INCX - jne .L14 + jne L(14) cmpq $SIZE, INCY - jne .L14 + jne L(14) movq N, %rax sarq $3, %rax - jle .L15 + jle L(15) ALIGN_3 -.L16: +L(16): #ifdef XDOUBLE movq 0(X), %mm0 movq 8(X), %mm1 @@ -250,16 +250,16 @@ addq $8 * SIZE, X addq $8 * SIZE, Y decq %rax - jg .L16 + jg L(16) ALIGN_3 -.L15: +L(15): movq N, %rax andq $7, %rax - jle .L27 + jle L(27) ALIGN_3 -.L22: +L(22): #ifdef XDOUBLE movq 0(X), %mm0 @@ -281,21 +281,21 @@ addq $SIZE, X addq $SIZE, Y decq %rax - jg .L22 - jmp .L27 + jg L(22) + jmp L(27) ALIGN_3 /* INCX != 1 or INCY != 1 */ -.L14: +L(14): movq N, %rax movq X, XX movq Y, YY sarq $2, %rax - jle .L28 + jle L(28) ALIGN_2 -.L29: +L(29): #ifdef XDOUBLE movq 0(X), %mm0 movq 8(X), %mm1 @@ -391,16 +391,16 @@ #endif decq %rax - jg .L29 + jg L(29) ALIGN_3 -.L28: +L(28): movq N, %rax andq $3, %rax - jle .L27 + jle L(27) ALIGN_3 -.L35: +L(35): #ifdef XDOUBLE movq 0(X), %mm0 movq 8(X), %mm1 @@ -422,10 +422,10 @@ addq INCY, Y decq %rax - jg .L35 + jg L(35) ALIGN_3 -.L27: +L(27): EMMS xorq %rax,%rax diff --git a/kernel/x86_64/swap_sse.S b/kernel/x86_64/swap_sse.S index dc964dab22..1559772dea 100644 --- a/kernel/x86_64/swap_sse.S +++ b/kernel/x86_64/swap_sse.S @@ -75,18 +75,18 @@ leaq (, INCY, SIZE), INCY cmpq $SIZE, INCX - jne .L50 + jne L(50) cmpq $SIZE, INCY - jne .L50 + jne L(50) subq $-32 * SIZE, X subq $-32 * SIZE, Y cmpq $3, M - jle .L16 + jle L(16) testq $SIZE, Y - je .L05 + je L(05) movss -32 * SIZE(X), %xmm0 movss -32 * SIZE(Y), %xmm1 @@ -99,9 +99,9 @@ decq M ALIGN_3 -.L05: +L(05): testq $2 * SIZE, Y - je .L10 + je L(10) movsd -32 * SIZE(X), %xmm0 movsd -32 * SIZE(Y), %xmm1 @@ -112,25 +112,25 @@ addq $2 * SIZE, X addq $2 * SIZE, Y subq $2, M - jle .L19 + jle L(19) ALIGN_3 -.L10: +L(10): cmpq $3, M - jle .L16 + jle L(16) testq $2 * SIZE, X - jne .L30 + jne L(30) testq $1 * SIZE, X - jne .L20 + jne L(20) movq M, %rax sarq $5, %rax - jle .L13 + jle L(13) ALIGN_3 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -199,12 +199,12 @@ subq $-32 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L13: +L(13): testq $16, M - jle .L14 + jle L(14) movaps -32 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -234,9 +234,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $8, M - jle .L15 + jle L(15) movaps -32 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -254,9 +254,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $4, M - jle .L16 + jle L(16) movaps -32 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -268,9 +268,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $2, M - jle .L17 + jle L(17) movsd -32 * SIZE(X), %xmm0 movsd -32 * SIZE(Y), %xmm1 @@ -281,9 +281,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $1, M - jle .L19 + jle L(19) movss -32 * SIZE(X), %xmm0 movss -32 * SIZE(Y), %xmm1 @@ -292,7 +292,7 @@ movss %xmm0, -32 * SIZE(Y) ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -304,7 +304,7 @@ ret ALIGN_3 -.L20: +L(20): movaps -33 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -316,10 +316,10 @@ movq M, %rax sarq $5, %rax - jle .L23 + jle L(23) ALIGN_4 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -420,12 +420,12 @@ subq $-32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L23: +L(23): testq $16, M - jle .L24 + jle L(24) movaps -29 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -471,9 +471,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $8, M - jle .L25 + jle L(25) movaps -29 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -499,9 +499,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $4, M - jle .L26 + jle L(26) movaps -29 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -520,7 +520,7 @@ addq $4 * SIZE, Y ALIGN_3 -.L26: +L(26): pshufd $0x39, %xmm0, %xmm2 pshufd $0xff, %xmm0, %xmm0 @@ -528,7 +528,7 @@ movss %xmm0, -30 * SIZE(Y) testq $2, M - jle .L27 + jle L(27) movsd -29 * SIZE(X), %xmm0 movsd -29 * SIZE(Y), %xmm1 @@ -540,9 +540,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $1, M - jle .L29 + jle L(29) movss -29 * SIZE(X), %xmm0 movss -29 * SIZE(Y), %xmm1 @@ -551,7 +551,7 @@ movss %xmm1, -29 * SIZE(X) ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -563,9 +563,9 @@ ret ALIGN_3 -.L30: +L(30): testq $1 * SIZE, X - jne .L40 + jne L(40) movhps -32 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -575,10 +575,10 @@ movq M, %rax sarq $5, %rax - jle .L33 + jle L(33) ALIGN_4 -.L31: +L(31): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -663,12 +663,12 @@ subq $-32 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L33: +L(33): testq $16, M - jle .L34 + jle L(34) movaps -30 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -706,9 +706,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L34: +L(34): testq $8, M - jle .L35 + jle L(35) movaps -30 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -730,9 +730,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L35: +L(35): testq $4, M - jle .L36 + jle L(36) movaps -30 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -749,11 +749,11 @@ addq $4 * SIZE, Y ALIGN_3 -.L36: +L(36): movhps %xmm0, -32 * SIZE(Y) testq $2, M - jle .L37 + jle L(37) movsd -30 * SIZE(X), %xmm0 movsd -30 * SIZE(Y), %xmm1 @@ -765,9 +765,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L37: +L(37): testq $1, M - jle .L39 + jle L(39) movss -30 * SIZE(X), %xmm0 movss -30 * SIZE(Y), %xmm1 @@ -776,7 +776,7 @@ movss %xmm1, -30 * SIZE(X) ALIGN_3 -.L39: +L(39): xorq %rax,%rax RESTOREREGISTERS @@ -788,7 +788,7 @@ ret ALIGN_3 -.L40: +L(40): movaps -35 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -798,10 +798,10 @@ movq M, %rax sarq $5, %rax - jle .L43 + jle L(43) ALIGN_4 -.L41: +L(41): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -902,12 +902,12 @@ subq $-32 * SIZE, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L43: +L(43): testq $16, M - jle .L44 + jle L(44) movaps -31 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -953,9 +953,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L44: +L(44): testq $8, M - jle .L45 + jle L(45) movaps -31 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -981,9 +981,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L45: +L(45): testq $4, M - jle .L46 + jle L(46) movaps -31 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -1002,7 +1002,7 @@ addq $4 * SIZE, Y ALIGN_3 -.L46: +L(46): movsd -31 * SIZE(X), %xmm2 pshufd $0x39, %xmm1, %xmm1 @@ -1017,7 +1017,7 @@ addq $3 * SIZE, Y testq $2, M - jle .L47 + jle L(47) movsd -32 * SIZE(X), %xmm0 movsd -32 * SIZE(Y), %xmm1 @@ -1029,9 +1029,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L47: +L(47): testq $1, M - jle .L49 + jle L(49) movss -32 * SIZE(X), %xmm0 movss -32 * SIZE(Y), %xmm1 @@ -1040,7 +1040,7 @@ movss %xmm1, -32 * SIZE(X) ALIGN_3 -.L49: +L(49): xorq %rax,%rax RESTOREREGISTERS @@ -1052,13 +1052,13 @@ ret ALIGN_3 -.L50: +L(50): movq M, %rax sarq $3, %rax - jle .L55 + jle L(55) ALIGN_3 -.L51: +L(51): movss (X), %xmm0 movss (Y), %xmm1 @@ -1124,16 +1124,16 @@ addq INCY, Y decq %rax - jg .L51 + jg L(51) ALIGN_3 -.L55: +L(55): movq M, %rax andq $7, %rax - jle .L57 + jle L(57) ALIGN_3 -.L56: +L(56): movss (X), %xmm0 movss (Y), %xmm1 @@ -1143,10 +1143,10 @@ addq INCX, X addq INCY, Y decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L57: +L(57): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/swap_sse2.S b/kernel/x86_64/swap_sse2.S index e9260b979f..9fa9590103 100644 --- a/kernel/x86_64/swap_sse2.S +++ b/kernel/x86_64/swap_sse2.S @@ -75,12 +75,12 @@ leaq (, INCY, SIZE), INCY cmpq $SIZE, INCX - jne .L40 + jne L(40) cmpq $SIZE, INCY - jne .L40 + jne L(40) testq $SIZE, Y - je .L10 + je L(10) movsd 0 * SIZE(X), %xmm0 movsd 0 * SIZE(Y), %xmm8 @@ -91,22 +91,22 @@ addq $1 * SIZE, X addq $1 * SIZE, Y decq M - jle .L19 + jle L(19) ALIGN_4 -.L10: +L(10): subq $-16 * SIZE, X subq $-16 * SIZE, Y testq $SIZE, X - jne .L20 + jne L(20) movq M, %rax sarq $4, %rax - jle .L13 + jle L(13) ALIGN_3 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -175,12 +175,12 @@ subq $-16 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L13: +L(13): testq $8, M - jle .L14 + jle L(14) movaps -16 * SIZE(X), %xmm0 movaps -16 * SIZE(Y), %xmm1 @@ -210,9 +210,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $4, M - jle .L15 + jle L(15) movaps -16 * SIZE(X), %xmm0 movaps -16 * SIZE(Y), %xmm1 @@ -230,9 +230,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $2, M - jle .L16 + jle L(16) movaps -16 * SIZE(X), %xmm0 movaps -16 * SIZE(Y), %xmm1 @@ -244,9 +244,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $1, M - jle .L19 + jle L(19) movsd -16 * SIZE(X), %xmm0 movsd -16 * SIZE(Y), %xmm1 @@ -255,7 +255,7 @@ movlps %xmm0, -16 * SIZE(Y) ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -267,20 +267,20 @@ ret ALIGN_3 -.L20: +L(20): movhps -16 * SIZE(X), %xmm0 movaps -16 * SIZE(Y), %xmm1 movlps %xmm1, -16 * SIZE(X) decq M - jle .L29 + jle L(29) movq M, %rax sarq $4, %rax - jle .L23 + jle L(23) ALIGN_4 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -365,12 +365,12 @@ subq $-16 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L23: +L(23): testq $8, M - jle .L24 + jle L(24) movaps -15 * SIZE(X), %xmm2 movaps -14 * SIZE(Y), %xmm3 @@ -408,9 +408,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $4, M - jle .L25 + jle L(25) movaps -15 * SIZE(X), %xmm2 movaps -14 * SIZE(Y), %xmm3 @@ -432,9 +432,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $2, M - jle .L26 + jle L(26) movaps -15 * SIZE(X), %xmm2 movaps -14 * SIZE(Y), %xmm3 @@ -451,9 +451,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $1, M - jle .L29 + jle L(29) movhps %xmm0, -16 * SIZE(Y) movhps -15 * SIZE(X), %xmm0 @@ -463,7 +463,7 @@ addq $SIZE, Y ALIGN_3 -.L29: +L(29): movhps %xmm0, -16 * SIZE(Y) xorq %rax,%rax @@ -477,13 +477,13 @@ ret ALIGN_3 -.L40: +L(40): movq M, %rax sarq $3, %rax - jle .L45 + jle L(45) ALIGN_3 -.L41: +L(41): movsd (X), %xmm0 movsd (Y), %xmm1 @@ -549,16 +549,16 @@ addq INCY, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L45: +L(45): movq M, %rax andq $7, %rax - jle .L47 + jle L(47) ALIGN_3 -.L46: +L(46): movsd (X), %xmm0 movsd (Y), %xmm1 @@ -568,10 +568,10 @@ addq INCX, X addq INCY, Y decq %rax - jg .L46 + jg L(46) ALIGN_3 -.L47: +L(47): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/symv_L_sse.S b/kernel/x86_64/symv_L_sse.S index 55780734f2..7a76e73030 100644 --- a/kernel/x86_64/symv_L_sse.S +++ b/kernel/x86_64/symv_L_sse.S @@ -206,7 +206,7 @@ leaq (,LDA, SIZE), LDA testq M, M - jle .L999 + jle L(999) shufps $0, ALPHA, ALPHA @@ -214,10 +214,10 @@ movq M, %rax sarq $3, %rax - jle .L02 + jle L(02) ALIGN_3 -.L01: +L(01): movss 0 * SIZE(X), %xmm1 addq INCX, X movss 0 * SIZE(X), %xmm2 @@ -255,16 +255,16 @@ addq $8 * SIZE, XX decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L02: +L(02): movq M, %rax andq $7, %rax - jle .L05 + jle L(05) ALIGN_3 -.L03: +L(03): movss 0 * SIZE(X), %xmm1 addq INCX, X @@ -274,10 +274,10 @@ addq $1 * SIZE, XX decq %rax - jg .L03 + jg L(03) ALIGN_3 -.L05: +L(05): /* now we don't need original X */ movq Y, NEW_Y @@ -285,17 +285,17 @@ andq $-512, XX cmpq $SIZE, INCY - je .L10 + je L(10) movq Y, YY movq XX, NEW_Y movq M, %rax sarq $3, %rax - jle .L07 + jle L(07) ALIGN_3 -.L06: +L(06): movss 0 * SIZE(YY), %xmm0 addq INCY, YY movss 0 * SIZE(YY), %xmm1 @@ -324,16 +324,16 @@ addq $8 * SIZE, XX decq %rax - jg .L06 + jg L(06) ALIGN_3 -.L07: +L(07): movq M, %rax andq $7, %rax - jle .L10 + jle L(10) ALIGN_3 -.L08: +L(08): movss 0 * SIZE(YY), %xmm0 addq INCY, YY @@ -341,17 +341,17 @@ addq $1 * SIZE, XX decq %rax - jg .L08 + jg L(08) ALIGN_3 -.L10: +L(10): xorq IS, IS # is = 0 cmpq $4, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): movq A, A1 leaq (A, LDA, 2), A2 leaq 4 * SIZE(A, LDA, 4), A @@ -420,10 +420,10 @@ subq IS, I subq $4, I sarq $4, I - jle .L14 + jle L(14) ALIGN_3 -.L12: +L(12): movaps xtemp1, xt1 mulps a1, xt1 mulps atemp1, a1 @@ -599,15 +599,15 @@ addq $16 * SIZE, A2 decq I - jg .L12 + jg L(12) ALIGN_3 -.L14: +L(14): movq M, I subq IS, I subq $4, I test $8, I - jle .L15 + jle L(15) movaps xtemp1, xt1 mulps a1, xt1 @@ -691,9 +691,9 @@ addq $8 * SIZE, A2 ALIGN_3 -.L15: +L(15): test $4, I - jle .L17 + jle L(17) movaps xtemp1, xt1 mulps a1, xt1 @@ -734,9 +734,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L17: +L(17): testq $2, M - jle .L18 + jle L(18) pxor xtemp2, xtemp2 @@ -782,9 +782,9 @@ addq $2 * SIZE, A2 ALIGN_3 -.L18: +L(18): testq $1, M - jle .L19 + jle L(19) movss 0 * SIZE(XX), xtemp1 @@ -822,7 +822,7 @@ movss yy1, 0 * SIZE(YY) ALIGN_3 -.L19: +L(19): #ifndef HAVE_SSE3 movaps xsum1, xtemp1 unpcklps xsum3, xsum1 @@ -863,12 +863,12 @@ movq IS, I addq $4, I cmpq N, I - jle .L11 + jle L(11) ALIGN_3 -.L20: +L(20): testq $2, N - jle .L30 + jle L(30) movq A, A1 leaq 2 * SIZE(A, LDA, 2), A @@ -890,7 +890,7 @@ pshufd $0x55, atemp4, atemp2 testq $1, M - jle .L29 + jle L(29) movss 2 * SIZE(A1), a1 movss 2 * SIZE(A1, LDA, 1), a2 @@ -912,7 +912,7 @@ movss yy1, 2 * SIZE(NEW_Y, IS, SIZE) ALIGN_3 -.L29: +L(29): #ifndef HAVE_SSE3 unpcklps xsum2, xsum1 @@ -932,9 +932,9 @@ addq $2, IS ALIGN_3 -.L30: +L(30): testq $1, N - jle .L990 + jle L(990) movss 0 * SIZE(NEW_X, IS, SIZE), xsum1 mulss 0 * SIZE(A), xsum1 @@ -942,16 +942,16 @@ movss xsum1, 0 * SIZE(NEW_Y, IS, SIZE) ALIGN_3 -.L990: +L(990): cmpq $SIZE, INCY - je .L999 + je L(999) movq M, %rax sarq $3, %rax - jle .L997 + jle L(997) ALIGN_3 -.L996: +L(996): movss 0 * SIZE(NEW_Y), %xmm0 movss 1 * SIZE(NEW_Y), %xmm1 movss 2 * SIZE(NEW_Y), %xmm2 @@ -980,16 +980,16 @@ addq $8 * SIZE, NEW_Y decq %rax - jg .L996 + jg L(996) ALIGN_3 -.L997: +L(997): movq M, %rax andq $7, %rax - jle .L999 + jle L(999) ALIGN_3 -.L998: +L(998): movss 0 * SIZE(NEW_Y), %xmm0 movss %xmm0, 0 * SIZE(Y) @@ -998,10 +998,10 @@ addq $1 * SIZE, NEW_Y decq %rax - jg .L998 + jg L(998) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/symv_L_sse2.S b/kernel/x86_64/symv_L_sse2.S index 77331d95f2..12c20a5550 100644 --- a/kernel/x86_64/symv_L_sse2.S +++ b/kernel/x86_64/symv_L_sse2.S @@ -206,7 +206,7 @@ leaq (,LDA, SIZE), LDA testq M, M - jle .L999 + jle L(999) unpcklpd ALPHA, ALPHA @@ -214,10 +214,10 @@ movq M, %rax sarq $3, %rax - jle .L02 + jle L(02) ALIGN_3 -.L01: +L(01): movsd 0 * SIZE(X), %xmm1 addq INCX, X movhpd 0 * SIZE(X), %xmm1 @@ -247,16 +247,16 @@ addq $8 * SIZE, XX decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L02: +L(02): movq M, %rax andq $7, %rax - jle .L05 + jle L(05) ALIGN_3 -.L03: +L(03): movsd 0 * SIZE(X), %xmm1 addq INCX, X @@ -266,10 +266,10 @@ addq $1 * SIZE, XX decq %rax - jg .L03 + jg L(03) ALIGN_3 -.L05: +L(05): /* now we don't need original X */ movq Y, NEW_Y @@ -277,17 +277,17 @@ andq $-512, XX cmpq $SIZE, INCY - je .L10 + je L(10) movq Y, YY movq XX, NEW_Y movq M, %rax sarq $3, %rax - jle .L07 + jle L(07) ALIGN_3 -.L06: +L(06): movsd 0 * SIZE(YY), %xmm0 addq INCY, YY movhpd 0 * SIZE(YY), %xmm0 @@ -312,16 +312,16 @@ addq $8 * SIZE, XX decq %rax - jg .L06 + jg L(06) ALIGN_3 -.L07: +L(07): movq M, %rax andq $7, %rax - jle .L10 + jle L(10) ALIGN_3 -.L08: +L(08): movsd 0 * SIZE(YY), %xmm0 addq INCY, YY @@ -329,17 +329,17 @@ addq $1 * SIZE, XX decq %rax - jg .L08 + jg L(08) ALIGN_3 -.L10: +L(10): xorq IS, IS # is = 0 cmpq $4, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): movq A, A1 leaq (A, LDA, 2), A2 leaq 4 * SIZE(A, LDA, 4), A @@ -423,10 +423,10 @@ subq IS, I subq $4, I sarq $3, I - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): movapd xtemp1, xt1 mulpd a1, xt1 mulpd atemp1, a1 @@ -601,15 +601,15 @@ addq $8 * SIZE, A2 decq I - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): movq M, I subq IS, I subq $4, I test $4, I - jle .L17 + jle L(17) movapd xtemp1, xt1 mulpd a1, xt1 @@ -693,9 +693,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L17: +L(17): testq $2, M - jle .L18 + jle L(18) movapd xtemp1, xt1 mulpd a1, xt1 @@ -739,9 +739,9 @@ addq $2 * SIZE, A2 ALIGN_3 -.L18: +L(18): testq $1, M - jle .L19 + jle L(19) movapd xtemp1, xt1 mulsd a1, xt1 @@ -773,7 +773,7 @@ movsd yy1, 0 * SIZE(YY) ALIGN_3 -.L19: +L(19): #ifndef HAVE_SSE3 movapd xsum1, atemp1 movapd xsum3, atemp3 @@ -809,12 +809,12 @@ movq IS, I addq $4, I cmpq N, I - jle .L11 + jle L(11) ALIGN_3 -.L20: +L(20): testq $2, N - jle .L30 + jle L(30) movq A, A1 leaq 2 * SIZE(A, LDA, 2), A @@ -838,7 +838,7 @@ unpckhpd atemp2, atemp2 testq $1, M - jle .L29 + jle L(29) movsd 2 * SIZE(A1), a1 movsd 2 * SIZE(A1, LDA, 1), a2 @@ -860,7 +860,7 @@ movsd yy1, 2 * SIZE(NEW_Y, IS, SIZE) ALIGN_3 -.L29: +L(29): #ifndef HAVE_SSE3 movapd xsum1, atemp1 unpcklpd xsum2, xsum1 @@ -881,9 +881,9 @@ addq $2, IS ALIGN_3 -.L30: +L(30): testq $1, N - jle .L990 + jle L(990) movsd 0 * SIZE(A), xsum1 movsd 0 * SIZE(NEW_X, IS, SIZE), atemp1 @@ -894,16 +894,16 @@ movsd yy1, 0 * SIZE(NEW_Y, IS, SIZE) ALIGN_3 -.L990: +L(990): cmpq $SIZE, INCY - je .L999 + je L(999) movq M, %rax sarq $3, %rax - jle .L997 + jle L(997) ALIGN_3 -.L996: +L(996): movapd 0 * SIZE(NEW_Y), %xmm0 movapd 2 * SIZE(NEW_Y), %xmm1 movapd 4 * SIZE(NEW_Y), %xmm2 @@ -928,16 +928,16 @@ addq $8 * SIZE, NEW_Y decq %rax - jg .L996 + jg L(996) ALIGN_3 -.L997: +L(997): movq M, %rax andq $7, %rax - jle .L999 + jle L(999) ALIGN_3 -.L998: +L(998): movsd 0 * SIZE(NEW_Y), %xmm0 movsd %xmm0, 0 * SIZE(Y) @@ -946,11 +946,11 @@ addq $1 * SIZE, NEW_Y decq %rax - jg .L998 + jg L(998) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/symv_U_sse.S b/kernel/x86_64/symv_U_sse.S index b611823039..d11a2a7ede 100644 --- a/kernel/x86_64/symv_U_sse.S +++ b/kernel/x86_64/symv_U_sse.S @@ -206,7 +206,7 @@ leaq (,LDA, SIZE), LDA testq M, M - jle .L999 + jle L(999) negq IS addq M, IS @@ -221,10 +221,10 @@ movq M, %rax sarq $3, %rax - jle .L02 + jle L(02) ALIGN_3 -.L01: +L(01): movss 0 * SIZE(X), %xmm1 addq INCX, X movss 0 * SIZE(X), %xmm2 @@ -262,16 +262,16 @@ addq $8 * SIZE, XX decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L02: +L(02): movq M, %rax andq $7, %rax - jle .L05 + jle L(05) ALIGN_3 -.L03: +L(03): movss 0 * SIZE(X), %xmm1 addq INCX, X @@ -281,10 +281,10 @@ addq $1 * SIZE, XX decq %rax - jg .L03 + jg L(03) ALIGN_3 -.L05: +L(05): /* now we don't need original X */ movq Y, NEW_Y @@ -292,17 +292,17 @@ andq $-512, XX cmpq $SIZE, INCY - je .L10 + je L(10) movq Y, YY movq XX, NEW_Y movq M, %rax sarq $3, %rax - jle .L07 + jle L(07) ALIGN_3 -.L06: +L(06): movss 0 * SIZE(YY), %xmm0 addq INCY, YY movss 0 * SIZE(YY), %xmm1 @@ -331,16 +331,16 @@ addq $8 * SIZE, XX decq %rax - jg .L06 + jg L(06) ALIGN_3 -.L07: +L(07): movq M, %rax andq $7, %rax - jle .L10 + jle L(10) ALIGN_3 -.L08: +L(08): movss 0 * SIZE(YY), %xmm0 addq INCY, YY @@ -348,17 +348,17 @@ addq $1 * SIZE, XX decq %rax - jg .L08 + jg L(08) ALIGN_3 -.L10: +L(10): movq IS, I addq $4, I cmpq M, I - jg .L20 + jg L(20) ALIGN_3 -.L11: +L(11): movq A, A1 leaq (A, LDA, 2), A2 leaq (A, LDA, 4), A @@ -395,10 +395,10 @@ movq IS, I sarq $4, I - jle .L14 + jle L(14) ALIGN_3 -.L12: +L(12): movaps xtemp1, xt1 mulps a1, xt1 mulps atemp1, a1 @@ -574,12 +574,12 @@ addq $16 * SIZE, A2 decq I - jg .L12 + jg L(12) ALIGN_3 -.L14: +L(14): testq $8, IS - jle .L15 + jle L(15) movaps xtemp1, xt1 mulps a1, xt1 @@ -663,9 +663,9 @@ addq $8 * SIZE, A2 ALIGN_3 -.L15: +L(15): testq $4, IS - jle .L18 + jle L(18) movaps xtemp1, xt1 mulps a1, xt1 @@ -702,7 +702,7 @@ addq $4 * SIZE, A2 ALIGN_3 -.L18: +L(18): movaps 0 * SIZE(NEW_X, IS, SIZE), atemp1 movss 0 * SIZE(A1), a1 @@ -779,12 +779,12 @@ movq IS, I addq $4, I cmpq M, I - jle .L11 + jle L(11) ALIGN_3 -.L20: +L(20): testq $2, M - jle .L30 + jle L(30) movq A, A1 leaq (A, LDA, 2), A @@ -812,10 +812,10 @@ movq IS, I sarq $2, I - jle .L28 + jle L(28) ALIGN_3 -.L22: +L(22): movaps xtemp1, xt1 mulps a1, xt1 mulps atemp1, a1 @@ -843,10 +843,10 @@ addq $4 * SIZE, A1 decq I - jg .L22 + jg L(22) ALIGN_3 -.L28: +L(28): movsd 0 * SIZE(NEW_X, IS, SIZE), atemp1 movss 0 * SIZE(A1), a1 @@ -883,9 +883,9 @@ addq $2, IS ALIGN_3 -.L30: +L(30): testq $1, M - jle .L990 + jle L(990) movq A, A1 @@ -909,10 +909,10 @@ movq IS, I sarq $1, I - jle .L38 + jle L(38) ALIGN_3 -.L32: +L(32): movaps xtemp1, xt1 movss 2 * SIZE(XX), xtemp1 mulps a1, xt1 @@ -940,10 +940,10 @@ addq $2 * SIZE, A1 decq I - jg .L32 + jg L(32) ALIGN_3 -.L38: +L(38): movsd 0 * SIZE(NEW_X, IS, SIZE), atemp1 movss 0 * SIZE(A1), a1 @@ -971,16 +971,16 @@ addq $2, IS ALIGN_3 -.L990: +L(990): cmpq $SIZE, INCY - je .L999 + je L(999) movq M, %rax sarq $3, %rax - jle .L997 + jle L(997) ALIGN_3 -.L996: +L(996): movss 0 * SIZE(NEW_Y), %xmm0 movss 1 * SIZE(NEW_Y), %xmm1 movss 2 * SIZE(NEW_Y), %xmm2 @@ -1009,16 +1009,16 @@ addq $8 * SIZE, NEW_Y decq %rax - jg .L996 + jg L(996) ALIGN_3 -.L997: +L(997): movq M, %rax andq $7, %rax - jle .L999 + jle L(999) ALIGN_3 -.L998: +L(998): movss 0 * SIZE(NEW_Y), %xmm0 movss %xmm0, 0 * SIZE(Y) @@ -1027,11 +1027,11 @@ addq $1 * SIZE, NEW_Y decq %rax - jg .L998 + jg L(998) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/symv_U_sse2.S b/kernel/x86_64/symv_U_sse2.S index 99bc07d502..d531febd99 100644 --- a/kernel/x86_64/symv_U_sse2.S +++ b/kernel/x86_64/symv_U_sse2.S @@ -205,7 +205,7 @@ leaq (,LDA, SIZE), LDA testq M, M - jle .L999 + jle L(999) negq IS addq M, IS @@ -220,10 +220,10 @@ movq M, %rax sarq $3, %rax - jle .L02 + jle L(02) ALIGN_3 -.L01: +L(01): movsd 0 * SIZE(X), %xmm1 addq INCX, X movhpd 0 * SIZE(X), %xmm1 @@ -253,16 +253,16 @@ addq $8 * SIZE, XX decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L02: +L(02): movq M, %rax andq $7, %rax - jle .L05 + jle L(05) ALIGN_3 -.L03: +L(03): movsd 0 * SIZE(X), %xmm1 addq INCX, X @@ -272,10 +272,10 @@ addq $1 * SIZE, XX decq %rax - jg .L03 + jg L(03) ALIGN_3 -.L05: +L(05): /* now we don't need original X */ movq Y, NEW_Y @@ -283,17 +283,17 @@ andq $-512, XX cmpq $SIZE, INCY - je .L10 + je L(10) movq Y, YY movq XX, NEW_Y movq M, %rax sarq $3, %rax - jle .L07 + jle L(07) ALIGN_3 -.L06: +L(06): movsd 0 * SIZE(YY), %xmm0 addq INCY, YY movhpd 0 * SIZE(YY), %xmm0 @@ -318,16 +318,16 @@ addq $8 * SIZE, XX decq %rax - jg .L06 + jg L(06) ALIGN_3 -.L07: +L(07): movq M, %rax andq $7, %rax - jle .L10 + jle L(10) ALIGN_3 -.L08: +L(08): movsd 0 * SIZE(YY), %xmm0 addq INCY, YY @@ -335,17 +335,17 @@ addq $1 * SIZE, XX decq %rax - jg .L08 + jg L(08) ALIGN_3 -.L10: +L(10): movq IS, I addq $4, I cmpq M, I - jg .L20 + jg L(20) ALIGN_3 -.L11: +L(11): movq A, A1 leaq (A, LDA, 2), A2 leaq (A, LDA, 4), A @@ -391,10 +391,10 @@ movq IS, I sarq $3, I - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): movapd xtemp1, xt1 mulpd a1, xt1 mulpd atemp1, a1 @@ -569,12 +569,12 @@ addq $8 * SIZE, A2 decq I - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): testq $4, IS - jle .L18 + jle L(18) movapd xtemp1, xt1 mulpd a1, xt1 @@ -652,7 +652,7 @@ addq $4 * SIZE, A2 ALIGN_3 -.L18: +L(18): unpckhpd atemp2, atemp1 unpckhpd atemp4, atemp3 @@ -726,15 +726,15 @@ movq IS, I addq $4, I cmpq M, I - jle .L11 + jle L(11) ALIGN_3 -.L20: +L(20): testq $2, M - je .L30 + je L(30) ALIGN_3 -.L21: +L(21): movq A, A1 leaq (A, LDA, 2), A @@ -766,10 +766,10 @@ movq IS, I sarq $1, I - jle .L28 + jle L(28) ALIGN_3 -.L22: +L(22): movapd xtemp1, xt1 mulpd a1, xt1 mulpd atemp1, a1 @@ -797,10 +797,10 @@ addq $2 * SIZE, A1 decq I - jg .L22 + jg L(22) ALIGN_3 -.L28: +L(28): unpckhpd atemp2, atemp1 movsd 0 * SIZE(A1), a1 @@ -832,12 +832,12 @@ addq $2, IS ALIGN_3 -.L30: +L(30): testq $1, M - je .L990 + je L(990) ALIGN_3 -.L31: +L(31): movq A, A1 #ifdef HAVE_SSE3 @@ -858,10 +858,10 @@ movq IS, I testq I, I - jle .L38 + jle L(38) ALIGN_3 -.L32: +L(32): movapd xtemp1, xt1 mulpd a1, xt1 mulpd atemp1, a1 @@ -879,10 +879,10 @@ addq $1 * SIZE, A1 decq I - jg .L32 + jg L(32) ALIGN_3 -.L38: +L(38): movsd 0 * SIZE(A1), a1 mulsd atemp1, a1 addsd a1, xsum1 @@ -892,16 +892,16 @@ movsd yy1, 0 * SIZE(YY) ALIGN_3 -.L990: +L(990): cmpq $SIZE, INCY - je .L999 + je L(999) movq M, %rax sarq $3, %rax - jle .L997 + jle L(997) ALIGN_3 -.L996: +L(996): movapd 0 * SIZE(NEW_Y), %xmm0 movapd 2 * SIZE(NEW_Y), %xmm1 movapd 4 * SIZE(NEW_Y), %xmm2 @@ -926,16 +926,16 @@ addq $8 * SIZE, NEW_Y decq %rax - jg .L996 + jg L(996) ALIGN_3 -.L997: +L(997): movq M, %rax andq $7, %rax - jle .L999 + jle L(999) ALIGN_3 -.L998: +L(998): movsd 0 * SIZE(NEW_Y), %xmm0 movsd %xmm0, 0 * SIZE(Y) @@ -944,11 +944,11 @@ addq $1 * SIZE, NEW_Y decq %rax - jg .L998 + jg L(998) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LN_2x8_nehalem.S b/kernel/x86_64/trsm_kernel_LN_2x8_nehalem.S index 8deff20aa9..30cc844359 100644 --- a/kernel/x86_64/trsm_kernel_LN_2x8_nehalem.S +++ b/kernel/x86_64/trsm_kernel_LN_2x8_nehalem.S @@ -167,10 +167,10 @@ movq N, J sarq $3, J NOBRANCH - jle .L30 + jle L(30) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -209,7 +209,7 @@ testq $1, M BRANCH - jle .L20 + jle L(20) ALIGN_4 #ifdef LN @@ -243,10 +243,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -308,10 +308,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -320,10 +320,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -343,10 +343,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -683,14 +683,14 @@ #endif ALIGN_4 -.L20: +L(20): movq M, I sarq $1, I NOBRANCH - jle .L29 + jle L(29) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -744,10 +744,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm12 @@ -870,10 +870,10 @@ subq $-8 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -882,10 +882,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addpd %xmm1, %xmm12 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm13 @@ -921,10 +921,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1372,10 +1372,10 @@ decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L29: +L(29): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 8), B @@ -1394,12 +1394,12 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L30: +L(30): testq $4, N - jle .L50 + jle L(50) ALIGN_4 #if defined(LT) || defined(RN) @@ -1436,7 +1436,7 @@ testq $1, M BRANCH - jle .L40 + jle L(40) ALIGN_4 #ifdef LN @@ -1470,10 +1470,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1511,10 +1511,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1523,10 +1523,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1540,10 +1540,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1701,14 +1701,14 @@ #endif ALIGN_4 -.L40: +L(40): movq M, I sarq $1, I NOBRANCH - jle .L49 + jle L(49) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -1748,10 +1748,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -1823,10 +1823,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1835,10 +1835,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): addpd %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm9 @@ -1860,10 +1860,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2091,10 +2091,10 @@ decq I BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L49: +L(49): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -2112,9 +2112,9 @@ #endif ALIGN_4 -.L50: +L(50): testq $2, N - jle .L70 + jle L(70) ALIGN_4 #if defined(LT) || defined(RN) @@ -2151,7 +2151,7 @@ testq $1, M BRANCH - jle .L60 + jle L(60) ALIGN_4 #ifdef LN @@ -2183,10 +2183,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): mulpd %xmm0, %xmm1 movddup -15 * SIZE(AO), %xmm0 addpd %xmm1, %xmm8 @@ -2212,10 +2212,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2224,10 +2224,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): mulpd %xmm0, %xmm1 movddup -15 * SIZE(AO), %xmm0 addpd %xmm1, %xmm8 @@ -2238,10 +2238,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2344,14 +2344,14 @@ #endif ALIGN_4 -.L60: +L(60): movq M, I sarq $1, I NOBRANCH - jle .L69 + jle L(69) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -2387,10 +2387,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -2430,13 +2430,13 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 ALIGN_3 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2445,10 +2445,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addpd %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm9 @@ -2462,10 +2462,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2600,10 +2600,10 @@ decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L69: +L(69): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2621,9 +2621,9 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, N - jle .L999 + jle L(999) ALIGN_4 #if defined(LT) || defined(RN) @@ -2658,7 +2658,7 @@ testq $1, M BRANCH - jle .L80 + jle L(80) ALIGN_4 #ifdef LN @@ -2692,10 +2692,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): mulpd %xmm0, %xmm1 movsd -14 * SIZE(AO), %xmm0 movhps -13 * SIZE(AO), %xmm0 @@ -2715,12 +2715,12 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) addpd %xmm9, %xmm8 ALIGN_3 -.L85: +L(85): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2729,10 +2729,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): mulsd %xmm0, %xmm1 movsd -15 * SIZE(AO), %xmm0 addsd %xmm1, %xmm8 @@ -2743,10 +2743,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -2819,14 +2819,14 @@ ALIGN_4 -.L80: +L(80): movq M, I sarq $1, I NOBRANCH - jle .L89 + jle L(89) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -2861,10 +2861,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -2892,12 +2892,12 @@ subq $1, %rax BRANCH - jg .L72 + jg L(72) addpd %xmm9, %xmm8 ALIGN_3 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2906,10 +2906,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addpd %xmm1, %xmm8 movddup -16 * SIZE(BO), %xmm1 mulpd %xmm0, %xmm1 @@ -2920,10 +2920,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3024,10 +3024,10 @@ decq I BRANCH - jg .L71 + jg L(71) ALIGN_4 -.L89: +L(89): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -3046,7 +3046,7 @@ ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LN_4x2_atom.S b/kernel/x86_64/trsm_kernel_LN_4x2_atom.S index a192674999..60bec8cc0a 100644 --- a/kernel/x86_64/trsm_kernel_LN_4x2_atom.S +++ b/kernel/x86_64/trsm_kernel_LN_4x2_atom.S @@ -156,10 +156,10 @@ movq N, J sarq $1, J - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -197,7 +197,7 @@ #endif testq $1, M - je .L20 + je L(20) #ifdef LN movq K, %rax @@ -233,10 +233,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): addsd %xmm5, %xmm8 movsd 2 * SIZE(BO), %xmm5 mulsd %xmm0, %xmm1 @@ -278,10 +278,10 @@ addq $8 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -294,10 +294,10 @@ andq $3, %rax BRANCH BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulsd %xmm0, %xmm1 addq $2 * SIZE, BO mulsd %xmm0, %xmm3 @@ -311,10 +311,10 @@ addq $1 * SIZE, AO decq %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -413,10 +413,10 @@ #endif ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - je .L30 + je L(30) #ifdef LN movq K, %rax @@ -456,10 +456,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addsd %xmm2, %xmm9 movaps %xmm0, %xmm2 @@ -537,10 +537,10 @@ addq $8 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -549,10 +549,10 @@ #endif andq $3, %rax BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): addsd %xmm2, %xmm9 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -575,10 +575,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): addsd %xmm2, %xmm9 addsd %xmm6, %xmm11 @@ -740,13 +740,13 @@ #endif ALIGN_4 -.L30: +L(30): movq M, I sarq $2, I - jle .L39 + jle L(39) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -795,10 +795,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): addsd %xmm2, %xmm13 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps %xmm0, %xmm2 @@ -941,10 +941,10 @@ mulsd %xmm3, %xmm6 movsd 1 * SIZE(BO), %xmm3 - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -953,10 +953,10 @@ #endif andq $3, %rax BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): addsd %xmm2, %xmm13 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -995,10 +995,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): addsd %xmm2, %xmm13 addsd %xmm7, %xmm14 addsd %xmm6, %xmm15 @@ -1298,10 +1298,10 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -1319,12 +1319,12 @@ #endif decq J # j -- - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $1, N - je .L999 + je L(999) ALIGN_4 #if defined(LT) || defined(RN) @@ -1358,7 +1358,7 @@ #endif testq $1, M - je .L50 + je L(50) #ifdef LN movq K, %rax @@ -1395,10 +1395,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): addsd %xmm5, %xmm8 movsd 2 * SIZE(BO), %xmm5 mulsd %xmm0, %xmm1 @@ -1423,13 +1423,13 @@ addq $4 * SIZE, BO decq %rax - jne .L62 + jne L(62) addsd %xmm5, %xmm8 addsd %xmm7, %xmm9 ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1438,10 +1438,10 @@ #endif andq $3, %rax BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): movsd 0 * SIZE(AO), %xmm0 movsd 0 * SIZE(BO), %xmm1 @@ -1453,10 +1453,10 @@ decq %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addsd %xmm9, %xmm8 #if defined(LN) || defined(RT) @@ -1530,9 +1530,9 @@ #endif ALIGN_4 -.L50: +L(50): testq $2, M - je .L60 + je L(60) #ifdef LN movq K, %rax @@ -1567,10 +1567,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): addsd %xmm2, %xmm8 movsd 2 * SIZE(AO), %xmm2 mulsd %xmm4, %xmm0 @@ -1611,10 +1611,10 @@ mulsd %xmm5, %xmm3 movsd 1 * SIZE(BO), %xmm5 - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1626,10 +1626,10 @@ andq $3, %rax BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): mulsd %xmm4, %xmm0 mulsd %xmm4, %xmm1 movsd 1 * SIZE(BO), %xmm4 @@ -1643,10 +1643,10 @@ addq $1 * SIZE, BO decq %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1743,13 +1743,13 @@ #endif ALIGN_4 -.L60: +L(60): movq M, I sarq $2, I - jle .L69 + jle L(69) ALIGN_4 -.L41: +L(41): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1790,10 +1790,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): addsd %xmm9, %xmm8 movsd 4 * SIZE(AO), %xmm9 mulsd %xmm4, %xmm0 @@ -1867,10 +1867,10 @@ mulsd %xmm5, %xmm15 movsd 1 * SIZE(BO), %xmm5 - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1886,10 +1886,10 @@ andq $3, %rax BRANCH BRANCH - je .L49 + je L(49) ALIGN_4 -.L46: +L(46): mulsd %xmm4, %xmm0 mulsd %xmm4, %xmm1 mulsd %xmm4, %xmm2 @@ -1909,10 +1909,10 @@ addq $1 * SIZE, BO decq %rax BRANCH - jg .L46 + jg L(46) ALIGN_4 -.L49: +L(49): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2065,10 +2065,10 @@ #endif decq I # i -- - jg .L41 + jg L(41) ALIGN_4 -.L69: +L(69): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -2087,7 +2087,7 @@ #endif ALIGN_2 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LN_4x4_barcelona.S b/kernel/x86_64/trsm_kernel_LN_4x4_barcelona.S index 69278bb5e8..e5e69c5451 100644 --- a/kernel/x86_64/trsm_kernel_LN_4x4_barcelona.S +++ b/kernel/x86_64/trsm_kernel_LN_4x4_barcelona.S @@ -480,9 +480,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -520,7 +520,7 @@ #endif testq $1, M - je .L20 + je L(20) #ifdef LN movq K, %rax @@ -565,10 +565,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L36 + je L(36) ALIGN_4 -.L32: +L(32): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO, %rax, 4), %xmm0 addpd %xmm1, %xmm8 @@ -596,10 +596,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L32 + jl L(32) ALIGN_4 -.L36: +L(36): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -607,7 +607,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L38 + je L(38) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -615,7 +615,7 @@ negq %rax ALIGN_4 -.L37: +L(37): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO, %rax, 4), %xmm0 addpd %xmm1, %xmm8 @@ -624,10 +624,10 @@ movddup -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L37 + jl L(37) ALIGN_4 -.L38: +L(38): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -808,12 +808,12 @@ #endif ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -857,10 +857,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L26 + je L(26) ALIGN_4 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -916,10 +916,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L22 + jl L(22) ALIGN_4 -.L26: +L(26): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -927,7 +927,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L29 + je L(29) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -935,7 +935,7 @@ negq %rax ALIGN_4 -.L27: +L(27): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -951,10 +951,10 @@ movddup -11 * SIZE(BO, %rax, 4), %xmm5 addq $SIZE, %rax - jl .L27 + jl L(27) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1172,13 +1172,13 @@ #endif ALIGN_4 -.L30: +L(30): movq M, I sarq $2, I # i = (m >> 2) - jle .L39 + jle L(39) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1247,10 +1247,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -1260,10 +1260,10 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) BRANCH - jl .L12 + jl L(12) ALIGN_4 -.L15: +L(15): prefetch 14 * SIZE(BB) subq $-16 * SIZE, BB @@ -1274,7 +1274,7 @@ subq KK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_4 @@ -1287,7 +1287,7 @@ subq $-16 * SIZE, AO ALIGN_4 -.L16: +L(16): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1295,7 +1295,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L19 + je L(19) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -1303,7 +1303,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -1330,10 +1330,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L17 + jl L(17) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1740,10 +1740,10 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1762,12 +1762,12 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $2, N - je .L80 + je L(80) #if defined(LT) || defined(RN) movq A, AO @@ -1802,10 +1802,10 @@ #endif testq $1, M - je .L60 + je L(60) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1848,10 +1848,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L76 + je L(76) ALIGN_4 -.L72: +L(72): mulpd -16 * SIZE(BO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -12 * SIZE(AO, %rax, 1), %xmm0 @@ -1870,10 +1870,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L72 + jl L(72) ALIGN_4 -.L76: +L(76): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1881,7 +1881,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L78 + je L(78) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -1889,16 +1889,16 @@ negq %rax ALIGN_4 -.L77: +L(77): mulpd -16 * SIZE(BO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L77 + jl L(77) ALIGN_4 -.L78: +L(78): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -2001,9 +2001,9 @@ #endif ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) #ifdef LN movq K, %rax @@ -2047,10 +2047,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L66 + je L(66) ALIGN_4 -.L62: +L(62): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -2082,10 +2082,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L62 + jl L(62) ALIGN_4 -.L66: +L(66): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2093,7 +2093,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L69 + je L(69) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -2101,7 +2101,7 @@ negq %rax ALIGN_4 -.L67: +L(67): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -2111,10 +2111,10 @@ movddup -13 * SIZE(BO, %rax, 2), %xmm3 addq $SIZE, %rax - jl .L67 + jl L(67) ALIGN_4 -.L69: +L(69): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -2255,13 +2255,13 @@ #endif ALIGN_4 -.L70: +L(70): movq M, I sarq $2, I # i = (m >> 2) - jle .L79 + jle L(79) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2316,10 +2316,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L56 + je L(56) ALIGN_4 -.L52: +L(52): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -2371,10 +2371,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L52 + jl L(52) ALIGN_4 -.L56: +L(56): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2382,7 +2382,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L59 + je L(59) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -2390,7 +2390,7 @@ negq %rax ALIGN_4 -.L57: +L(57): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -2405,10 +2405,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L57 + jl L(57) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2621,10 +2621,10 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2643,9 +2643,9 @@ #endif ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) #if defined(LT) || defined(RN) movq A, AO @@ -2678,7 +2678,7 @@ #endif testq $1, M - je .L100 + je L(100) #ifdef LN movq K, %rax @@ -2717,10 +2717,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L116 + je L(116) ALIGN_4 -.L112: +L(112): mulpd -16 * SIZE(BO, %rax, 1), %xmm0 addpd %xmm0, %xmm8 movapd -12 * SIZE(AO, %rax, 1), %xmm0 @@ -2731,10 +2731,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L112 + jl L(112) ALIGN_4 -.L116: +L(116): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2742,7 +2742,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L118 + je L(118) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -2750,16 +2750,16 @@ negq %rax ALIGN_4 -.L117: +L(117): mulsd -16 * SIZE(BO, %rax, 1), %xmm0 addsd %xmm0, %xmm8 movsd -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L117 + jl L(117) ALIGN_4 -.L118: +L(118): addpd %xmm9, %xmm8 haddpd %xmm8, %xmm8 @@ -2835,9 +2835,9 @@ #endif ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) #ifdef LN movq K, %rax @@ -2880,10 +2880,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L106 + je L(106) ALIGN_4 -.L102: +L(102): mulpd -16 * SIZE(AO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -12 * SIZE(BO, %rax, 1), %xmm0 @@ -2902,10 +2902,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L102 + jl L(102) ALIGN_4 -.L106: +L(106): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2913,7 +2913,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L109 + je L(109) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -2921,16 +2921,16 @@ negq %rax ALIGN_4 -.L107: +L(107): movddup -16 * SIZE(BO, %rax, 1), %xmm0 mulpd -16 * SIZE(AO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 addq $SIZE, %rax - jl .L107 + jl L(107) ALIGN_4 -.L109: +L(109): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -3047,13 +3047,13 @@ #endif ALIGN_4 -.L110: +L(110): movq M, I sarq $2, I # i = (m >> 2) - jle .L119 + jle L(119) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -3102,10 +3102,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L96 + je L(96) ALIGN_4 -.L92: +L(92): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -3133,10 +3133,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L92 + jl L(92) ALIGN_4 -.L96: +L(96): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3144,7 +3144,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L99 + je L(99) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -3152,7 +3152,7 @@ negq %rax ALIGN_4 -.L97: +L(97): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -3161,9 +3161,9 @@ movddup -15 * SIZE(BO, %rax, 1), %xmm1 addq $SIZE, %rax - jl .L97 + jl L(97) ALIGN_4 -.L99: +L(99): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -3340,10 +3340,10 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L119: +L(119): #ifdef LN leaq (B, K, SIZE), B #endif @@ -3361,7 +3361,7 @@ #endif ALIGN_4 -.L999: +L(999): movq (%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LN_4x4_core2.S b/kernel/x86_64/trsm_kernel_LN_4x4_core2.S index 3a16bfc3f3..8ce9402c31 100644 --- a/kernel/x86_64/trsm_kernel_LN_4x4_core2.S +++ b/kernel/x86_64/trsm_kernel_LN_4x4_core2.S @@ -171,9 +171,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) -.L01: +L(01): /* Copying to Sub Buffer */ #ifdef LN @@ -210,10 +210,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): prefetcht0 (PREFETCH_R + 0) * SIZE(B) movapd -16 * SIZE(B), %xmm0 movapd -14 * SIZE(B), %xmm1 @@ -269,10 +269,10 @@ subq $-16 * SIZE, B subq $-32 * SIZE, BO subq $1, %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -281,10 +281,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movapd -16 * SIZE(B), %xmm0 movapd -14 * SIZE(B), %xmm1 @@ -301,10 +301,10 @@ addq $4 * SIZE, B addq $8 * SIZE, BO subq $1, %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): leaq (PREFETCH_R + 0) * SIZE(B), BB #if defined(LT) || defined(RN) @@ -325,10 +325,10 @@ #endif testq $1, M - je .L20 + je L(20) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -362,10 +362,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -16 * SIZE(AO), %xmm0 @@ -435,10 +435,10 @@ subq $ -4 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -447,10 +447,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): movsd -16 * SIZE(AO), %xmm0 movsd -16 * SIZE(BO), %xmm2 movsd -14 * SIZE(BO), %xmm3 @@ -470,10 +470,10 @@ addq $1 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -649,12 +649,12 @@ #endif ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -688,10 +688,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -761,10 +761,10 @@ subq $ -8 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -773,10 +773,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -796,10 +796,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jne .L26 + jne L(26) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1042,13 +1042,13 @@ #endif ALIGN_4 -.L30: +L(30): movq M, I sarq $2, I # i = (m >> 2) - jle .L39 + jle L(39) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1111,10 +1111,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm2, %xmm10 @@ -1234,10 +1234,10 @@ mulpd %xmm1, %xmm5 BRANCH - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1246,10 +1246,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): movapd -16 * SIZE(AO), %xmm0 addpd %xmm2, %xmm10 movapd -16 * SIZE(BO), %xmm2 @@ -1282,10 +1282,10 @@ addq $8 * SIZE, BO # boffset1 += 8 subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): addpd %xmm2, %xmm10 addpd %xmm3, %xmm14 addpd %xmm4, %xmm11 @@ -1738,11 +1738,11 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1764,18 +1764,18 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $3, N - je .L999 + je L(999) testq $2, N - je .L80 + je L(80) ALIGN_4 -.L41: +L(41): /* Copying to Sub Buffer */ #ifdef LN @@ -1812,10 +1812,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): movddup -16 * SIZE(B), %xmm0 movddup -15 * SIZE(B), %xmm1 movddup -14 * SIZE(B), %xmm2 @@ -1837,10 +1837,10 @@ addq $8 * SIZE, B addq $16 * SIZE, BO subq $1, %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1849,10 +1849,10 @@ #endif andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movddup -16 * SIZE(B), %xmm0 movddup -15 * SIZE(B), %xmm1 @@ -1862,10 +1862,10 @@ addq $2 * SIZE, B addq $4 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): #if defined(LT) || defined(RN) movq A, AO #else @@ -1884,10 +1884,10 @@ #endif testq $1, M - je .L60 + je L(60) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1921,10 +1921,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -16 * SIZE(AO), %xmm0 @@ -1964,10 +1964,10 @@ subq $ -4 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1976,10 +1976,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): movsd -16 * SIZE(AO), %xmm0 movsd -16 * SIZE(BO), %xmm2 movsd -14 * SIZE(BO), %xmm3 @@ -1993,10 +1993,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addsd %xmm10, %xmm8 addsd %xmm11, %xmm9 @@ -2115,12 +2115,12 @@ #endif ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -2154,10 +2154,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -2197,10 +2197,10 @@ subq $ -8 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2209,10 +2209,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_4 -.L66: +L(66): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -2226,10 +2226,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L66 + jg L(66) ALIGN_4 -.L69: +L(69): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -2387,13 +2387,13 @@ #endif ALIGN_4 -.L70: +L(70): movq M, I sarq $2, I # i = (m >> 2) - jle .L79 + jle L(79) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2437,10 +2437,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -2518,10 +2518,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2530,10 +2530,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): movapd -16 * SIZE(AO), %xmm0 movapd -14 * SIZE(AO), %xmm1 @@ -2555,10 +2555,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2796,10 +2796,10 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2821,12 +2821,12 @@ #endif ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L81: +L(81): /* Copying to Sub Buffer */ #ifdef LN @@ -2863,10 +2863,10 @@ subq KK, %rax #endif sarq $3, %rax - jle .L83 + jle L(83) ALIGN_4 -.L82: +L(82): movddup -16 * SIZE(B), %xmm0 movddup -15 * SIZE(B), %xmm1 movddup -14 * SIZE(B), %xmm2 @@ -2888,10 +2888,10 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO subq $1, %rax - jne .L82 + jne L(82) ALIGN_4 -.L83: +L(83): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2900,10 +2900,10 @@ #endif andq $7, %rax BRANCH - jle .L90 + jle L(90) ALIGN_4 -.L84: +L(84): movddup -16 * SIZE(B), %xmm0 movapd %xmm0, 0 * SIZE(BO) @@ -2911,10 +2911,10 @@ addq $1 * SIZE, B addq $2 * SIZE, BO subq $1, %rax - jne .L84 + jne L(84) ALIGN_4 -.L90: +L(90): #if defined(LT) || defined(RN) movq A, AO #else @@ -2931,10 +2931,10 @@ #endif testq $1, M - je .L100 + je L(100) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -2968,10 +2968,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -16 * SIZE(AO), %xmm0 @@ -2997,10 +2997,10 @@ subq $-4 * SIZE, AO subq $-8 * SIZE, BO subq $1, %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3009,10 +3009,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): movsd -16 * SIZE(AO), %xmm0 movsd -16 * SIZE(BO), %xmm2 @@ -3022,10 +3022,10 @@ addq $1 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addsd %xmm10, %xmm8 addsd %xmm11, %xmm9 addsd %xmm9, %xmm8 @@ -3126,12 +3126,12 @@ #endif ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -3165,10 +3165,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L105 + je L(105) ALIGN_4 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -3196,10 +3196,10 @@ subq $-8 * SIZE, AO subq $-8 * SIZE, BO subq $1, %rax - jne .L102 + jne L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3208,10 +3208,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_4 -.L106: +L(106): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 @@ -3221,10 +3221,10 @@ addq $2 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L106 + jg L(106) ALIGN_4 -.L109: +L(109): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -3354,13 +3354,13 @@ #endif ALIGN_4 -.L110: +L(110): movq M, I sarq $2, I # i = (m >> 2) - jle .L119 + jle L(119) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -3399,10 +3399,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -3448,10 +3448,10 @@ subq $-16 * SIZE, AO subq $ -8 * SIZE, BO subq $1, %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3460,10 +3460,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_4 -.L96: +L(96): movapd -16 * SIZE(AO), %xmm0 movapd -14 * SIZE(AO), %xmm1 @@ -3478,10 +3478,10 @@ addq $4 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L96 + jg L(96) ALIGN_4 -.L99: +L(99): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -3682,10 +3682,10 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L119: +L(119): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -3708,7 +3708,7 @@ ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/trsm_kernel_LN_4x4_penryn.S b/kernel/x86_64/trsm_kernel_LN_4x4_penryn.S index 3bc7ae1f85..e222635b78 100644 --- a/kernel/x86_64/trsm_kernel_LN_4x4_penryn.S +++ b/kernel/x86_64/trsm_kernel_LN_4x4_penryn.S @@ -166,10 +166,10 @@ movq N, J sarq $2, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -208,7 +208,7 @@ testq $1, M BRANCH - jle .L20 + jle L(20) ALIGN_4 #ifdef LN @@ -244,10 +244,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) shufps $0x44, %xmm0, %xmm0 @@ -294,10 +294,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -306,10 +306,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): shufps $0x44, %xmm0, %xmm0 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm3 @@ -325,10 +325,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -483,10 +483,10 @@ #endif ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) ALIGN_4 #ifdef LN @@ -523,10 +523,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_4 -.L22: +L(22): addpd %xmm3, %xmm11 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -592,10 +592,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -604,10 +604,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): addpd %xmm3, %xmm11 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -628,10 +628,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -859,14 +859,14 @@ #endif ALIGN_4 -.L30: +L(30): movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L39 + jle L(39) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -933,10 +933,10 @@ #endif sarq $3, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): addpd %xmm3, %xmm11 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -14 * SIZE(BO), %xmm3 @@ -1186,10 +1186,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1198,10 +1198,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addpd %xmm3, %xmm11 movaps -14 * SIZE(BO), %xmm3 addpd %xmm4, %xmm15 @@ -1237,10 +1237,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax subq $4, %rax @@ -1665,10 +1665,10 @@ decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1687,13 +1687,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $2, N BRANCH - jle .L80 + jle L(80) #if defined(LT) || defined(RN) movq A, AO @@ -1733,7 +1733,7 @@ testq $1, M BRANCH - jle .L60 + jle L(60) ALIGN_4 #ifdef LN @@ -1766,10 +1766,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_4 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) shufps $0x44, %xmm0, %xmm0 @@ -1800,10 +1800,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1812,10 +1812,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): shufps $0x44, %xmm0, %xmm0 mulpd %xmm0, %xmm2 movsd -15 * SIZE(AO), %xmm0 @@ -1827,10 +1827,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1937,10 +1937,10 @@ #endif ALIGN_4 -.L60: +L(60): testq $2, M BRANCH - jle .L70 + jle L(70) ALIGN_4 #ifdef LN @@ -1974,10 +1974,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x4e, %xmm2, %xmm7 @@ -2020,10 +2020,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2032,10 +2032,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm7 @@ -2050,10 +2050,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2198,14 +2198,14 @@ #endif ALIGN_4 -.L70: +L(70): movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L79 + jle L(79) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2254,10 +2254,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_4 -.L52: +L(52): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -2332,10 +2332,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2344,10 +2344,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -2370,10 +2370,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2595,10 +2595,10 @@ decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2616,10 +2616,10 @@ #endif ALIGN_4 -.L80: +L(80): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(LT) || defined(RN) movq A, AO @@ -2653,7 +2653,7 @@ testq $1, M BRANCH - jle .L90 + jle L(90) #ifdef LN movq K, %rax @@ -2685,10 +2685,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_4 -.L112: +L(112): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulsd %xmm0, %xmm2 @@ -2715,10 +2715,10 @@ subq $-4 * SIZE, BO subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2727,10 +2727,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd %xmm0, %xmm2 addsd %xmm2, %xmm8 movsd -15 * SIZE(AO), %xmm0 @@ -2741,10 +2741,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -2830,10 +2830,10 @@ #endif ALIGN_4 -.L90: +L(90): testq $2, M BRANCH - jle .L110 + jle L(110) ALIGN_4 #ifdef LN @@ -2866,10 +2866,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_4 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm2, %xmm3 @@ -2904,10 +2904,10 @@ subq $-4 * SIZE, BO subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2916,10 +2916,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L108 + je L(108) ALIGN_4 -.L106: +L(106): pshufd $0x44, %xmm2, %xmm3 movsd -15 * SIZE(BO), %xmm2 @@ -2932,10 +2932,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_4 -.L108: +L(108): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3051,14 +3051,14 @@ #endif ALIGN_4 -.L110: +L(110): movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L119 + jle L(119) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -3097,10 +3097,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_4 -.L92: +L(92): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm2, %xmm3 @@ -3157,10 +3157,10 @@ subq $ -4 * SIZE, BO subq $1, %rax BRANCH - jg .L92 + jg L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3169,10 +3169,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): pshufd $0x44, %xmm2, %xmm3 pshufd $0x44, %xmm2, %xmm4 movsd -15 * SIZE(BO), %xmm2 @@ -3190,10 +3190,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3376,10 +3376,10 @@ decq I BRANCH - jg .L91 + jg L(91) ALIGN_4 -.L119: +L(119): #ifdef LN leaq (B, K, SIZE), B #endif @@ -3396,7 +3396,7 @@ #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LN_4x4_sse2.S b/kernel/x86_64/trsm_kernel_LN_4x4_sse2.S index c846080a40..17fa5bf912 100644 --- a/kernel/x86_64/trsm_kernel_LN_4x4_sse2.S +++ b/kernel/x86_64/trsm_kernel_LN_4x4_sse2.S @@ -294,9 +294,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) -.L01: +L(01): /* Copying to Sub Buffer */ #ifdef LN @@ -333,12 +333,12 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): PREFETCHNTA 40 * SIZE(B) movsd 0 * SIZE(B), %xmm0 @@ -371,10 +371,10 @@ movsd %xmm7, -1 * SIZE(BO) decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -383,10 +383,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movsd 0 * SIZE(B), %xmm0 movsd 1 * SIZE(B), %xmm1 movsd 2 * SIZE(B), %xmm2 @@ -404,10 +404,10 @@ addq $4 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -426,10 +426,10 @@ #endif testq $1, M - je .L20 + je L(20) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -470,10 +470,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulsd %xmm8, %xmm9 addsd %xmm9, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -582,10 +582,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -594,10 +594,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulsd %xmm8, %xmm9 addsd %xmm9, %xmm0 movsd 2 * SIZE(BO), %xmm9 @@ -614,10 +614,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -793,12 +793,12 @@ #endif ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -839,10 +839,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -952,10 +952,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -964,10 +964,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -984,10 +984,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1258,13 +1258,13 @@ ALIGN_4 -.L30: +L(30): movq M, I sarq $2, I # i = (m >> 2) - jle .L39 + jle L(39) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1328,8 +1328,8 @@ #endif andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -1347,7 +1347,7 @@ KERNEL7(16 * 1) KERNEL8(16 * 1) cmpq $64 * 2, %rax - jle .L12 + jle L(12) KERNEL1(16 * 2) KERNEL2(16 * 2) KERNEL3(16 * 2) @@ -1365,7 +1365,7 @@ KERNEL7(16 * 3) KERNEL8(16 * 3) cmpq $64 * 4, %rax - jle .L12 + jle L(12) KERNEL1(16 * 4) KERNEL2(16 * 4) KERNEL3(16 * 4) @@ -1383,7 +1383,7 @@ KERNEL7(16 * 5) KERNEL8(16 * 5) cmpq $64 * 6, %rax - jle .L12 + jle L(12) KERNEL1(16 * 6) KERNEL2(16 * 6) KERNEL3(16 * 6) @@ -1404,14 +1404,14 @@ addq $16 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $64 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 4), BO # * 64 ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1420,10 +1420,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -1452,10 +1452,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1967,10 +1967,10 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1992,18 +1992,18 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $3, N - je .L999 + je L(999) testq $2, N - je .L80 + je L(80) ALIGN_4 -.L41: +L(41): /* Copying to Sub Buffer */ #ifdef LN @@ -2040,10 +2040,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): PREFETCH 56 * SIZE(B) movsd 0 * SIZE(B), %xmm0 @@ -2076,10 +2076,10 @@ movsd %xmm7, -1 * SIZE(BO) decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2088,10 +2088,10 @@ #endif andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movsd 0 * SIZE(B), %xmm0 movsd 1 * SIZE(B), %xmm1 @@ -2103,10 +2103,10 @@ addq $2 * SIZE, B addq $4 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): #if defined(LT) || defined(RN) movq A, AO #else @@ -2125,10 +2125,10 @@ #endif testq $1, M - je .L60 + je L(60) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -2169,10 +2169,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulsd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulsd 2 * SIZE(BO), %xmm8 @@ -2233,10 +2233,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2245,10 +2245,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulsd %xmm8, %xmm9 mulsd 2 * SIZE(BO), %xmm8 addsd %xmm9, %xmm0 @@ -2259,10 +2259,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addsd %xmm2, %xmm0 addsd %xmm3, %xmm1 @@ -2381,12 +2381,12 @@ #endif ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -2427,10 +2427,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(BO), %xmm8 @@ -2492,10 +2492,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2504,10 +2504,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_4 -.L66: +L(66): mulpd %xmm8, %xmm9 mulpd 2 * SIZE(BO), %xmm8 addpd %xmm9, %xmm0 @@ -2518,10 +2518,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L69: +L(69): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -2690,13 +2690,13 @@ #endif ALIGN_4 -.L70: +L(70): movq M, I sarq $2, I # i = (m >> 2) - jle .L79 + jle L(79) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2747,10 +2747,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(BO), %xmm8 @@ -2862,10 +2862,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2874,10 +2874,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 @@ -2894,10 +2894,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3162,10 +3162,10 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -3187,12 +3187,12 @@ #endif ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L81: +L(81): /* Copying to Sub Buffer */ #ifdef LN @@ -3229,10 +3229,10 @@ subq KK, %rax #endif sarq $3, %rax - jle .L83 + jle L(83) ALIGN_4 -.L82: +L(82): PREFETCH 56 * SIZE(B) movsd 0 * SIZE(B), %xmm0 @@ -3265,10 +3265,10 @@ movsd %xmm7, -1 * SIZE(BO) decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L83: +L(83): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3277,10 +3277,10 @@ #endif andq $7, %rax BRANCH - jle .L90 + jle L(90) ALIGN_4 -.L84: +L(84): movsd 0 * SIZE(B), %xmm0 movsd %xmm0, 0 * SIZE(BO) @@ -3289,10 +3289,10 @@ addq $1 * SIZE, B addq $2 * SIZE, BO decq %rax - jne .L84 + jne L(84) ALIGN_4 -.L90: +L(90): #if defined(LT) || defined(RN) movq A, AO #else @@ -3309,10 +3309,10 @@ #endif testq $1, M - je .L100 + je L(100) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -3350,10 +3350,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulsd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd 1 * SIZE(AO), %xmm8 @@ -3385,10 +3385,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3397,10 +3397,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd %xmm8, %xmm9 movsd 1 * SIZE(AO), %xmm8 addsd %xmm9, %xmm0 @@ -3409,10 +3409,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addsd %xmm2, %xmm0 addsd %xmm3, %xmm1 addsd %xmm1, %xmm0 @@ -3513,12 +3513,12 @@ #endif ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -3556,10 +3556,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L105 + je L(105) ALIGN_4 -.L102: +L(102): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd 2 * SIZE(AO), %xmm8 @@ -3592,10 +3592,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3604,10 +3604,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_4 -.L106: +L(106): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(AO), %xmm8 @@ -3616,10 +3616,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L106 + jg L(106) ALIGN_4 -.L109: +L(109): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -3750,13 +3750,13 @@ #endif ALIGN_4 -.L110: +L(110): movq M, I sarq $2, I # i = (m >> 2) - jle .L119 + jle L(119) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -3803,10 +3803,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(AO), %xmm9 @@ -3863,10 +3863,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3875,10 +3875,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_4 -.L96: +L(96): mulpd %xmm9, %xmm8 mulpd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -3889,10 +3889,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L99: +L(99): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -4094,10 +4094,10 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L119: +L(119): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -4119,7 +4119,7 @@ #endif ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/trsm_kernel_LN_4x4_sse3.S b/kernel/x86_64/trsm_kernel_LN_4x4_sse3.S index fedeb5a580..aeb2b482c5 100644 --- a/kernel/x86_64/trsm_kernel_LN_4x4_sse3.S +++ b/kernel/x86_64/trsm_kernel_LN_4x4_sse3.S @@ -400,10 +400,10 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -441,10 +441,10 @@ #endif testq $1, M - je .L30 + je L(30) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -478,10 +478,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -543,10 +543,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -555,10 +555,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -570,10 +570,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax @@ -758,13 +758,13 @@ #endif ALIGN_4 -.L30: +L(30): testq $2, M BRANCH - je .L20 + je L(20) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax @@ -798,10 +798,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -911,10 +911,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -923,10 +923,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -944,10 +944,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1161,13 +1161,13 @@ #endif ALIGN_4 -.L20: +L(20): movq M, I sarq $2, I # i = (m >> 2) - jle .L39 + jle L(39) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax @@ -1232,8 +1232,8 @@ #if 1 andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1 (16 * 0) KERNEL2 (16 * 0) KERNEL3 (16 * 0) @@ -1252,7 +1252,7 @@ KERNEL16(16 * 0) cmpq $128 * 1, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 1) KERNEL2 (16 * 1) KERNEL3 (16 * 1) @@ -1271,7 +1271,7 @@ KERNEL16(16 * 1) cmpq $128 * 2, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 2) KERNEL2 (16 * 2) KERNEL3 (16 * 2) @@ -1290,7 +1290,7 @@ KERNEL16(16 * 2) cmpq $128 * 3, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 3) KERNEL2 (16 * 3) KERNEL3 (16 * 3) @@ -1309,7 +1309,7 @@ KERNEL16(16 * 3) cmpq $128 * 4, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 4) KERNEL2 (16 * 4) KERNEL3 (16 * 4) @@ -1328,7 +1328,7 @@ KERNEL16(16 * 4) cmpq $128 * 5, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 5) KERNEL2 (16 * 5) KERNEL3 (16 * 5) @@ -1347,7 +1347,7 @@ KERNEL16(16 * 5) cmpq $128 * 6, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 6) KERNEL2 (16 * 6) KERNEL3 (16 * 6) @@ -1366,7 +1366,7 @@ KERNEL16(16 * 6) cmpq $128 * 7, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 7) KERNEL2 (16 * 7) KERNEL3 (16 * 7) @@ -1387,18 +1387,18 @@ addq $32 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $128 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 2), BO # * 64 #else sarq $3, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1625,11 +1625,11 @@ addq $32 * SIZE, AO decq %rax BRANCH - jne .L12 + jne L(12) #endif ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1638,10 +1638,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 addpd %xmm9, %xmm0 @@ -1672,10 +1672,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax @@ -2081,11 +2081,11 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -2103,12 +2103,12 @@ #endif decq J # j -- - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $2, N - je .L80 + je L(80) ALIGN_4 #if defined(LT) || defined(RN) @@ -2149,10 +2149,10 @@ testq $1, M - je .L70 + je L(70) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -2185,10 +2185,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movddup 1 * SIZE(AO), %xmm8 @@ -2220,10 +2220,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2232,10 +2232,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulpd %xmm8, %xmm9 movddup 1 * SIZE(AO), %xmm8 addpd %xmm9, %xmm0 @@ -2244,10 +2244,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -2364,12 +2364,12 @@ #endif ALIGN_4 -.L70: +L(70): testq $2, M - je .L60 + je L(60) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -2402,10 +2402,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -2467,10 +2467,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2479,10 +2479,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_4 -.L66: +L(66): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -2494,10 +2494,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L69: +L(69): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -2633,13 +2633,13 @@ #endif ALIGN_4 -.L60: +L(60): movq M, I sarq $2, I # i = (m >> 2) - jle .L79 + jle L(79) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2683,10 +2683,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -2805,10 +2805,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2817,10 +2817,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 addpd %xmm9, %xmm0 @@ -2839,10 +2839,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3059,11 +3059,11 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -3082,9 +3082,9 @@ #endif ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 #if defined(LT) || defined(RN) @@ -3118,10 +3118,10 @@ #endif testq $1, M - je .L110 + je L(110) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -3154,10 +3154,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulpd %xmm9, %xmm8 movapd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -3176,10 +3176,10 @@ addq $8 * SIZE, AO addq $8 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3188,10 +3188,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd 0 * SIZE(BO), %xmm9 addsd %xmm9, %xmm0 movsd 1 * SIZE(AO), %xmm9 @@ -3199,10 +3199,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addpd %xmm1, %xmm0 haddpd %xmm0, %xmm0 @@ -3291,12 +3291,12 @@ #endif ALIGN_4 -.L110: +L(110): testq $2, M - je .L100 + je L(100) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -3329,10 +3329,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L105 + je L(105) ALIGN_4 -.L102: +L(102): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movddup 1 * SIZE(BO), %xmm9 @@ -3364,10 +3364,10 @@ addq $16 * SIZE, AO addq $ 8 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3376,10 +3376,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_4 -.L106: +L(106): mulpd %xmm9, %xmm8 movddup 1 * SIZE(BO), %xmm9 addpd %xmm8, %xmm0 @@ -3388,10 +3388,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L106 + jg L(106) ALIGN_4 -.L109: +L(109): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -3507,13 +3507,13 @@ #endif ALIGN_4 -.L100: +L(100): movq M, I sarq $2, I # i = (m >> 2) - jle .L119 + jle L(119) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -3552,10 +3552,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(AO), %xmm9 @@ -3610,10 +3610,10 @@ addq $32 * SIZE, AO addq $8 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3622,10 +3622,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_4 -.L96: +L(96): mulpd %xmm9, %xmm8 mulpd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -3636,10 +3636,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L99: +L(99): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -3822,10 +3822,10 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L119: +L(119): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -3844,7 +3844,7 @@ #endif ALIGN_2 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LN_4x8_nehalem.S b/kernel/x86_64/trsm_kernel_LN_4x8_nehalem.S index 8717fc3368..4f55ba0165 100644 --- a/kernel/x86_64/trsm_kernel_LN_4x8_nehalem.S +++ b/kernel/x86_64/trsm_kernel_LN_4x8_nehalem.S @@ -166,10 +166,10 @@ movq N, J sarq $3, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -208,7 +208,7 @@ testq $1, M BRANCH - jle .L20 + jle L(20) #ifdef LN movq K, %rax @@ -240,10 +240,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movaps -32 * SIZE(BO), %xmm2 @@ -283,10 +283,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -295,10 +295,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -313,10 +313,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -715,10 +715,10 @@ #endif ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) #ifdef LN movq K, %rax @@ -756,10 +756,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -837,10 +837,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -849,10 +849,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -876,10 +876,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1329,14 +1329,14 @@ #endif ALIGN_4 -.L30: +L(30): movq M, I sarq $2, I NOBRANCH - jle .L39 + jle L(39) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1390,10 +1390,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm12 @@ -1520,10 +1520,10 @@ subq $-16 * SIZE, AO subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1532,10 +1532,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addps %xmm1, %xmm12 movaps -32 * SIZE(BO), %xmm1 addps %xmm2, %xmm13 @@ -1571,10 +1571,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2194,10 +2194,10 @@ decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 8), B @@ -2216,12 +2216,12 @@ subq $1, J BRANCH - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $4, N - jle .L70 + jle L(70) #if defined(LT) || defined(RN) movq A, AO @@ -2257,7 +2257,7 @@ testq $1, M BRANCH - jle .L50 + jle L(50) #ifdef LN movq K, %rax @@ -2288,10 +2288,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movaps -32 * SIZE(BO), %xmm2 @@ -2319,11 +2319,11 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) addps %xmm9, %xmm8 ALIGN_3 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2332,10 +2332,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -2347,10 +2347,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2524,10 +2524,10 @@ #endif ALIGN_4 -.L50: +L(50): testq $2, M BRANCH - jle .L60 + jle L(60) #ifdef LN movq K, %rax @@ -2560,10 +2560,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2605,10 +2605,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2617,10 +2617,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2635,10 +2635,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2842,14 +2842,14 @@ #endif ALIGN_4 -.L60: +L(60): movq M, I sarq $2, I NOBRANCH - jle .L69 + jle L(69) ALIGN_4 -.L41: +L(41): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2889,10 +2889,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -2959,10 +2959,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2971,10 +2971,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): addps %xmm1, %xmm8 movaps -32 * SIZE(BO), %xmm1 addps %xmm2, %xmm9 @@ -2995,10 +2995,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3300,10 +3300,10 @@ decq I BRANCH - jg .L41 + jg L(41) ALIGN_4 -.L69: +L(69): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -3321,9 +3321,9 @@ #endif ALIGN_4 -.L70: +L(70): testq $2, N - jle .L100 + jle L(100) #if defined(LT) || defined(RN) movq A, AO @@ -3359,7 +3359,7 @@ testq $1, M BRANCH - jle .L80 + jle L(80) #ifdef LN movq K, %rax @@ -3390,10 +3390,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_3 -.L92: +L(92): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movsd -32 * SIZE(BO), %xmm2 @@ -3421,11 +3421,11 @@ subq $1, %rax BRANCH - jg .L92 + jg L(92) addps %xmm9, %xmm8 ALIGN_3 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3434,10 +3434,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_3 -.L96: +L(96): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -3449,10 +3449,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_3 -.L98: +L(98): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3558,10 +3558,10 @@ #endif ALIGN_4 -.L80: +L(80): testq $2, M BRANCH - jle .L90 + jle L(90) #ifdef LN movq K, %rax @@ -3594,10 +3594,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): addps %xmm1, %xmm8 movsd -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -3627,10 +3627,10 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) ALIGN_3 -.L85: +L(85): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3639,10 +3639,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): addps %xmm1, %xmm8 movsd -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -3654,10 +3654,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_3 -.L88: +L(88): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3784,14 +3784,14 @@ #endif ALIGN_4 -.L90: +L(90): movq M, I sarq $2, I NOBRANCH - jle .L99 + jle L(99) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -3826,10 +3826,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -3872,10 +3872,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_3 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3884,10 +3884,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addps %xmm1, %xmm8 pshufd $0x00, %xmm3, %xmm1 mulps %xmm0, %xmm1 @@ -3902,10 +3902,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_3 -.L78: +L(78): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -4109,10 +4109,10 @@ decq I BRANCH - jg .L71 + jg L(71) ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -4130,9 +4130,9 @@ #endif ALIGN_4 -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) #if defined(LT) || defined(RN) movq A, AO @@ -4166,7 +4166,7 @@ testq $1, M BRANCH - jle .L110 + jle L(110) #ifdef LN movq K, %rax @@ -4196,10 +4196,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L125 + jle L(125) ALIGN_3 -.L122: +L(122): addss %xmm2, %xmm8 movss -32 * SIZE(BO), %xmm2 mulss %xmm0, %xmm2 @@ -4225,10 +4225,10 @@ subq $1, %rax BRANCH - jg .L122 + jg L(122) ALIGN_3 -.L125: +L(125): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4237,10 +4237,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_3 -.L126: +L(126): addss %xmm2, %xmm8 movss -32 * SIZE(BO), %xmm2 mulss %xmm0, %xmm2 @@ -4251,10 +4251,10 @@ subq $1, %rax BRANCH - jg .L126 + jg L(126) ALIGN_3 -.L128: +L(128): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -4327,10 +4327,10 @@ #endif ALIGN_4 -.L110: +L(110): testq $2, M BRANCH - jle .L120 + jle L(120) #ifdef LN movq K, %rax @@ -4360,10 +4360,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_3 -.L112: +L(112): addps %xmm1, %xmm8 movss -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -4393,10 +4393,10 @@ subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_3 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4405,10 +4405,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_3 -.L116: +L(116): addps %xmm1, %xmm8 movss -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -4420,10 +4420,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_3 -.L118: +L(118): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -4527,14 +4527,14 @@ #endif ALIGN_4 -.L120: +L(120): movq M, I sarq $2, I NOBRANCH - jle .L129 + jle L(129) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -4565,10 +4565,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_3 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -4599,10 +4599,10 @@ subq $ -4 * SIZE, BO subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_3 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4611,10 +4611,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L108 + je L(108) ALIGN_3 -.L106: +L(106): addps %xmm1, %xmm8 pshufd $0x00, %xmm3, %xmm1 movss -31 * SIZE(BO), %xmm3 @@ -4626,10 +4626,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_3 -.L108: +L(108): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -4797,10 +4797,10 @@ decq I BRANCH - jg .L101 + jg L(101) ALIGN_4 -.L129: +L(129): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -4818,7 +4818,7 @@ #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LN_8x4_sse.S b/kernel/x86_64/trsm_kernel_LN_8x4_sse.S index a318747632..aeae331398 100644 --- a/kernel/x86_64/trsm_kernel_LN_8x4_sse.S +++ b/kernel/x86_64/trsm_kernel_LN_8x4_sse.S @@ -189,9 +189,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L50 + jle L(50) -.L01: +L(01): /* Copying to Sub Buffer */ #ifdef LN @@ -228,10 +228,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 movaps 8 * SIZE(B), %xmm11 @@ -279,10 +279,10 @@ addq $64 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -291,10 +291,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movaps 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -310,10 +310,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -332,7 +332,7 @@ #endif testq $1, M - je .L20 + je L(20) #ifdef LN movq K, %rax @@ -374,10 +374,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): mulss %xmm8, %xmm9 addss %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -496,10 +496,10 @@ addq $ 8 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -508,10 +508,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -529,10 +529,10 @@ addq $ 1 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -733,9 +733,9 @@ #endif ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) #ifdef LN movq K, %rax @@ -778,10 +778,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -900,10 +900,10 @@ addq $ 16 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -912,10 +912,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -933,10 +933,10 @@ addq $ 2 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1181,9 +1181,9 @@ #endif ALIGN_4 -.L30: +L(30): testq $4, M - je .L40 + je L(40) #ifdef LN movq K, %rax @@ -1226,10 +1226,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -1343,10 +1343,10 @@ addq $ 32 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1355,10 +1355,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1375,10 +1375,10 @@ addq $ 4 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1702,13 +1702,13 @@ #endif ALIGN_4 -.L40: +L(40): movq M, I sarq $3, I # i = (m >> 3) - jle .L49 + jle L(49) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $3 + BASE_SHIFT, %rax @@ -1760,10 +1760,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 0 * SIZE(BO), %xmm9 @@ -1864,10 +1864,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1876,9 +1876,9 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1907,10 +1907,10 @@ addq $8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2573,10 +2573,10 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L49: +L(49): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -2598,11 +2598,11 @@ #endif decq J # j -- - jg .L01 + jg L(01) -.L50: +L(50): testq $2, N - je .L100 + je L(100) #ifdef LN movq OFFSET, %rax @@ -2638,10 +2638,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 @@ -2668,10 +2668,10 @@ addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2680,10 +2680,10 @@ #endif andq $3, %rax BRANCH - jle .L60 + jle L(60) ALIGN_4 -.L54: +L(54): movsd 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -2695,10 +2695,10 @@ addq $2 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L54 + jne L(54) ALIGN_4 -.L60: +L(60): #if defined(LT) || defined(RN) movq A, AO #else @@ -2716,7 +2716,7 @@ #endif testq $1, M - je .L70 + je L(70) #ifdef LN movq K, %rax @@ -2758,10 +2758,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -2832,10 +2832,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2844,10 +2844,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -2859,10 +2859,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): addss %xmm2, %xmm0 addss %xmm3, %xmm1 @@ -2989,9 +2989,9 @@ #endif ALIGN_4 -.L70: +L(70): testq $2, M - je .L80 + je L(80) #ifdef LN movq K, %rax @@ -3034,10 +3034,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L85 + je L(85) ALIGN_4 -.L82: +L(82): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -3108,10 +3108,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L85: +L(85): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3120,10 +3120,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -3135,10 +3135,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -3308,9 +3308,9 @@ #endif ALIGN_4 -.L80: +L(80): testq $4, M - je .L90 + je L(90) #ifdef LN movq K, %rax @@ -3353,10 +3353,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3420,10 +3420,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3432,10 +3432,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -3446,10 +3446,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -3697,13 +3697,13 @@ #endif ALIGN_4 -.L90: +L(90): movq M, I sarq $3, I # i = (m >> 3) - jle .L99 + jle L(99) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $3 + BASE_SHIFT, %rax @@ -3750,10 +3750,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3873,10 +3873,10 @@ addq $64 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3885,10 +3885,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -3905,10 +3905,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -4426,10 +4426,10 @@ #endif decq I # i -- - jg .L61 + jg L(61) ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -4451,9 +4451,9 @@ #endif ALIGN_4 -.L100: +L(100): testq $1, N - je .L999 + je L(999) #ifdef LN movq OFFSET, %rax @@ -4489,10 +4489,10 @@ subq KK, %rax #endif sarq $3, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movsd 0 * SIZE(B), %xmm3 movhps 2 * SIZE(B), %xmm3 movsd 4 * SIZE(B), %xmm7 @@ -4521,10 +4521,10 @@ addq $32 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4533,10 +4533,10 @@ #endif andq $7, %rax BRANCH - jle .L110 + jle L(110) ALIGN_4 -.L104: +L(104): movss 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -4546,10 +4546,10 @@ addq $ 1 * SIZE, B addq $ 4 * SIZE, BO decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L110: +L(110): #if defined(LT) || defined(RN) movq A, AO #else @@ -4566,7 +4566,7 @@ #endif testq $1, M - je .L120 + je L(120) #ifdef LN movq K, %rax @@ -4606,10 +4606,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L145 + je L(145) ALIGN_4 -.L142: +L(142): mulss %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -4643,10 +4643,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L142 + jne L(142) ALIGN_4 -.L145: +L(145): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4655,10 +4655,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): mulss %xmm8, %xmm9 movss 1 * SIZE(AO), %xmm8 addss %xmm9, %xmm0 @@ -4667,10 +4667,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): addss %xmm1, %xmm0 addss %xmm3, %xmm2 addss %xmm2, %xmm0 @@ -4755,9 +4755,9 @@ #endif ALIGN_4 -.L120: +L(120): testq $2, M - je .L130 + je L(130) #ifdef LN movq K, %rax @@ -4798,10 +4798,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L135 + je L(135) ALIGN_4 -.L132: +L(132): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -4847,10 +4847,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L132 + jne L(132) ALIGN_4 -.L135: +L(135): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4859,10 +4859,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): mulps %xmm8, %xmm9 movsd 2 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -4871,10 +4871,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): addps %xmm1, %xmm0 #if defined(LN) || defined(RT) @@ -5003,9 +5003,9 @@ #endif ALIGN_4 -.L130: +L(130): testq $4, M - je .L140 + je L(140) #ifdef LN movq K, %rax @@ -5046,10 +5046,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L125 + je L(125) ALIGN_4 -.L122: +L(122): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -5087,10 +5087,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L122 + jne L(122) ALIGN_4 -.L125: +L(125): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -5099,10 +5099,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): mulps %xmm8, %xmm9 movaps 4 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -5111,10 +5111,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): addps %xmm1, %xmm0 addps %xmm3, %xmm2 addps %xmm2, %xmm0 @@ -5313,13 +5313,13 @@ #endif ALIGN_4 -.L140: +L(140): movq M, I sarq $3, I # i = (m >> 3) - jle .L149 + jle L(149) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $3 + BASE_SHIFT, %rax @@ -5365,10 +5365,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulps %xmm9, %xmm8 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -5441,10 +5441,10 @@ addq $64 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -5453,10 +5453,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulps %xmm9, %xmm8 mulps 4 * SIZE(AO), %xmm9 addps %xmm8, %xmm0 @@ -5467,10 +5467,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -5894,10 +5894,10 @@ #endif decq I # i -- - jg .L111 + jg L(111) ALIGN_4 -.L149: +L(149): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -5919,7 +5919,7 @@ #endif ALIGN_4 -.L999: +L(999): movq %rbx, %rsp EMMS movq 0(%rsp), %rbx diff --git a/kernel/x86_64/trsm_kernel_LT_2x8_nehalem.S b/kernel/x86_64/trsm_kernel_LT_2x8_nehalem.S index 39ed586ce4..52987fb0f1 100644 --- a/kernel/x86_64/trsm_kernel_LT_2x8_nehalem.S +++ b/kernel/x86_64/trsm_kernel_LT_2x8_nehalem.S @@ -167,10 +167,10 @@ movq N, J sarq $3, J NOBRANCH - jle .L30 + jle L(30) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -210,10 +210,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -267,10 +267,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm12 @@ -393,10 +393,10 @@ subq $-8 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -405,10 +405,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addpd %xmm1, %xmm12 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm13 @@ -444,10 +444,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -897,13 +897,13 @@ decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M BRANCH - jle .L29 + jle L(29) ALIGN_4 #ifdef LN @@ -937,10 +937,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1002,10 +1002,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1014,10 +1014,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1037,10 +1037,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1378,7 +1378,7 @@ #endif ALIGN_4 -.L29: +L(29): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 8), B @@ -1397,12 +1397,12 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L30: +L(30): testq $4, N - jle .L50 + jle L(50) ALIGN_4 #if defined(LT) || defined(RN) @@ -1440,10 +1440,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -1483,10 +1483,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -1558,10 +1558,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1570,10 +1570,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): addpd %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm9 @@ -1595,10 +1595,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1826,13 +1826,13 @@ decq I BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L40: +L(40): testq $1, M BRANCH - jle .L49 + jle L(49) ALIGN_4 #ifdef LN @@ -1866,10 +1866,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1907,10 +1907,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1919,10 +1919,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1936,10 +1936,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2097,7 +2097,7 @@ #endif ALIGN_4 -.L49: +L(49): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -2115,9 +2115,9 @@ #endif ALIGN_4 -.L50: +L(50): testq $2, N - jle .L70 + jle L(70) ALIGN_4 #if defined(LT) || defined(RN) @@ -2155,10 +2155,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -2194,10 +2194,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -2237,13 +2237,13 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 ALIGN_3 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2252,10 +2252,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addpd %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm9 @@ -2269,10 +2269,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2407,13 +2407,13 @@ decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L69 + jle L(69) ALIGN_4 #ifdef LN @@ -2445,10 +2445,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): mulpd %xmm0, %xmm1 movddup -15 * SIZE(AO), %xmm0 addpd %xmm1, %xmm8 @@ -2474,10 +2474,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2486,10 +2486,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): mulpd %xmm0, %xmm1 movddup -15 * SIZE(AO), %xmm0 addpd %xmm1, %xmm8 @@ -2500,10 +2500,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2606,7 +2606,7 @@ #endif ALIGN_4 -.L69: +L(69): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2624,9 +2624,9 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, N - jle .L999 + jle L(999) ALIGN_4 #if defined(LT) || defined(RN) @@ -2662,10 +2662,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L80 + jle L(80) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -2700,10 +2700,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -2731,12 +2731,12 @@ subq $1, %rax BRANCH - jg .L72 + jg L(72) addpd %xmm9, %xmm8 ALIGN_3 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2745,10 +2745,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addpd %xmm1, %xmm8 movddup -16 * SIZE(BO), %xmm1 mulpd %xmm0, %xmm1 @@ -2759,10 +2759,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2863,13 +2863,13 @@ decq I BRANCH - jg .L71 + jg L(71) ALIGN_4 -.L80: +L(80): testq $1, M BRANCH - jle .L89 + jle L(89) ALIGN_4 #ifdef LN @@ -2903,10 +2903,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): mulpd %xmm0, %xmm1 movsd -14 * SIZE(AO), %xmm0 movhps -13 * SIZE(AO), %xmm0 @@ -2926,12 +2926,12 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) addpd %xmm9, %xmm8 ALIGN_3 -.L85: +L(85): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2940,10 +2940,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): mulsd %xmm0, %xmm1 movsd -15 * SIZE(AO), %xmm0 addsd %xmm1, %xmm8 @@ -2954,10 +2954,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -3029,7 +3029,7 @@ #endif ALIGN_4 -.L89: +L(89): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -3048,7 +3048,7 @@ ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LT_4x2_atom.S b/kernel/x86_64/trsm_kernel_LT_4x2_atom.S index 04b7e2de19..8d63d73568 100644 --- a/kernel/x86_64/trsm_kernel_LT_4x2_atom.S +++ b/kernel/x86_64/trsm_kernel_LT_4x2_atom.S @@ -156,10 +156,10 @@ movq N, J sarq $1, J - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -198,10 +198,10 @@ movq M, I sarq $2, I - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -250,10 +250,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): addsd %xmm2, %xmm13 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps %xmm0, %xmm2 @@ -396,10 +396,10 @@ mulsd %xmm3, %xmm6 movsd 1 * SIZE(BO), %xmm3 - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -408,10 +408,10 @@ #endif andq $3, %rax BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): addsd %xmm2, %xmm13 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -450,10 +450,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): addsd %xmm2, %xmm13 addsd %xmm7, %xmm14 addsd %xmm6, %xmm15 @@ -753,13 +753,13 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - je .L30 + je L(30) #ifdef LN movq K, %rax @@ -799,10 +799,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addsd %xmm2, %xmm9 movaps %xmm0, %xmm2 @@ -880,10 +880,10 @@ addq $8 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -892,10 +892,10 @@ #endif andq $3, %rax BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): addsd %xmm2, %xmm9 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -918,10 +918,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): addsd %xmm2, %xmm9 addsd %xmm6, %xmm11 @@ -1083,9 +1083,9 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #ifdef LN movq K, %rax @@ -1121,10 +1121,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): addsd %xmm5, %xmm8 movsd 2 * SIZE(BO), %xmm5 mulsd %xmm0, %xmm1 @@ -1166,10 +1166,10 @@ addq $8 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1182,10 +1182,10 @@ andq $3, %rax BRANCH BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulsd %xmm0, %xmm1 addq $2 * SIZE, BO mulsd %xmm0, %xmm3 @@ -1199,10 +1199,10 @@ addq $1 * SIZE, AO decq %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1301,7 +1301,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -1319,12 +1319,12 @@ #endif decq J # j -- - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $1, N - je .L999 + je L(999) ALIGN_4 #if defined(LT) || defined(RN) @@ -1359,10 +1359,10 @@ movq M, I sarq $2, I - jle .L50 + jle L(50) ALIGN_4 -.L41: +L(41): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1403,10 +1403,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): addsd %xmm9, %xmm8 movsd 4 * SIZE(AO), %xmm9 mulsd %xmm4, %xmm0 @@ -1480,10 +1480,10 @@ mulsd %xmm5, %xmm15 movsd 1 * SIZE(BO), %xmm5 - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1499,10 +1499,10 @@ andq $3, %rax BRANCH BRANCH - je .L49 + je L(49) ALIGN_4 -.L46: +L(46): mulsd %xmm4, %xmm0 mulsd %xmm4, %xmm1 mulsd %xmm4, %xmm2 @@ -1522,10 +1522,10 @@ addq $1 * SIZE, BO decq %rax BRANCH - jg .L46 + jg L(46) ALIGN_4 -.L49: +L(49): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1678,12 +1678,12 @@ #endif decq I # i -- - jg .L41 + jg L(41) ALIGN_4 -.L50: +L(50): testq $2, M - je .L60 + je L(60) #ifdef LN movq K, %rax @@ -1718,10 +1718,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): addsd %xmm2, %xmm8 movsd 2 * SIZE(AO), %xmm2 mulsd %xmm4, %xmm0 @@ -1762,10 +1762,10 @@ mulsd %xmm5, %xmm3 movsd 1 * SIZE(BO), %xmm5 - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1777,10 +1777,10 @@ andq $3, %rax BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): mulsd %xmm4, %xmm0 mulsd %xmm4, %xmm1 movsd 1 * SIZE(BO), %xmm4 @@ -1794,10 +1794,10 @@ addq $1 * SIZE, BO decq %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1894,9 +1894,9 @@ #endif ALIGN_4 -.L60: +L(60): testq $1, M - je .L69 + je L(69) #ifdef LN movq K, %rax @@ -1933,10 +1933,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): addsd %xmm5, %xmm8 movsd 2 * SIZE(BO), %xmm5 mulsd %xmm0, %xmm1 @@ -1961,13 +1961,13 @@ addq $4 * SIZE, BO decq %rax - jne .L62 + jne L(62) addsd %xmm5, %xmm8 addsd %xmm7, %xmm9 ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1976,10 +1976,10 @@ #endif andq $3, %rax BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): movsd 0 * SIZE(AO), %xmm0 movsd 0 * SIZE(BO), %xmm1 @@ -1991,10 +1991,10 @@ decq %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addsd %xmm9, %xmm8 #if defined(LN) || defined(RT) @@ -2068,7 +2068,7 @@ #endif ALIGN_4 -.L69: +L(69): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -2087,7 +2087,7 @@ #endif ALIGN_2 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LT_4x4_barcelona.S b/kernel/x86_64/trsm_kernel_LT_4x4_barcelona.S index b3712004c3..e85dddf79b 100644 --- a/kernel/x86_64/trsm_kernel_LT_4x4_barcelona.S +++ b/kernel/x86_64/trsm_kernel_LT_4x4_barcelona.S @@ -480,9 +480,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -521,10 +521,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -593,10 +593,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -606,10 +606,10 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) BRANCH - jl .L12 + jl L(12) ALIGN_4 -.L15: +L(15): prefetch -8 * SIZE(BB) subq $-16 * SIZE, BB @@ -620,7 +620,7 @@ subq KK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_4 @@ -633,7 +633,7 @@ subq $-16 * SIZE, AO ALIGN_4 -.L16: +L(16): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -641,7 +641,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L19 + je L(19) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -649,7 +649,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -676,10 +676,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L17 + jl L(17) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1086,18 +1086,18 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $3, M - je .L39 + je L(39) testq $2, M - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -1141,10 +1141,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L26 + je L(26) ALIGN_4 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -1200,10 +1200,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L22 + jl L(22) ALIGN_4 -.L26: +L(26): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1211,7 +1211,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L29 + je L(29) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -1219,7 +1219,7 @@ negq %rax ALIGN_4 -.L27: +L(27): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -1235,10 +1235,10 @@ movddup -11 * SIZE(BO, %rax, 4), %xmm5 addq $SIZE, %rax - jl .L27 + jl L(27) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1456,9 +1456,9 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #ifdef LN movq K, %rax @@ -1503,10 +1503,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L36 + je L(36) ALIGN_4 -.L32: +L(32): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO, %rax, 4), %xmm0 addpd %xmm1, %xmm8 @@ -1534,10 +1534,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L32 + jl L(32) ALIGN_4 -.L36: +L(36): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1545,7 +1545,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L38 + je L(38) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -1553,7 +1553,7 @@ negq %rax ALIGN_4 -.L37: +L(37): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO, %rax, 4), %xmm0 addpd %xmm1, %xmm8 @@ -1562,10 +1562,10 @@ movddup -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L37 + jl L(37) ALIGN_4 -.L38: +L(38): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1746,7 +1746,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1765,12 +1765,12 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $2, N - je .L80 + je L(80) #if defined(LT) || defined(RN) movq A, AO @@ -1806,10 +1806,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1864,10 +1864,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L56 + je L(56) ALIGN_4 -.L52: +L(52): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -1919,10 +1919,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L52 + jl L(52) ALIGN_4 -.L56: +L(56): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1930,7 +1930,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L59 + je L(59) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -1938,7 +1938,7 @@ negq %rax ALIGN_4 -.L57: +L(57): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -1953,10 +1953,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L57 + jl L(57) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2169,12 +2169,12 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) #ifdef LN movq K, %rax @@ -2218,10 +2218,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L66 + je L(66) ALIGN_4 -.L62: +L(62): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -2253,10 +2253,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L62 + jl L(62) ALIGN_4 -.L66: +L(66): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2264,7 +2264,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L69 + je L(69) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -2272,7 +2272,7 @@ negq %rax ALIGN_4 -.L67: +L(67): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -2282,10 +2282,10 @@ movddup -13 * SIZE(BO, %rax, 2), %xmm3 addq $SIZE, %rax - jl .L67 + jl L(67) ALIGN_4 -.L69: +L(69): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -2426,12 +2426,12 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -2474,10 +2474,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L76 + je L(76) ALIGN_4 -.L72: +L(72): mulpd -16 * SIZE(BO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -12 * SIZE(AO, %rax, 1), %xmm0 @@ -2496,10 +2496,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L72 + jl L(72) ALIGN_4 -.L76: +L(76): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2507,7 +2507,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L78 + je L(78) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -2515,16 +2515,16 @@ negq %rax ALIGN_4 -.L77: +L(77): mulpd -16 * SIZE(BO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L77 + jl L(77) ALIGN_4 -.L78: +L(78): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -2627,7 +2627,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2646,9 +2646,9 @@ #endif ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) #if defined(LT) || defined(RN) movq A, AO @@ -2682,10 +2682,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2734,10 +2734,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L96 + je L(96) ALIGN_4 -.L92: +L(92): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -2765,10 +2765,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L92 + jl L(92) ALIGN_4 -.L96: +L(96): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2776,7 +2776,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L99 + je L(99) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -2784,7 +2784,7 @@ negq %rax ALIGN_4 -.L97: +L(97): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -2793,9 +2793,9 @@ movddup -15 * SIZE(BO, %rax, 1), %xmm1 addq $SIZE, %rax - jl .L97 + jl L(97) ALIGN_4 -.L99: +L(99): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -2972,12 +2972,12 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) #ifdef LN movq K, %rax @@ -3020,10 +3020,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L106 + je L(106) ALIGN_4 -.L102: +L(102): mulpd -16 * SIZE(AO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -12 * SIZE(BO, %rax, 1), %xmm0 @@ -3042,10 +3042,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L102 + jl L(102) ALIGN_4 -.L106: +L(106): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3053,7 +3053,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L109 + je L(109) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -3061,16 +3061,16 @@ negq %rax ALIGN_4 -.L107: +L(107): movddup -16 * SIZE(BO, %rax, 1), %xmm0 mulpd -16 * SIZE(AO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 addq $SIZE, %rax - jl .L107 + jl L(107) ALIGN_4 -.L109: +L(109): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -3187,12 +3187,12 @@ #endif ALIGN_4 -.L110: +L(110): testq $1, M - je .L119 + je L(119) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -3230,10 +3230,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L116 + je L(116) ALIGN_4 -.L112: +L(112): mulpd -16 * SIZE(BO, %rax, 1), %xmm0 addpd %xmm0, %xmm8 movapd -12 * SIZE(AO, %rax, 1), %xmm0 @@ -3244,10 +3244,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L112 + jl L(112) ALIGN_4 -.L116: +L(116): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3255,7 +3255,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L118 + je L(118) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -3263,16 +3263,16 @@ negq %rax ALIGN_4 -.L117: +L(117): mulsd -16 * SIZE(BO, %rax, 1), %xmm0 addsd %xmm0, %xmm8 movsd -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L117 + jl L(117) ALIGN_4 -.L118: +L(118): addpd %xmm9, %xmm8 haddpd %xmm8, %xmm8 @@ -3348,7 +3348,7 @@ #endif ALIGN_4 -.L119: +L(119): #ifdef LN leaq (B, K, SIZE), B #endif @@ -3367,7 +3367,7 @@ ALIGN_4 -.L999: +L(999): movq (%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LT_4x4_core2.S b/kernel/x86_64/trsm_kernel_LT_4x4_core2.S index c5fdc52827..de96086af1 100644 --- a/kernel/x86_64/trsm_kernel_LT_4x4_core2.S +++ b/kernel/x86_64/trsm_kernel_LT_4x4_core2.S @@ -171,9 +171,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) -.L01: +L(01): /* Copying to Sub Buffer */ #ifdef LN @@ -210,10 +210,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): prefetcht0 (PREFETCH_R + 0) * SIZE(B) movapd -16 * SIZE(B), %xmm0 movapd -14 * SIZE(B), %xmm1 @@ -269,10 +269,10 @@ subq $-16 * SIZE, B subq $-32 * SIZE, BO subq $1, %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -281,10 +281,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movapd -16 * SIZE(B), %xmm0 movapd -14 * SIZE(B), %xmm1 @@ -301,10 +301,10 @@ addq $4 * SIZE, B addq $8 * SIZE, BO subq $1, %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): leaq (PREFETCH_R + 0) * SIZE(B), BB #if defined(LT) || defined(RN) @@ -326,10 +326,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -381,10 +381,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm2, %xmm10 @@ -504,10 +504,10 @@ mulpd %xmm1, %xmm5 BRANCH - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -516,10 +516,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): movapd -16 * SIZE(AO), %xmm0 addpd %xmm2, %xmm10 movapd -16 * SIZE(BO), %xmm2 @@ -552,10 +552,10 @@ addq $8 * SIZE, BO # boffset1 += 8 subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): addpd %xmm2, %xmm10 addpd %xmm3, %xmm14 addpd %xmm4, %xmm11 @@ -1008,18 +1008,18 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $3, M - je .L39 + je L(39) testq $2, M - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -1053,10 +1053,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1126,10 +1126,10 @@ subq $ -8 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1138,10 +1138,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -1161,10 +1161,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jne .L26 + jne L(26) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1407,12 +1407,12 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1446,10 +1446,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -16 * SIZE(AO), %xmm0 @@ -1519,10 +1519,10 @@ subq $ -4 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1531,10 +1531,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): movsd -16 * SIZE(AO), %xmm0 movsd -16 * SIZE(BO), %xmm2 movsd -14 * SIZE(BO), %xmm3 @@ -1554,10 +1554,10 @@ addq $1 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1733,7 +1733,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1755,18 +1755,18 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $3, N - je .L999 + je L(999) testq $2, N - je .L80 + je L(80) ALIGN_4 -.L41: +L(41): /* Copying to Sub Buffer */ #ifdef LN @@ -1803,10 +1803,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): movddup -16 * SIZE(B), %xmm0 movddup -15 * SIZE(B), %xmm1 movddup -14 * SIZE(B), %xmm2 @@ -1828,10 +1828,10 @@ addq $8 * SIZE, B addq $16 * SIZE, BO subq $1, %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1840,10 +1840,10 @@ #endif andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movddup -16 * SIZE(B), %xmm0 movddup -15 * SIZE(B), %xmm1 @@ -1853,10 +1853,10 @@ addq $2 * SIZE, B addq $4 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): #if defined(LT) || defined(RN) movq A, AO #else @@ -1876,10 +1876,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1923,10 +1923,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -2004,10 +2004,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2016,10 +2016,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): movapd -16 * SIZE(AO), %xmm0 movapd -14 * SIZE(AO), %xmm1 @@ -2041,10 +2041,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2282,15 +2282,15 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -2324,10 +2324,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -2367,10 +2367,10 @@ subq $ -8 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2379,10 +2379,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_4 -.L66: +L(66): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -2396,10 +2396,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L66 + jg L(66) ALIGN_4 -.L69: +L(69): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -2557,12 +2557,12 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -2596,10 +2596,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -16 * SIZE(AO), %xmm0 @@ -2639,10 +2639,10 @@ subq $ -4 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2651,10 +2651,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): movsd -16 * SIZE(AO), %xmm0 movsd -16 * SIZE(BO), %xmm2 movsd -14 * SIZE(BO), %xmm3 @@ -2668,10 +2668,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addsd %xmm10, %xmm8 addsd %xmm11, %xmm9 @@ -2790,7 +2790,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2812,12 +2812,12 @@ #endif ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L81: +L(81): /* Copying to Sub Buffer */ #ifdef LN @@ -2854,10 +2854,10 @@ subq KK, %rax #endif sarq $3, %rax - jle .L83 + jle L(83) ALIGN_4 -.L82: +L(82): movddup -16 * SIZE(B), %xmm0 movddup -15 * SIZE(B), %xmm1 movddup -14 * SIZE(B), %xmm2 @@ -2879,10 +2879,10 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO subq $1, %rax - jne .L82 + jne L(82) ALIGN_4 -.L83: +L(83): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2891,10 +2891,10 @@ #endif andq $7, %rax BRANCH - jle .L90 + jle L(90) ALIGN_4 -.L84: +L(84): movddup -16 * SIZE(B), %xmm0 movapd %xmm0, 0 * SIZE(BO) @@ -2902,10 +2902,10 @@ addq $1 * SIZE, B addq $2 * SIZE, BO subq $1, %rax - jne .L84 + jne L(84) ALIGN_4 -.L90: +L(90): #if defined(LT) || defined(RN) movq A, AO #else @@ -2923,10 +2923,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2965,10 +2965,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -3014,10 +3014,10 @@ subq $-16 * SIZE, AO subq $ -8 * SIZE, BO subq $1, %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3026,10 +3026,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_4 -.L96: +L(96): movapd -16 * SIZE(AO), %xmm0 movapd -14 * SIZE(AO), %xmm1 @@ -3044,10 +3044,10 @@ addq $4 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L96 + jg L(96) ALIGN_4 -.L99: +L(99): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -3248,15 +3248,15 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -3290,10 +3290,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L105 + je L(105) ALIGN_4 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -3321,10 +3321,10 @@ subq $-8 * SIZE, AO subq $-8 * SIZE, BO subq $1, %rax - jne .L102 + jne L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3333,10 +3333,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_4 -.L106: +L(106): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 @@ -3346,10 +3346,10 @@ addq $2 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L106 + jg L(106) ALIGN_4 -.L109: +L(109): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -3479,12 +3479,12 @@ #endif ALIGN_4 -.L110: +L(110): testq $1, M - je .L119 + je L(119) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -3518,10 +3518,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -16 * SIZE(AO), %xmm0 @@ -3547,10 +3547,10 @@ subq $-4 * SIZE, AO subq $-8 * SIZE, BO subq $1, %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3559,10 +3559,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): movsd -16 * SIZE(AO), %xmm0 movsd -16 * SIZE(BO), %xmm2 @@ -3572,10 +3572,10 @@ addq $1 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addsd %xmm10, %xmm8 addsd %xmm11, %xmm9 addsd %xmm9, %xmm8 @@ -3676,7 +3676,7 @@ #endif ALIGN_4 -.L119: +L(119): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -3699,7 +3699,7 @@ ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/trsm_kernel_LT_4x4_penryn.S b/kernel/x86_64/trsm_kernel_LT_4x4_penryn.S index e186b94522..5c43a929f5 100644 --- a/kernel/x86_64/trsm_kernel_LT_4x4_penryn.S +++ b/kernel/x86_64/trsm_kernel_LT_4x4_penryn.S @@ -166,10 +166,10 @@ movq N, J sarq $2, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -209,10 +209,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -279,10 +279,10 @@ #endif sarq $3, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): addpd %xmm3, %xmm11 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -14 * SIZE(BO), %xmm3 @@ -532,10 +532,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -544,10 +544,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addpd %xmm3, %xmm11 movaps -14 * SIZE(BO), %xmm3 addpd %xmm4, %xmm15 @@ -583,10 +583,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax subq $4, %rax @@ -1011,13 +1011,13 @@ decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) ALIGN_4 #ifdef LN @@ -1054,10 +1054,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_4 -.L22: +L(22): addpd %xmm3, %xmm11 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -1123,10 +1123,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1135,10 +1135,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): addpd %xmm3, %xmm11 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -1159,10 +1159,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1390,10 +1390,10 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M BRANCH - jle .L39 + jle L(39) ALIGN_4 #ifdef LN @@ -1429,10 +1429,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) shufps $0x44, %xmm0, %xmm0 @@ -1479,10 +1479,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1491,10 +1491,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): shufps $0x44, %xmm0, %xmm0 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm3 @@ -1510,10 +1510,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1668,7 +1668,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1687,13 +1687,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $2, N BRANCH - jle .L80 + jle L(80) #if defined(LT) || defined(RN) movq A, AO @@ -1734,10 +1734,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1785,10 +1785,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_4 -.L52: +L(52): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -1863,10 +1863,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1875,10 +1875,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -1901,10 +1901,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2126,13 +2126,13 @@ decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M BRANCH - jle .L70 + jle L(70) ALIGN_4 #ifdef LN @@ -2166,10 +2166,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x4e, %xmm2, %xmm7 @@ -2212,10 +2212,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2224,10 +2224,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm7 @@ -2242,10 +2242,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2390,10 +2390,10 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M BRANCH - jle .L79 + jle L(79) ALIGN_4 #ifdef LN @@ -2426,10 +2426,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_4 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) shufps $0x44, %xmm0, %xmm0 @@ -2460,10 +2460,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2472,10 +2472,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): shufps $0x44, %xmm0, %xmm0 mulpd %xmm0, %xmm2 movsd -15 * SIZE(AO), %xmm0 @@ -2487,10 +2487,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2597,7 +2597,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2615,10 +2615,10 @@ #endif ALIGN_4 -.L80: +L(80): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(LT) || defined(RN) movq A, AO @@ -2653,10 +2653,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2695,10 +2695,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_4 -.L92: +L(92): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm2, %xmm3 @@ -2755,10 +2755,10 @@ subq $ -4 * SIZE, BO subq $1, %rax BRANCH - jg .L92 + jg L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2767,10 +2767,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): pshufd $0x44, %xmm2, %xmm3 pshufd $0x44, %xmm2, %xmm4 movsd -15 * SIZE(BO), %xmm2 @@ -2788,10 +2788,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2974,13 +2974,13 @@ decq I BRANCH - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M BRANCH - jle .L110 + jle L(110) ALIGN_4 #ifdef LN @@ -3013,10 +3013,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_4 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm2, %xmm3 @@ -3051,10 +3051,10 @@ subq $-4 * SIZE, BO subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3063,10 +3063,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L108 + je L(108) ALIGN_4 -.L106: +L(106): pshufd $0x44, %xmm2, %xmm3 movsd -15 * SIZE(BO), %xmm2 @@ -3079,10 +3079,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_4 -.L108: +L(108): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3198,10 +3198,10 @@ #endif ALIGN_4 -.L110: +L(110): testq $1, M BRANCH - jle .L119 + jle L(119) #ifdef LN movq K, %rax @@ -3233,10 +3233,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_4 -.L112: +L(112): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulsd %xmm0, %xmm2 @@ -3263,10 +3263,10 @@ subq $-4 * SIZE, BO subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3275,10 +3275,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd %xmm0, %xmm2 addsd %xmm2, %xmm8 movsd -15 * SIZE(AO), %xmm0 @@ -3289,10 +3289,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -3378,7 +3378,7 @@ #endif ALIGN_4 -.L119: +L(119): #ifdef LN leaq (B, K, SIZE), B #endif @@ -3395,7 +3395,7 @@ #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LT_4x4_sse2.S b/kernel/x86_64/trsm_kernel_LT_4x4_sse2.S index 583fb47a5b..25d71c33b6 100644 --- a/kernel/x86_64/trsm_kernel_LT_4x4_sse2.S +++ b/kernel/x86_64/trsm_kernel_LT_4x4_sse2.S @@ -298,9 +298,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) -.L01: +L(01): /* Copying to Sub Buffer */ #ifdef LN @@ -337,12 +337,12 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): PREFETCHNTA 40 * SIZE(B) movsd 0 * SIZE(B), %xmm0 @@ -375,10 +375,10 @@ movsd %xmm7, -1 * SIZE(BO) decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -387,10 +387,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movsd 0 * SIZE(B), %xmm0 movsd 1 * SIZE(B), %xmm1 movsd 2 * SIZE(B), %xmm2 @@ -408,10 +408,10 @@ addq $4 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -431,10 +431,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -489,8 +489,8 @@ #if 1 andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -508,7 +508,7 @@ KERNEL7(16 * 1) KERNEL8(16 * 1) cmpq $64 * 2, %rax - jle .L12 + jle L(12) KERNEL1(16 * 2) KERNEL2(16 * 2) KERNEL3(16 * 2) @@ -526,7 +526,7 @@ KERNEL7(16 * 3) KERNEL8(16 * 3) cmpq $64 * 4, %rax - jle .L12 + jle L(12) KERNEL1(16 * 4) KERNEL2(16 * 4) KERNEL3(16 * 4) @@ -544,7 +544,7 @@ KERNEL7(16 * 5) KERNEL8(16 * 5) cmpq $64 * 6, %rax - jle .L12 + jle L(12) KERNEL1(16 * 6) KERNEL2(16 * 6) KERNEL3(16 * 6) @@ -565,18 +565,18 @@ addq $16 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $64 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 4), BO # * 64 ALIGN_4 #else sarq $3, %rax - je .L15 + je L(15) -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -597,12 +597,12 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jg .L12 + jg L(12) ALIGN_4 #endif -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -611,10 +611,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -643,10 +643,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1158,18 +1158,18 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $3, M - je .L39 + je L(39) testq $2, M - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -1210,10 +1210,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1323,10 +1323,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1335,10 +1335,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -1355,10 +1355,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1628,12 +1628,12 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1674,10 +1674,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulsd %xmm8, %xmm9 addsd %xmm9, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1786,10 +1786,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1798,10 +1798,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulsd %xmm8, %xmm9 addsd %xmm9, %xmm0 movsd 2 * SIZE(BO), %xmm9 @@ -1818,10 +1818,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1997,7 +1997,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -2019,18 +2019,18 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $3, N - je .L999 + je L(999) testq $2, N - je .L80 + je L(80) ALIGN_4 -.L41: +L(41): /* Copying to Sub Buffer */ #ifdef LN @@ -2067,10 +2067,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): PREFETCH 56 * SIZE(B) movsd 0 * SIZE(B), %xmm0 @@ -2103,10 +2103,10 @@ movsd %xmm7, -1 * SIZE(BO) decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2115,10 +2115,10 @@ #endif andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movsd 0 * SIZE(B), %xmm0 movsd 1 * SIZE(B), %xmm1 @@ -2130,10 +2130,10 @@ addq $2 * SIZE, B addq $4 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): #if defined(LT) || defined(RN) movq A, AO #else @@ -2153,10 +2153,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2202,10 +2202,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(BO), %xmm8 @@ -2317,10 +2317,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2329,10 +2329,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 @@ -2349,10 +2349,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2617,15 +2617,15 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -2666,10 +2666,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(BO), %xmm8 @@ -2731,10 +2731,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2743,10 +2743,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_4 -.L66: +L(66): mulpd %xmm8, %xmm9 mulpd 2 * SIZE(BO), %xmm8 addpd %xmm9, %xmm0 @@ -2757,10 +2757,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L69: +L(69): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -2929,12 +2929,12 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -2975,10 +2975,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulsd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulsd 2 * SIZE(BO), %xmm8 @@ -3039,10 +3039,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3051,10 +3051,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulsd %xmm8, %xmm9 mulsd 2 * SIZE(BO), %xmm8 addsd %xmm9, %xmm0 @@ -3065,10 +3065,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addsd %xmm2, %xmm0 addsd %xmm3, %xmm1 @@ -3187,7 +3187,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -3209,12 +3209,12 @@ #endif ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L81: +L(81): /* Copying to Sub Buffer */ #ifdef LN @@ -3251,10 +3251,10 @@ subq KK, %rax #endif sarq $3, %rax - jle .L83 + jle L(83) ALIGN_4 -.L82: +L(82): PREFETCH 56 * SIZE(B) movsd 0 * SIZE(B), %xmm0 @@ -3287,10 +3287,10 @@ movsd %xmm7, -1 * SIZE(BO) decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L83: +L(83): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3299,10 +3299,10 @@ #endif andq $7, %rax BRANCH - jle .L90 + jle L(90) ALIGN_4 -.L84: +L(84): movsd 0 * SIZE(B), %xmm0 movsd %xmm0, 0 * SIZE(BO) @@ -3311,10 +3311,10 @@ addq $1 * SIZE, B addq $2 * SIZE, BO decq %rax - jne .L84 + jne L(84) ALIGN_4 -.L90: +L(90): #if defined(LT) || defined(RN) movq A, AO #else @@ -3332,10 +3332,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -3378,10 +3378,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(AO), %xmm9 @@ -3438,10 +3438,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3450,10 +3450,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_4 -.L96: +L(96): mulpd %xmm9, %xmm8 mulpd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -3464,10 +3464,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L99: +L(99): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -3669,15 +3669,15 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -3715,10 +3715,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L105 + je L(105) ALIGN_4 -.L102: +L(102): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd 2 * SIZE(AO), %xmm8 @@ -3751,10 +3751,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3763,10 +3763,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_4 -.L106: +L(106): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(AO), %xmm8 @@ -3775,10 +3775,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L106 + jg L(106) ALIGN_4 -.L109: +L(109): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -3909,12 +3909,12 @@ #endif ALIGN_4 -.L110: +L(110): testq $1, M - je .L119 + je L(119) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -3952,10 +3952,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulsd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd 1 * SIZE(AO), %xmm8 @@ -3987,10 +3987,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3999,10 +3999,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd %xmm8, %xmm9 movsd 1 * SIZE(AO), %xmm8 addsd %xmm9, %xmm0 @@ -4011,10 +4011,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addsd %xmm2, %xmm0 addsd %xmm3, %xmm1 addsd %xmm1, %xmm0 @@ -4115,7 +4115,7 @@ #endif ALIGN_4 -.L119: +L(119): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -4138,7 +4138,7 @@ ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/trsm_kernel_LT_4x4_sse3.S b/kernel/x86_64/trsm_kernel_LT_4x4_sse3.S index 3856c72098..004f85591e 100644 --- a/kernel/x86_64/trsm_kernel_LT_4x4_sse3.S +++ b/kernel/x86_64/trsm_kernel_LT_4x4_sse3.S @@ -399,10 +399,10 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -441,10 +441,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -497,8 +497,8 @@ #if 1 andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1 (16 * 0) KERNEL2 (16 * 0) KERNEL3 (16 * 0) @@ -517,7 +517,7 @@ KERNEL16(16 * 0) cmpq $128 * 1, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 1) KERNEL2 (16 * 1) KERNEL3 (16 * 1) @@ -536,7 +536,7 @@ KERNEL16(16 * 1) cmpq $128 * 2, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 2) KERNEL2 (16 * 2) KERNEL3 (16 * 2) @@ -555,7 +555,7 @@ KERNEL16(16 * 2) cmpq $128 * 3, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 3) KERNEL2 (16 * 3) KERNEL3 (16 * 3) @@ -574,7 +574,7 @@ KERNEL16(16 * 3) cmpq $128 * 4, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 4) KERNEL2 (16 * 4) KERNEL3 (16 * 4) @@ -593,7 +593,7 @@ KERNEL16(16 * 4) cmpq $128 * 5, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 5) KERNEL2 (16 * 5) KERNEL3 (16 * 5) @@ -612,7 +612,7 @@ KERNEL16(16 * 5) cmpq $128 * 6, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 6) KERNEL2 (16 * 6) KERNEL3 (16 * 6) @@ -631,7 +631,7 @@ KERNEL16(16 * 6) cmpq $128 * 7, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 7) KERNEL2 (16 * 7) KERNEL3 (16 * 7) @@ -652,17 +652,17 @@ addq $32 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $128 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 2), BO # * 64 #else sarq $3, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -889,11 +889,11 @@ addq $32 * SIZE, AO decq %rax BRANCH - jne .L12 + jne L(12) #endif ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -902,10 +902,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 addpd %xmm9, %xmm0 @@ -936,10 +936,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax subq $4, %rax @@ -1344,16 +1344,16 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax @@ -1387,10 +1387,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1500,10 +1500,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1512,10 +1512,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -1533,10 +1533,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1750,12 +1750,12 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1789,10 +1789,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1854,10 +1854,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1866,10 +1866,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -1881,10 +1881,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax @@ -2069,7 +2069,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -2087,12 +2087,12 @@ #endif decq J # j -- - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $2, N - je .L80 + je L(80) ALIGN_4 #if defined(LT) || defined(RN) @@ -2133,10 +2133,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2180,10 +2180,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -2302,10 +2302,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2314,10 +2314,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 addpd %xmm9, %xmm0 @@ -2336,10 +2336,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2556,15 +2556,15 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -2597,10 +2597,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -2662,10 +2662,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2674,10 +2674,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_4 -.L66: +L(66): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -2689,10 +2689,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L69: +L(69): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -2828,12 +2828,12 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -2866,10 +2866,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movddup 1 * SIZE(AO), %xmm8 @@ -2901,10 +2901,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2913,10 +2913,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulpd %xmm8, %xmm9 movddup 1 * SIZE(AO), %xmm8 addpd %xmm9, %xmm0 @@ -2925,10 +2925,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -3045,7 +3045,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -3064,9 +3064,9 @@ #endif ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 #if defined(LT) || defined(RN) @@ -3102,10 +3102,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -3144,10 +3144,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(AO), %xmm9 @@ -3202,10 +3202,10 @@ addq $32 * SIZE, AO addq $8 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3214,10 +3214,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_4 -.L96: +L(96): mulpd %xmm9, %xmm8 mulpd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -3228,10 +3228,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L99: +L(99): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -3414,15 +3414,15 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -3455,10 +3455,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L105 + je L(105) ALIGN_4 -.L102: +L(102): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movddup 1 * SIZE(BO), %xmm9 @@ -3490,10 +3490,10 @@ addq $16 * SIZE, AO addq $ 8 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3502,10 +3502,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_4 -.L106: +L(106): mulpd %xmm9, %xmm8 movddup 1 * SIZE(BO), %xmm9 addpd %xmm8, %xmm0 @@ -3514,10 +3514,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L106 + jg L(106) ALIGN_4 -.L109: +L(109): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -3633,12 +3633,12 @@ #endif ALIGN_4 -.L110: +L(110): testq $1, M - je .L119 + je L(119) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -3671,10 +3671,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulpd %xmm9, %xmm8 movapd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -3693,10 +3693,10 @@ addq $8 * SIZE, AO addq $8 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3705,10 +3705,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd 0 * SIZE(BO), %xmm9 addsd %xmm9, %xmm0 movsd 1 * SIZE(AO), %xmm9 @@ -3716,10 +3716,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addpd %xmm1, %xmm0 haddpd %xmm0, %xmm0 @@ -3808,7 +3808,7 @@ #endif ALIGN_4 -.L119: +L(119): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -3827,7 +3827,7 @@ #endif ALIGN_2 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LT_4x8_nehalem.S b/kernel/x86_64/trsm_kernel_LT_4x8_nehalem.S index 28f38bd617..17043e26fc 100644 --- a/kernel/x86_64/trsm_kernel_LT_4x8_nehalem.S +++ b/kernel/x86_64/trsm_kernel_LT_4x8_nehalem.S @@ -166,10 +166,10 @@ movq N, J sarq $3, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -209,10 +209,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -266,10 +266,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm12 @@ -396,10 +396,10 @@ subq $-16 * SIZE, AO subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -408,10 +408,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addps %xmm1, %xmm12 movaps -32 * SIZE(BO), %xmm1 addps %xmm2, %xmm13 @@ -447,10 +447,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1070,13 +1070,13 @@ decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) #ifdef LN movq K, %rax @@ -1114,10 +1114,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1195,10 +1195,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1207,10 +1207,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1234,10 +1234,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1687,10 +1687,10 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M BRANCH - jle .L39 + jle L(39) #ifdef LN movq K, %rax @@ -1722,10 +1722,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movaps -32 * SIZE(BO), %xmm2 @@ -1765,10 +1765,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1777,10 +1777,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -1795,10 +1795,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2197,7 +2197,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 8), B @@ -2216,12 +2216,12 @@ subq $1, J BRANCH - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $4, N - jle .L70 + jle L(70) #if defined(LT) || defined(RN) movq A, AO @@ -2258,10 +2258,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L50 + jle L(50) ALIGN_4 -.L41: +L(41): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2301,10 +2301,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -2371,10 +2371,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2383,10 +2383,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): addps %xmm1, %xmm8 movaps -32 * SIZE(BO), %xmm1 addps %xmm2, %xmm9 @@ -2407,10 +2407,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2712,13 +2712,13 @@ decq I BRANCH - jg .L41 + jg L(41) ALIGN_4 -.L50: +L(50): testq $2, M BRANCH - jle .L60 + jle L(60) #ifdef LN movq K, %rax @@ -2751,10 +2751,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2796,10 +2796,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2808,10 +2808,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2826,10 +2826,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3033,10 +3033,10 @@ #endif ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L69 + jle L(69) #ifdef LN movq K, %rax @@ -3067,10 +3067,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movaps -32 * SIZE(BO), %xmm2 @@ -3098,11 +3098,11 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) addps %xmm9, %xmm8 ALIGN_3 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3111,10 +3111,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -3126,10 +3126,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3303,7 +3303,7 @@ #endif ALIGN_4 -.L69: +L(69): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -3321,9 +3321,9 @@ #endif ALIGN_4 -.L70: +L(70): testq $2, N - jle .L100 + jle L(100) #if defined(LT) || defined(RN) movq A, AO @@ -3360,10 +3360,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L80 + jle L(80) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -3398,10 +3398,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -3444,10 +3444,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_3 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3456,10 +3456,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addps %xmm1, %xmm8 pshufd $0x00, %xmm3, %xmm1 mulps %xmm0, %xmm1 @@ -3474,10 +3474,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_3 -.L78: +L(78): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3681,13 +3681,13 @@ decq I BRANCH - jg .L71 + jg L(71) ALIGN_4 -.L80: +L(80): testq $2, M BRANCH - jle .L90 + jle L(90) #ifdef LN movq K, %rax @@ -3720,10 +3720,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): addps %xmm1, %xmm8 movsd -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -3753,10 +3753,10 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) ALIGN_3 -.L85: +L(85): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3765,10 +3765,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): addps %xmm1, %xmm8 movsd -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -3780,10 +3780,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_3 -.L88: +L(88): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3910,10 +3910,10 @@ #endif ALIGN_4 -.L90: +L(90): testq $1, M BRANCH - jle .L99 + jle L(99) #ifdef LN movq K, %rax @@ -3944,10 +3944,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_3 -.L92: +L(92): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movsd -32 * SIZE(BO), %xmm2 @@ -3975,11 +3975,11 @@ subq $1, %rax BRANCH - jg .L92 + jg L(92) addps %xmm9, %xmm8 ALIGN_3 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3988,10 +3988,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_3 -.L96: +L(96): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -4003,10 +4003,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_3 -.L98: +L(98): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -4112,7 +4112,7 @@ #endif ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -4130,9 +4130,9 @@ #endif ALIGN_4 -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) #if defined(LT) || defined(RN) movq A, AO @@ -4167,10 +4167,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L110 + jle L(110) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -4201,10 +4201,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_3 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -4235,10 +4235,10 @@ subq $ -4 * SIZE, BO subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_3 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4247,10 +4247,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L108 + je L(108) ALIGN_3 -.L106: +L(106): addps %xmm1, %xmm8 pshufd $0x00, %xmm3, %xmm1 movss -31 * SIZE(BO), %xmm3 @@ -4262,10 +4262,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_3 -.L108: +L(108): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -4433,13 +4433,13 @@ decq I BRANCH - jg .L101 + jg L(101) ALIGN_4 -.L110: +L(110): testq $2, M BRANCH - jle .L120 + jle L(120) #ifdef LN movq K, %rax @@ -4469,10 +4469,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_3 -.L112: +L(112): addps %xmm1, %xmm8 movss -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -4502,10 +4502,10 @@ subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_3 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4514,10 +4514,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_3 -.L116: +L(116): addps %xmm1, %xmm8 movss -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -4529,10 +4529,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_3 -.L118: +L(118): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -4636,10 +4636,10 @@ #endif ALIGN_4 -.L120: +L(120): testq $1, M BRANCH - jle .L129 + jle L(129) #ifdef LN movq K, %rax @@ -4669,10 +4669,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L125 + jle L(125) ALIGN_3 -.L122: +L(122): addss %xmm2, %xmm8 movss -32 * SIZE(BO), %xmm2 mulss %xmm0, %xmm2 @@ -4698,10 +4698,10 @@ subq $1, %rax BRANCH - jg .L122 + jg L(122) ALIGN_3 -.L125: +L(125): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4710,10 +4710,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_3 -.L126: +L(126): addss %xmm2, %xmm8 movss -32 * SIZE(BO), %xmm2 mulss %xmm0, %xmm2 @@ -4724,10 +4724,10 @@ subq $1, %rax BRANCH - jg .L126 + jg L(126) ALIGN_3 -.L128: +L(128): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -4800,7 +4800,7 @@ #endif ALIGN_4 -.L129: +L(129): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -4818,7 +4818,7 @@ #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_LT_8x4_sse.S b/kernel/x86_64/trsm_kernel_LT_8x4_sse.S index 887c071ba8..81e69910f9 100644 --- a/kernel/x86_64/trsm_kernel_LT_8x4_sse.S +++ b/kernel/x86_64/trsm_kernel_LT_8x4_sse.S @@ -189,9 +189,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L50 + jle L(50) -.L01: +L(01): /* Copying to Sub Buffer */ #ifdef LN @@ -228,10 +228,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 movaps 8 * SIZE(B), %xmm11 @@ -279,10 +279,10 @@ addq $64 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -291,10 +291,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movaps 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -310,10 +310,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -333,10 +333,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $3 + BASE_SHIFT, %rax @@ -388,10 +388,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 0 * SIZE(BO), %xmm9 @@ -492,10 +492,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -504,9 +504,9 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -535,10 +535,10 @@ addq $8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1201,12 +1201,12 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M - je .L30 + je L(30) #ifdef LN movq K, %rax @@ -1249,10 +1249,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -1366,10 +1366,10 @@ addq $ 32 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1378,10 +1378,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1398,10 +1398,10 @@ addq $ 4 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1725,9 +1725,9 @@ #endif ALIGN_4 -.L30: +L(30): testq $2, M - je .L40 + je L(40) #ifdef LN movq K, %rax @@ -1770,10 +1770,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -1892,10 +1892,10 @@ addq $ 16 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1904,10 +1904,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1925,10 +1925,10 @@ addq $ 2 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2173,9 +2173,9 @@ #endif ALIGN_4 -.L40: +L(40): testq $1, M - je .L49 + je L(49) #ifdef LN movq K, %rax @@ -2217,10 +2217,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): mulss %xmm8, %xmm9 addss %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -2339,10 +2339,10 @@ addq $ 8 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2351,10 +2351,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -2372,10 +2372,10 @@ addq $ 1 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2576,7 +2576,7 @@ #endif ALIGN_4 -.L49: +L(49): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -2598,11 +2598,11 @@ #endif decq J # j -- - jg .L01 + jg L(01) -.L50: +L(50): testq $2, N - je .L100 + je L(100) #ifdef LN movq OFFSET, %rax @@ -2638,10 +2638,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 @@ -2668,10 +2668,10 @@ addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2680,10 +2680,10 @@ #endif andq $3, %rax BRANCH - jle .L60 + jle L(60) ALIGN_4 -.L54: +L(54): movsd 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -2695,10 +2695,10 @@ addq $2 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L54 + jne L(54) ALIGN_4 -.L60: +L(60): #if defined(LT) || defined(RN) movq A, AO #else @@ -2717,10 +2717,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L70 + jle L(70) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $3 + BASE_SHIFT, %rax @@ -2767,10 +2767,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2890,10 +2890,10 @@ addq $64 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2902,10 +2902,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -2922,10 +2922,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3443,12 +3443,12 @@ #endif decq I # i -- - jg .L61 + jg L(61) ALIGN_4 -.L70: +L(70): testq $4, M - je .L80 + je L(80) #ifdef LN movq K, %rax @@ -3491,10 +3491,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3558,10 +3558,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3570,10 +3570,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -3584,10 +3584,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -3835,9 +3835,9 @@ #endif ALIGN_4 -.L80: +L(80): testq $2, M - je .L90 + je L(90) #ifdef LN movq K, %rax @@ -3880,10 +3880,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L85 + je L(85) ALIGN_4 -.L82: +L(82): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -3954,10 +3954,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L85: +L(85): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3966,10 +3966,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -3981,10 +3981,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -4154,9 +4154,9 @@ #endif ALIGN_4 -.L90: +L(90): testq $1, M - je .L99 + je L(99) #ifdef LN movq K, %rax @@ -4198,10 +4198,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -4272,10 +4272,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4284,10 +4284,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -4299,10 +4299,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): addss %xmm2, %xmm0 addss %xmm3, %xmm1 @@ -4428,7 +4428,7 @@ #endif ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -4450,9 +4450,9 @@ #endif ALIGN_4 -.L100: +L(100): testq $1, N - je .L999 + je L(999) #ifdef LN movq OFFSET, %rax @@ -4488,10 +4488,10 @@ subq KK, %rax #endif sarq $3, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movsd 0 * SIZE(B), %xmm3 movhps 2 * SIZE(B), %xmm3 movsd 4 * SIZE(B), %xmm7 @@ -4520,10 +4520,10 @@ addq $32 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4532,10 +4532,10 @@ #endif andq $7, %rax BRANCH - jle .L110 + jle L(110) ALIGN_4 -.L104: +L(104): movss 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -4545,10 +4545,10 @@ addq $ 1 * SIZE, B addq $ 4 * SIZE, BO decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L110: +L(110): #if defined(LT) || defined(RN) movq A, AO #else @@ -4566,10 +4566,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L120 + jle L(120) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $3 + BASE_SHIFT, %rax @@ -4615,10 +4615,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulps %xmm9, %xmm8 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -4691,10 +4691,10 @@ addq $64 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4703,10 +4703,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulps %xmm9, %xmm8 mulps 4 * SIZE(AO), %xmm9 addps %xmm8, %xmm0 @@ -4717,10 +4717,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -5144,12 +5144,12 @@ #endif decq I # i -- - jg .L111 + jg L(111) ALIGN_4 -.L120: +L(120): testq $4, M - je .L130 + je L(130) #ifdef LN movq K, %rax @@ -5190,10 +5190,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L125 + je L(125) ALIGN_4 -.L122: +L(122): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -5231,10 +5231,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L122 + jne L(122) ALIGN_4 -.L125: +L(125): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -5243,10 +5243,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): mulps %xmm8, %xmm9 movaps 4 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -5255,10 +5255,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): addps %xmm1, %xmm0 addps %xmm3, %xmm2 addps %xmm2, %xmm0 @@ -5457,9 +5457,9 @@ #endif ALIGN_4 -.L130: +L(130): testq $2, M - je .L140 + je L(140) #ifdef LN movq K, %rax @@ -5500,10 +5500,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L135 + je L(135) ALIGN_4 -.L132: +L(132): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -5549,10 +5549,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L132 + jne L(132) ALIGN_4 -.L135: +L(135): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -5561,10 +5561,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): mulps %xmm8, %xmm9 movsd 2 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -5573,10 +5573,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): addps %xmm1, %xmm0 #if defined(LN) || defined(RT) @@ -5705,9 +5705,9 @@ #endif ALIGN_4 -.L140: +L(140): testq $1, M - je .L149 + je L(149) #ifdef LN movq K, %rax @@ -5747,10 +5747,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L145 + je L(145) ALIGN_4 -.L142: +L(142): mulss %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -5784,10 +5784,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L142 + jne L(142) ALIGN_4 -.L145: +L(145): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -5796,10 +5796,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): mulss %xmm8, %xmm9 movss 1 * SIZE(AO), %xmm8 addss %xmm9, %xmm0 @@ -5808,10 +5808,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): addss %xmm1, %xmm0 addss %xmm3, %xmm2 addss %xmm2, %xmm0 @@ -5896,7 +5896,7 @@ #endif ALIGN_4 -.L149: +L(149): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -5918,7 +5918,7 @@ #endif ALIGN_4 -.L999: +L(999): movq %rbx, %rsp EMMS movq 0(%rsp), %rbx diff --git a/kernel/x86_64/trsm_kernel_RT_2x8_nehalem.S b/kernel/x86_64/trsm_kernel_RT_2x8_nehalem.S index b8e75e798b..de58f5a281 100644 --- a/kernel/x86_64/trsm_kernel_RT_2x8_nehalem.S +++ b/kernel/x86_64/trsm_kernel_RT_2x8_nehalem.S @@ -165,7 +165,7 @@ #endif testq $1, N - jle .L30 + jle L(30) ALIGN_4 #if defined(LT) || defined(RN) @@ -201,10 +201,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L80 + jle L(80) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -239,10 +239,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -270,12 +270,12 @@ subq $1, %rax BRANCH - jg .L72 + jg L(72) addpd %xmm9, %xmm8 ALIGN_3 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -284,10 +284,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addpd %xmm1, %xmm8 movddup -16 * SIZE(BO), %xmm1 mulpd %xmm0, %xmm1 @@ -298,10 +298,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -402,13 +402,13 @@ decq I BRANCH - jg .L71 + jg L(71) ALIGN_4 -.L80: +L(80): testq $1, M BRANCH - jle .L89 + jle L(89) ALIGN_4 #ifdef LN @@ -442,10 +442,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): mulpd %xmm0, %xmm1 movsd -14 * SIZE(AO), %xmm0 movhps -13 * SIZE(AO), %xmm0 @@ -465,12 +465,12 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) addpd %xmm9, %xmm8 ALIGN_3 -.L85: +L(85): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -479,10 +479,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): mulsd %xmm0, %xmm1 movsd -15 * SIZE(AO), %xmm0 addsd %xmm1, %xmm8 @@ -493,10 +493,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -568,7 +568,7 @@ #endif ALIGN_4 -.L89: +L(89): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -586,9 +586,9 @@ #endif ALIGN_4 -.L30: +L(30): testq $2, N - jle .L50 + jle L(50) ALIGN_4 #if defined(LT) || defined(RN) @@ -626,10 +626,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -665,10 +665,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -708,13 +708,13 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 ALIGN_3 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -723,10 +723,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addpd %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm9 @@ -740,10 +740,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -878,13 +878,13 @@ decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L69 + jle L(69) ALIGN_4 #ifdef LN @@ -916,10 +916,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): mulpd %xmm0, %xmm1 movddup -15 * SIZE(AO), %xmm0 addpd %xmm1, %xmm8 @@ -945,10 +945,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -957,10 +957,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): mulpd %xmm0, %xmm1 movddup -15 * SIZE(AO), %xmm0 addpd %xmm1, %xmm8 @@ -971,10 +971,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1077,7 +1077,7 @@ #endif ALIGN_4 -.L69: +L(69): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -1095,9 +1095,9 @@ #endif ALIGN_4 -.L50: +L(50): testq $4, N - jle .L70 + jle L(70) ALIGN_4 #if defined(LT) || defined(RN) @@ -1135,10 +1135,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -1178,10 +1178,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -1253,10 +1253,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1265,10 +1265,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): addpd %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm9 @@ -1290,10 +1290,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1521,13 +1521,13 @@ decq I BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L40: +L(40): testq $1, M BRANCH - jle .L49 + jle L(49) ALIGN_4 #ifdef LN @@ -1561,10 +1561,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1602,10 +1602,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1614,10 +1614,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1631,10 +1631,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1792,7 +1792,7 @@ #endif ALIGN_4 -.L49: +L(49): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1810,14 +1810,14 @@ #endif ALIGN_4 -.L70: +L(70): movq N, J sarq $3, J NOBRANCH - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -1857,10 +1857,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -1915,10 +1915,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm12 @@ -2041,10 +2041,10 @@ subq $-8 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2053,10 +2053,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addpd %xmm1, %xmm12 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm13 @@ -2092,10 +2092,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2545,13 +2545,13 @@ decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M BRANCH - jle .L29 + jle L(29) ALIGN_4 #ifdef LN @@ -2585,10 +2585,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -2650,10 +2650,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2662,10 +2662,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -2685,10 +2685,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3026,7 +3026,7 @@ #endif ALIGN_4 -.L29: +L(29): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 8), B @@ -3045,10 +3045,10 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_RT_4x2_atom.S b/kernel/x86_64/trsm_kernel_RT_4x2_atom.S index 9b5a93717c..a9740b88b0 100644 --- a/kernel/x86_64/trsm_kernel_RT_4x2_atom.S +++ b/kernel/x86_64/trsm_kernel_RT_4x2_atom.S @@ -155,7 +155,7 @@ #endif testq $1, N - je .L40 + je L(40) ALIGN_4 #if defined(LT) || defined(RN) @@ -190,10 +190,10 @@ movq M, I sarq $2, I - jle .L50 + jle L(50) ALIGN_4 -.L41: +L(41): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -234,10 +234,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): addsd %xmm9, %xmm8 movsd 4 * SIZE(AO), %xmm9 mulsd %xmm4, %xmm0 @@ -311,10 +311,10 @@ mulsd %xmm5, %xmm15 movsd 1 * SIZE(BO), %xmm5 - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -330,10 +330,10 @@ andq $3, %rax BRANCH BRANCH - je .L49 + je L(49) ALIGN_4 -.L46: +L(46): mulsd %xmm4, %xmm0 mulsd %xmm4, %xmm1 mulsd %xmm4, %xmm2 @@ -353,10 +353,10 @@ addq $1 * SIZE, BO decq %rax BRANCH - jg .L46 + jg L(46) ALIGN_4 -.L49: +L(49): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -509,12 +509,12 @@ #endif decq I # i -- - jg .L41 + jg L(41) ALIGN_4 -.L50: +L(50): testq $2, M - je .L60 + je L(60) #ifdef LN movq K, %rax @@ -549,10 +549,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): addsd %xmm2, %xmm8 movsd 2 * SIZE(AO), %xmm2 mulsd %xmm4, %xmm0 @@ -593,10 +593,10 @@ mulsd %xmm5, %xmm3 movsd 1 * SIZE(BO), %xmm5 - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -608,10 +608,10 @@ andq $3, %rax BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): mulsd %xmm4, %xmm0 mulsd %xmm4, %xmm1 movsd 1 * SIZE(BO), %xmm4 @@ -625,10 +625,10 @@ addq $1 * SIZE, BO decq %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -725,9 +725,9 @@ #endif ALIGN_4 -.L60: +L(60): testq $1, M - je .L69 + je L(69) #ifdef LN movq K, %rax @@ -764,10 +764,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): addsd %xmm5, %xmm8 movsd 2 * SIZE(BO), %xmm5 mulsd %xmm0, %xmm1 @@ -792,13 +792,13 @@ addq $4 * SIZE, BO decq %rax - jne .L62 + jne L(62) addsd %xmm5, %xmm8 addsd %xmm7, %xmm9 ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -807,10 +807,10 @@ #endif andq $3, %rax BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): movsd 0 * SIZE(AO), %xmm0 movsd 0 * SIZE(BO), %xmm1 @@ -822,10 +822,10 @@ decq %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addsd %xmm9, %xmm8 #if defined(LN) || defined(RT) @@ -899,7 +899,7 @@ #endif ALIGN_4 -.L69: +L(69): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -918,13 +918,13 @@ #endif ALIGN_2 -.L40: +L(40): movq N, J sarq $1, J - jle .L999 + jle L(999) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -963,10 +963,10 @@ movq M, I sarq $2, I - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1015,10 +1015,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): addsd %xmm2, %xmm13 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps %xmm0, %xmm2 @@ -1161,10 +1161,10 @@ mulsd %xmm3, %xmm6 movsd 1 * SIZE(BO), %xmm3 - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1173,10 +1173,10 @@ #endif andq $3, %rax BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): addsd %xmm2, %xmm13 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -1215,10 +1215,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): addsd %xmm2, %xmm13 addsd %xmm7, %xmm14 addsd %xmm6, %xmm15 @@ -1518,13 +1518,13 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - je .L30 + je L(30) #ifdef LN movq K, %rax @@ -1564,10 +1564,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addsd %xmm2, %xmm9 movaps %xmm0, %xmm2 @@ -1645,10 +1645,10 @@ addq $8 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1657,10 +1657,10 @@ #endif andq $3, %rax BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): addsd %xmm2, %xmm9 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -1683,10 +1683,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): addsd %xmm2, %xmm9 addsd %xmm6, %xmm11 @@ -1848,9 +1848,9 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #ifdef LN movq K, %rax @@ -1886,10 +1886,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): addsd %xmm5, %xmm8 movsd 2 * SIZE(BO), %xmm5 mulsd %xmm0, %xmm1 @@ -1931,10 +1931,10 @@ addq $8 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1947,10 +1947,10 @@ andq $3, %rax BRANCH BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulsd %xmm0, %xmm1 addq $2 * SIZE, BO mulsd %xmm0, %xmm3 @@ -1964,10 +1964,10 @@ addq $1 * SIZE, AO decq %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2066,7 +2066,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2084,10 +2084,10 @@ #endif decq J # j -- - jg .L10 + jg L(10) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_RT_4x4_barcelona.S b/kernel/x86_64/trsm_kernel_RT_4x4_barcelona.S index 08e92dc30c..24f70694df 100644 --- a/kernel/x86_64/trsm_kernel_RT_4x4_barcelona.S +++ b/kernel/x86_64/trsm_kernel_RT_4x4_barcelona.S @@ -479,7 +479,7 @@ #endif testq $1, N - je .L40 + je L(40) #if defined(LT) || defined(RN) movq A, AO @@ -513,10 +513,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -565,10 +565,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L96 + je L(96) ALIGN_4 -.L92: +L(92): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -596,10 +596,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L92 + jl L(92) ALIGN_4 -.L96: +L(96): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -607,7 +607,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L99 + je L(99) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -615,7 +615,7 @@ negq %rax ALIGN_4 -.L97: +L(97): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -624,9 +624,9 @@ movddup -15 * SIZE(BO, %rax, 1), %xmm1 addq $SIZE, %rax - jl .L97 + jl L(97) ALIGN_4 -.L99: +L(99): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -803,12 +803,12 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) #ifdef LN movq K, %rax @@ -851,10 +851,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L106 + je L(106) ALIGN_4 -.L102: +L(102): mulpd -16 * SIZE(AO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -12 * SIZE(BO, %rax, 1), %xmm0 @@ -873,10 +873,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L102 + jl L(102) ALIGN_4 -.L106: +L(106): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -884,7 +884,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L109 + je L(109) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -892,16 +892,16 @@ negq %rax ALIGN_4 -.L107: +L(107): movddup -16 * SIZE(BO, %rax, 1), %xmm0 mulpd -16 * SIZE(AO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 addq $SIZE, %rax - jl .L107 + jl L(107) ALIGN_4 -.L109: +L(109): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -1018,9 +1018,9 @@ #endif ALIGN_4 -.L110: +L(110): testq $1, M - je .L119 + je L(119) #ifdef LN movq K, %rax @@ -1059,10 +1059,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L116 + je L(116) ALIGN_4 -.L112: +L(112): mulpd -16 * SIZE(BO, %rax, 1), %xmm0 addpd %xmm0, %xmm8 movapd -12 * SIZE(AO, %rax, 1), %xmm0 @@ -1073,10 +1073,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L112 + jl L(112) ALIGN_4 -.L116: +L(116): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1084,7 +1084,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L118 + je L(118) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -1092,16 +1092,16 @@ negq %rax ALIGN_4 -.L117: +L(117): mulsd -16 * SIZE(BO, %rax, 1), %xmm0 addsd %xmm0, %xmm8 movsd -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L117 + jl L(117) ALIGN_4 -.L118: +L(118): addpd %xmm9, %xmm8 haddpd %xmm8, %xmm8 @@ -1177,7 +1177,7 @@ #endif ALIGN_4 -.L119: +L(119): #ifdef LN leaq (B, K, SIZE), B #endif @@ -1195,9 +1195,9 @@ #endif ALIGN_4 -.L40: +L(40): testq $2, N - je .L80 + je L(80) #if defined(LT) || defined(RN) movq A, AO @@ -1233,10 +1233,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1291,10 +1291,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L56 + je L(56) ALIGN_4 -.L52: +L(52): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -1346,10 +1346,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L52 + jl L(52) ALIGN_4 -.L56: +L(56): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1357,7 +1357,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L59 + je L(59) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -1365,7 +1365,7 @@ negq %rax ALIGN_4 -.L57: +L(57): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -1380,10 +1380,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L57 + jl L(57) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1596,12 +1596,12 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) #ifdef LN movq K, %rax @@ -1645,10 +1645,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L66 + je L(66) ALIGN_4 -.L62: +L(62): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -1680,10 +1680,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L62 + jl L(62) ALIGN_4 -.L66: +L(66): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1691,7 +1691,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L69 + je L(69) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -1699,7 +1699,7 @@ negq %rax ALIGN_4 -.L67: +L(67): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -1709,10 +1709,10 @@ movddup -13 * SIZE(BO, %rax, 2), %xmm3 addq $SIZE, %rax - jl .L67 + jl L(67) ALIGN_4 -.L69: +L(69): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1853,12 +1853,12 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1901,10 +1901,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L76 + je L(76) ALIGN_4 -.L72: +L(72): mulpd -16 * SIZE(BO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -12 * SIZE(AO, %rax, 1), %xmm0 @@ -1923,10 +1923,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L72 + jl L(72) ALIGN_4 -.L76: +L(76): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1934,7 +1934,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L78 + je L(78) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -1942,16 +1942,16 @@ negq %rax ALIGN_4 -.L77: +L(77): mulpd -16 * SIZE(BO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L77 + jl L(77) ALIGN_4 -.L78: +L(78): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -2054,7 +2054,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2073,12 +2073,12 @@ #endif ALIGN_4 -.L80: +L(80): movq N, J sarq $2, J # j = (n >> 2) - jle .L999 + jle L(999) -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -2118,10 +2118,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2192,10 +2192,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -2205,10 +2205,10 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) BRANCH - jl .L12 + jl L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2216,7 +2216,7 @@ subq KK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_4 @@ -2229,7 +2229,7 @@ subq $-16 * SIZE, AO ALIGN_4 -.L16: +L(16): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2237,7 +2237,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L19 + je L(19) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -2245,7 +2245,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -2272,10 +2272,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L17 + jl L(17) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2682,18 +2682,18 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $3, M - je .L39 + je L(39) testq $2, M - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -2737,10 +2737,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L26 + je L(26) ALIGN_4 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -2796,10 +2796,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L22 + jl L(22) ALIGN_4 -.L26: +L(26): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2807,7 +2807,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L29 + je L(29) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -2815,7 +2815,7 @@ negq %rax ALIGN_4 -.L27: +L(27): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -2831,10 +2831,10 @@ movddup -11 * SIZE(BO, %rax, 4), %xmm5 addq $SIZE, %rax - jl .L27 + jl L(27) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3052,9 +3052,9 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #ifdef LN movq K, %rax @@ -3099,10 +3099,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L36 + je L(36) ALIGN_4 -.L32: +L(32): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO, %rax, 4), %xmm0 addpd %xmm1, %xmm8 @@ -3130,10 +3130,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L32 + jl L(32) ALIGN_4 -.L36: +L(36): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3141,7 +3141,7 @@ subq KK, %rax #endif andq $3, %rax # if (k & 1) - je .L38 + je L(38) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -3149,7 +3149,7 @@ negq %rax ALIGN_4 -.L37: +L(37): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO, %rax, 4), %xmm0 addpd %xmm1, %xmm8 @@ -3158,10 +3158,10 @@ movddup -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L37 + jl L(37) ALIGN_4 -.L38: +L(38): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -3342,7 +3342,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -3361,10 +3361,10 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq (%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_RT_4x4_core2.S b/kernel/x86_64/trsm_kernel_RT_4x4_core2.S index 64e0342de0..2b1bb8d9f6 100644 --- a/kernel/x86_64/trsm_kernel_RT_4x4_core2.S +++ b/kernel/x86_64/trsm_kernel_RT_4x4_core2.S @@ -170,10 +170,10 @@ #endif testq $1, N - je .L40 + je L(40) ALIGN_4 -.L81: +L(81): /* Copying to Sub Buffer */ #ifdef LN @@ -210,10 +210,10 @@ subq KK, %rax #endif sarq $3, %rax - jle .L83 + jle L(83) ALIGN_4 -.L82: +L(82): movddup -16 * SIZE(B), %xmm0 movddup -15 * SIZE(B), %xmm1 movddup -14 * SIZE(B), %xmm2 @@ -235,10 +235,10 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO subq $1, %rax - jne .L82 + jne L(82) ALIGN_4 -.L83: +L(83): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -247,10 +247,10 @@ #endif andq $7, %rax BRANCH - jle .L90 + jle L(90) ALIGN_4 -.L84: +L(84): movddup -16 * SIZE(B), %xmm0 movapd %xmm0, 0 * SIZE(BO) @@ -258,10 +258,10 @@ addq $1 * SIZE, B addq $2 * SIZE, BO subq $1, %rax - jne .L84 + jne L(84) ALIGN_4 -.L90: +L(90): #if defined(LT) || defined(RN) movq A, AO #else @@ -279,10 +279,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -321,10 +321,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -370,10 +370,10 @@ subq $-16 * SIZE, AO subq $ -8 * SIZE, BO subq $1, %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -382,10 +382,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_4 -.L96: +L(96): movapd -16 * SIZE(AO), %xmm0 movapd -14 * SIZE(AO), %xmm1 @@ -400,10 +400,10 @@ addq $4 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L96 + jg L(96) ALIGN_4 -.L99: +L(99): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -604,15 +604,15 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -646,10 +646,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L105 + je L(105) ALIGN_4 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -677,10 +677,10 @@ subq $-8 * SIZE, AO subq $-8 * SIZE, BO subq $1, %rax - jne .L102 + jne L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -689,10 +689,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_4 -.L106: +L(106): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 @@ -702,10 +702,10 @@ addq $2 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L106 + jg L(106) ALIGN_4 -.L109: +L(109): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -835,12 +835,12 @@ #endif ALIGN_4 -.L110: +L(110): testq $1, M - je .L119 + je L(119) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -874,10 +874,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -16 * SIZE(AO), %xmm0 @@ -903,10 +903,10 @@ subq $-4 * SIZE, AO subq $-8 * SIZE, BO subq $1, %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -915,10 +915,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): movsd -16 * SIZE(AO), %xmm0 movsd -16 * SIZE(BO), %xmm2 @@ -928,10 +928,10 @@ addq $1 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addsd %xmm10, %xmm8 addsd %xmm11, %xmm9 addsd %xmm9, %xmm8 @@ -1032,7 +1032,7 @@ #endif ALIGN_4 -.L119: +L(119): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -1054,12 +1054,12 @@ #endif ALIGN_4 -.L40: +L(40): testq $2, N - je .L80 + je L(80) ALIGN_4 -.L41: +L(41): /* Copying to Sub Buffer */ #ifdef LN @@ -1096,10 +1096,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): movddup -16 * SIZE(B), %xmm0 movddup -15 * SIZE(B), %xmm1 movddup -14 * SIZE(B), %xmm2 @@ -1121,10 +1121,10 @@ addq $8 * SIZE, B addq $16 * SIZE, BO subq $1, %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1133,10 +1133,10 @@ #endif andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movddup -16 * SIZE(B), %xmm0 movddup -15 * SIZE(B), %xmm1 @@ -1146,10 +1146,10 @@ addq $2 * SIZE, B addq $4 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): #if defined(LT) || defined(RN) movq A, AO #else @@ -1169,10 +1169,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1216,10 +1216,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1297,10 +1297,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1309,10 +1309,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): movapd -16 * SIZE(AO), %xmm0 movapd -14 * SIZE(AO), %xmm1 @@ -1334,10 +1334,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1575,15 +1575,15 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -1617,10 +1617,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1660,10 +1660,10 @@ subq $ -8 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1672,10 +1672,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_4 -.L66: +L(66): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -1689,10 +1689,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L66 + jg L(66) ALIGN_4 -.L69: +L(69): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1850,12 +1850,12 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1889,10 +1889,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -16 * SIZE(AO), %xmm0 @@ -1932,10 +1932,10 @@ subq $ -4 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1944,10 +1944,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): movsd -16 * SIZE(AO), %xmm0 movsd -16 * SIZE(BO), %xmm2 movsd -14 * SIZE(BO), %xmm3 @@ -1961,10 +1961,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addsd %xmm10, %xmm8 addsd %xmm11, %xmm9 @@ -2083,7 +2083,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2105,12 +2105,12 @@ #endif ALIGN_4 -.L80: +L(80): movq N, J sarq $2, J # j = (n >> 2) - jle .L999 + jle L(999) -.L01: +L(01): /* Copying to Sub Buffer */ #ifdef LN @@ -2147,10 +2147,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): prefetcht0 (PREFETCH_R + 0) * SIZE(B) movapd -16 * SIZE(B), %xmm0 movapd -14 * SIZE(B), %xmm1 @@ -2206,10 +2206,10 @@ subq $-16 * SIZE, B subq $-32 * SIZE, BO subq $1, %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2218,10 +2218,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movapd -16 * SIZE(B), %xmm0 movapd -14 * SIZE(B), %xmm1 @@ -2238,10 +2238,10 @@ addq $4 * SIZE, B addq $8 * SIZE, BO subq $1, %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): leaq (PREFETCH_R + 0) * SIZE(B), BB #if defined(LT) || defined(RN) @@ -2263,10 +2263,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2329,10 +2329,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm2, %xmm10 @@ -2452,10 +2452,10 @@ mulpd %xmm1, %xmm5 BRANCH - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2464,10 +2464,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): movapd -16 * SIZE(AO), %xmm0 addpd %xmm2, %xmm10 movapd -16 * SIZE(BO), %xmm2 @@ -2500,10 +2500,10 @@ addq $8 * SIZE, BO # boffset1 += 8 subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): addpd %xmm2, %xmm10 addpd %xmm3, %xmm14 addpd %xmm4, %xmm11 @@ -2956,18 +2956,18 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $3, M - je .L39 + je L(39) testq $2, M - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -3001,10 +3001,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -3074,10 +3074,10 @@ subq $ -8 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3086,10 +3086,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -3109,10 +3109,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jne .L26 + jne L(26) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3355,12 +3355,12 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -3394,10 +3394,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -16 * SIZE(AO), %xmm0 @@ -3467,10 +3467,10 @@ subq $ -4 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3479,10 +3479,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): movsd -16 * SIZE(AO), %xmm0 movsd -16 * SIZE(BO), %xmm2 movsd -14 * SIZE(BO), %xmm3 @@ -3502,10 +3502,10 @@ addq $1 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3681,7 +3681,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -3703,10 +3703,10 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/trsm_kernel_RT_4x4_penryn.S b/kernel/x86_64/trsm_kernel_RT_4x4_penryn.S index f95200ae55..e87e680854 100644 --- a/kernel/x86_64/trsm_kernel_RT_4x4_penryn.S +++ b/kernel/x86_64/trsm_kernel_RT_4x4_penryn.S @@ -165,7 +165,7 @@ testq $1, N BRANCH - jle .L40 + jle L(40) #if defined(LT) || defined(RN) movq A, AO @@ -200,10 +200,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -242,10 +242,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_4 -.L92: +L(92): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm2, %xmm3 @@ -302,10 +302,10 @@ subq $ -4 * SIZE, BO subq $1, %rax BRANCH - jg .L92 + jg L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -314,10 +314,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): pshufd $0x44, %xmm2, %xmm3 pshufd $0x44, %xmm2, %xmm4 movsd -15 * SIZE(BO), %xmm2 @@ -335,10 +335,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -521,13 +521,13 @@ decq I BRANCH - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M BRANCH - jle .L110 + jle L(110) ALIGN_4 #ifdef LN @@ -560,10 +560,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_4 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm2, %xmm3 @@ -598,10 +598,10 @@ subq $-4 * SIZE, BO subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -610,10 +610,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L108 + je L(108) ALIGN_4 -.L106: +L(106): pshufd $0x44, %xmm2, %xmm3 movsd -15 * SIZE(BO), %xmm2 @@ -626,10 +626,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_4 -.L108: +L(108): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -745,10 +745,10 @@ #endif ALIGN_4 -.L110: +L(110): testq $1, M BRANCH - jle .L119 + jle L(119) #ifdef LN movq K, %rax @@ -780,10 +780,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_4 -.L112: +L(112): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulsd %xmm0, %xmm2 @@ -810,10 +810,10 @@ subq $-4 * SIZE, BO subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -822,10 +822,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd %xmm0, %xmm2 addsd %xmm2, %xmm8 movsd -15 * SIZE(AO), %xmm0 @@ -836,10 +836,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -925,7 +925,7 @@ #endif ALIGN_4 -.L119: +L(119): #ifdef LN leaq (B, K, SIZE), B #endif @@ -942,10 +942,10 @@ #endif ALIGN_4 -.L40: +L(40): testq $2, N BRANCH - jle .L80 + jle L(80) #if defined(LT) || defined(RN) movq A, AO @@ -987,10 +987,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1038,10 +1038,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_4 -.L52: +L(52): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -1116,10 +1116,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1128,10 +1128,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -1154,10 +1154,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1379,13 +1379,13 @@ decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M BRANCH - jle .L70 + jle L(70) ALIGN_4 #ifdef LN @@ -1419,10 +1419,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x4e, %xmm2, %xmm7 @@ -1465,10 +1465,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1477,10 +1477,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm7 @@ -1495,10 +1495,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1643,10 +1643,10 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M BRANCH - jle .L79 + jle L(79) ALIGN_4 #ifdef LN @@ -1679,10 +1679,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_4 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) shufps $0x44, %xmm0, %xmm0 @@ -1713,10 +1713,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1725,10 +1725,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): shufps $0x44, %xmm0, %xmm0 mulpd %xmm0, %xmm2 movsd -15 * SIZE(AO), %xmm0 @@ -1740,10 +1740,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1850,7 +1850,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -1868,14 +1868,14 @@ #endif ALIGN_4 -.L80: +L(80): movq N, J sarq $2, J NOBRANCH - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -1916,10 +1916,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1986,10 +1986,10 @@ #endif sarq $3, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): addpd %xmm3, %xmm11 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -14 * SIZE(BO), %xmm3 @@ -2239,10 +2239,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2251,10 +2251,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addpd %xmm3, %xmm11 movaps -14 * SIZE(BO), %xmm3 addpd %xmm4, %xmm15 @@ -2290,10 +2290,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax subq $4, %rax @@ -2718,13 +2718,13 @@ decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) ALIGN_4 #ifdef LN @@ -2761,10 +2761,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_4 -.L22: +L(22): addpd %xmm3, %xmm11 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -2830,10 +2830,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2842,10 +2842,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): addpd %xmm3, %xmm11 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -2866,10 +2866,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3097,10 +3097,10 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M BRANCH - jle .L39 + jle L(39) ALIGN_4 #ifdef LN @@ -3136,10 +3136,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) shufps $0x44, %xmm0, %xmm0 @@ -3186,10 +3186,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3198,10 +3198,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): shufps $0x44, %xmm0, %xmm0 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm3 @@ -3217,10 +3217,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3375,7 +3375,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -3394,10 +3394,10 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_RT_4x4_sse2.S b/kernel/x86_64/trsm_kernel_RT_4x4_sse2.S index 49a5fe6c25..a3725cbba7 100644 --- a/kernel/x86_64/trsm_kernel_RT_4x4_sse2.S +++ b/kernel/x86_64/trsm_kernel_RT_4x4_sse2.S @@ -297,10 +297,10 @@ #endif testq $1, N - je .L40 + je L(40) ALIGN_4 -.L81: +L(81): /* Copying to Sub Buffer */ #ifdef LN @@ -337,10 +337,10 @@ subq KK, %rax #endif sarq $3, %rax - jle .L83 + jle L(83) ALIGN_4 -.L82: +L(82): PREFETCH 56 * SIZE(B) movsd 0 * SIZE(B), %xmm0 @@ -373,10 +373,10 @@ movsd %xmm7, -1 * SIZE(BO) decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L83: +L(83): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -385,10 +385,10 @@ #endif andq $7, %rax BRANCH - jle .L90 + jle L(90) ALIGN_4 -.L84: +L(84): movsd 0 * SIZE(B), %xmm0 movsd %xmm0, 0 * SIZE(BO) @@ -397,10 +397,10 @@ addq $1 * SIZE, B addq $2 * SIZE, BO decq %rax - jne .L84 + jne L(84) ALIGN_4 -.L90: +L(90): #if defined(LT) || defined(RN) movq A, AO #else @@ -418,10 +418,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -464,10 +464,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(AO), %xmm9 @@ -524,10 +524,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -536,10 +536,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_4 -.L96: +L(96): mulpd %xmm9, %xmm8 mulpd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -550,10 +550,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L99: +L(99): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -755,15 +755,15 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -801,10 +801,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L105 + je L(105) ALIGN_4 -.L102: +L(102): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd 2 * SIZE(AO), %xmm8 @@ -837,10 +837,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -849,10 +849,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_4 -.L106: +L(106): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(AO), %xmm8 @@ -861,10 +861,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L106 + jg L(106) ALIGN_4 -.L109: +L(109): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -995,12 +995,12 @@ #endif ALIGN_4 -.L110: +L(110): testq $1, M - je .L119 + je L(119) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1038,10 +1038,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulsd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd 1 * SIZE(AO), %xmm8 @@ -1073,10 +1073,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1085,10 +1085,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd %xmm8, %xmm9 movsd 1 * SIZE(AO), %xmm8 addsd %xmm9, %xmm0 @@ -1097,10 +1097,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addsd %xmm2, %xmm0 addsd %xmm3, %xmm1 addsd %xmm1, %xmm0 @@ -1201,7 +1201,7 @@ #endif ALIGN_4 -.L119: +L(119): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -1224,12 +1224,12 @@ ALIGN_4 -.L40: +L(40): testq $2, N - je .L80 + je L(80) ALIGN_4 -.L41: +L(41): /* Copying to Sub Buffer */ #ifdef LN @@ -1266,10 +1266,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): PREFETCH 56 * SIZE(B) movsd 0 * SIZE(B), %xmm0 @@ -1302,10 +1302,10 @@ movsd %xmm7, -1 * SIZE(BO) decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1314,10 +1314,10 @@ #endif andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movsd 0 * SIZE(B), %xmm0 movsd 1 * SIZE(B), %xmm1 @@ -1329,10 +1329,10 @@ addq $2 * SIZE, B addq $4 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): #if defined(LT) || defined(RN) movq A, AO #else @@ -1352,10 +1352,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1401,10 +1401,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(BO), %xmm8 @@ -1516,10 +1516,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1528,10 +1528,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 @@ -1548,10 +1548,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1816,15 +1816,15 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -1865,10 +1865,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(BO), %xmm8 @@ -1930,10 +1930,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1942,10 +1942,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_4 -.L66: +L(66): mulpd %xmm8, %xmm9 mulpd 2 * SIZE(BO), %xmm8 addpd %xmm9, %xmm0 @@ -1956,10 +1956,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L69: +L(69): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -2128,12 +2128,12 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -2174,10 +2174,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulsd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulsd 2 * SIZE(BO), %xmm8 @@ -2238,10 +2238,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2250,10 +2250,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulsd %xmm8, %xmm9 mulsd 2 * SIZE(BO), %xmm8 addsd %xmm9, %xmm0 @@ -2264,10 +2264,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addsd %xmm2, %xmm0 addsd %xmm3, %xmm1 @@ -2386,7 +2386,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2408,12 +2408,12 @@ #endif ALIGN_4 -.L80: +L(80): movq N, J sarq $2, J # j = (n >> 2) - jle .L999 + jle L(999) -.L01: +L(01): /* Copying to Sub Buffer */ #ifdef LN @@ -2450,12 +2450,12 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): PREFETCHNTA 40 * SIZE(B) movsd 0 * SIZE(B), %xmm0 @@ -2488,10 +2488,10 @@ movsd %xmm7, -1 * SIZE(BO) decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2500,10 +2500,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movsd 0 * SIZE(B), %xmm0 movsd 1 * SIZE(B), %xmm1 movsd 2 * SIZE(B), %xmm2 @@ -2521,10 +2521,10 @@ addq $4 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -2544,10 +2544,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2600,8 +2600,8 @@ #endif andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -2619,7 +2619,7 @@ KERNEL7(16 * 1) KERNEL8(16 * 1) cmpq $64 * 2, %rax - jle .L12 + jle L(12) KERNEL1(16 * 2) KERNEL2(16 * 2) KERNEL3(16 * 2) @@ -2637,7 +2637,7 @@ KERNEL7(16 * 3) KERNEL8(16 * 3) cmpq $64 * 4, %rax - jle .L12 + jle L(12) KERNEL1(16 * 4) KERNEL2(16 * 4) KERNEL3(16 * 4) @@ -2655,7 +2655,7 @@ KERNEL7(16 * 5) KERNEL8(16 * 5) cmpq $64 * 6, %rax - jle .L12 + jle L(12) KERNEL1(16 * 6) KERNEL2(16 * 6) KERNEL3(16 * 6) @@ -2676,14 +2676,14 @@ addq $16 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $64 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 4), BO # * 64 ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2692,10 +2692,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -2724,10 +2724,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3239,18 +3239,18 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $3, M - je .L39 + je L(39) testq $2, M - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -3291,10 +3291,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3404,10 +3404,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3416,10 +3416,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -3436,10 +3436,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3709,12 +3709,12 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -3755,10 +3755,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulsd %xmm8, %xmm9 addsd %xmm9, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3867,10 +3867,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3879,10 +3879,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulsd %xmm8, %xmm9 addsd %xmm9, %xmm0 movsd 2 * SIZE(BO), %xmm9 @@ -3899,10 +3899,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -4078,7 +4078,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -4100,10 +4100,10 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/trsm_kernel_RT_4x4_sse3.S b/kernel/x86_64/trsm_kernel_RT_4x4_sse3.S index b6c56e056b..c4601cb7ab 100644 --- a/kernel/x86_64/trsm_kernel_RT_4x4_sse3.S +++ b/kernel/x86_64/trsm_kernel_RT_4x4_sse3.S @@ -396,7 +396,7 @@ #endif testq $1, N - je .L80 + je L(80) ALIGN_4 #if defined(LT) || defined(RN) @@ -432,10 +432,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -474,10 +474,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(AO), %xmm9 @@ -532,10 +532,10 @@ addq $32 * SIZE, AO addq $8 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -544,10 +544,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_4 -.L96: +L(96): mulpd %xmm9, %xmm8 mulpd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -558,10 +558,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L99: +L(99): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -744,15 +744,15 @@ #endif decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -785,10 +785,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L105 + je L(105) ALIGN_4 -.L102: +L(102): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movddup 1 * SIZE(BO), %xmm9 @@ -820,10 +820,10 @@ addq $16 * SIZE, AO addq $ 8 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -832,10 +832,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_4 -.L106: +L(106): mulpd %xmm9, %xmm8 movddup 1 * SIZE(BO), %xmm9 addpd %xmm8, %xmm0 @@ -844,10 +844,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L106 + jg L(106) ALIGN_4 -.L109: +L(109): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -963,12 +963,12 @@ #endif ALIGN_4 -.L110: +L(110): testq $1, M - je .L119 + je L(119) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1001,10 +1001,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulpd %xmm9, %xmm8 movapd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -1023,10 +1023,10 @@ addq $8 * SIZE, AO addq $8 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1035,10 +1035,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd 0 * SIZE(BO), %xmm9 addsd %xmm9, %xmm0 movsd 1 * SIZE(AO), %xmm9 @@ -1046,10 +1046,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addpd %xmm1, %xmm0 haddpd %xmm0, %xmm0 @@ -1138,7 +1138,7 @@ #endif ALIGN_4 -.L119: +L(119): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -1157,9 +1157,9 @@ #endif ALIGN_2 -.L80: +L(80): testq $2, N - je .L40 + je L(40) ALIGN_4 #if defined(LT) || defined(RN) @@ -1196,10 +1196,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1240,10 +1240,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1362,10 +1362,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1374,10 +1374,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 addpd %xmm9, %xmm0 @@ -1396,10 +1396,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1616,15 +1616,15 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $1 + BASE_SHIFT, %rax @@ -1657,10 +1657,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1722,10 +1722,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1734,10 +1734,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_4 -.L66: +L(66): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -1749,10 +1749,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L69: +L(69): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -1888,12 +1888,12 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -1926,10 +1926,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movddup 1 * SIZE(AO), %xmm8 @@ -1961,10 +1961,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1973,10 +1973,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulpd %xmm8, %xmm9 movddup 1 * SIZE(AO), %xmm8 addpd %xmm9, %xmm0 @@ -1985,10 +1985,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -2105,7 +2105,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2124,13 +2124,13 @@ #endif ALIGN_4 -.L40: +L(40): movq N, J sarq $2, J # j = (n >> 2) - jle .L999 + jle L(999) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -2165,10 +2165,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax @@ -2219,8 +2219,8 @@ #if 1 andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1 (16 * 0) KERNEL2 (16 * 0) KERNEL3 (16 * 0) @@ -2239,7 +2239,7 @@ KERNEL16(16 * 0) cmpq $128 * 1, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 1) KERNEL2 (16 * 1) KERNEL3 (16 * 1) @@ -2258,7 +2258,7 @@ KERNEL16(16 * 1) cmpq $128 * 2, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 2) KERNEL2 (16 * 2) KERNEL3 (16 * 2) @@ -2277,7 +2277,7 @@ KERNEL16(16 * 2) cmpq $128 * 3, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 3) KERNEL2 (16 * 3) KERNEL3 (16 * 3) @@ -2296,7 +2296,7 @@ KERNEL16(16 * 3) cmpq $128 * 4, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 4) KERNEL2 (16 * 4) KERNEL3 (16 * 4) @@ -2315,7 +2315,7 @@ KERNEL16(16 * 4) cmpq $128 * 5, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 5) KERNEL2 (16 * 5) KERNEL3 (16 * 5) @@ -2334,7 +2334,7 @@ KERNEL16(16 * 5) cmpq $128 * 6, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 6) KERNEL2 (16 * 6) KERNEL3 (16 * 6) @@ -2353,7 +2353,7 @@ KERNEL16(16 * 6) cmpq $128 * 7, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 7) KERNEL2 (16 * 7) KERNEL3 (16 * 7) @@ -2374,18 +2374,18 @@ addq $32 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $128 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 2), BO # * 64 #else sarq $3, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -2612,11 +2612,11 @@ addq $32 * SIZE, AO decq %rax BRANCH - jne .L12 + jne L(12) #endif ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2625,10 +2625,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 addpd %xmm9, %xmm0 @@ -2659,10 +2659,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax @@ -3068,16 +3068,16 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax @@ -3111,10 +3111,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -3224,10 +3224,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3236,10 +3236,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -3257,10 +3257,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3474,12 +3474,12 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $0 + BASE_SHIFT, %rax @@ -3513,10 +3513,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -3578,10 +3578,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3590,10 +3590,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -3605,10 +3605,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax @@ -3793,7 +3793,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -3811,11 +3811,11 @@ #endif decq J # j -- - jg .L10 + jg L(10) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_RT_4x8_nehalem.S b/kernel/x86_64/trsm_kernel_RT_4x8_nehalem.S index 4942f46710..326bc0ee36 100644 --- a/kernel/x86_64/trsm_kernel_RT_4x8_nehalem.S +++ b/kernel/x86_64/trsm_kernel_RT_4x8_nehalem.S @@ -164,7 +164,7 @@ #endif testq $1, N - jle .L40 + jle L(40) #if defined(LT) || defined(RN) movq A, AO @@ -199,10 +199,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L110 + jle L(110) ALIGN_4 -.L101: +L(101): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -233,10 +233,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_3 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -267,10 +267,10 @@ subq $ -4 * SIZE, BO subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_3 -.L105: +L(105): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -279,10 +279,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L108 + je L(108) ALIGN_3 -.L106: +L(106): addps %xmm1, %xmm8 pshufd $0x00, %xmm3, %xmm1 movss -31 * SIZE(BO), %xmm3 @@ -294,10 +294,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_3 -.L108: +L(108): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -465,13 +465,13 @@ decq I BRANCH - jg .L101 + jg L(101) ALIGN_4 -.L110: +L(110): testq $2, M BRANCH - jle .L120 + jle L(120) #ifdef LN movq K, %rax @@ -501,10 +501,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_3 -.L112: +L(112): addps %xmm1, %xmm8 movss -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -534,10 +534,10 @@ subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_3 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -546,10 +546,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_3 -.L116: +L(116): addps %xmm1, %xmm8 movss -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -561,10 +561,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_3 -.L118: +L(118): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -668,10 +668,10 @@ #endif ALIGN_4 -.L120: +L(120): testq $1, M BRANCH - jle .L129 + jle L(129) #ifdef LN movq K, %rax @@ -701,10 +701,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L125 + jle L(125) ALIGN_3 -.L122: +L(122): addss %xmm2, %xmm8 movss -32 * SIZE(BO), %xmm2 mulss %xmm0, %xmm2 @@ -730,10 +730,10 @@ subq $1, %rax BRANCH - jg .L122 + jg L(122) ALIGN_3 -.L125: +L(125): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -742,10 +742,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_3 -.L126: +L(126): addss %xmm2, %xmm8 movss -32 * SIZE(BO), %xmm2 mulss %xmm0, %xmm2 @@ -756,10 +756,10 @@ subq $1, %rax BRANCH - jg .L126 + jg L(126) ALIGN_3 -.L128: +L(128): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -832,7 +832,7 @@ #endif ALIGN_4 -.L129: +L(129): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -850,9 +850,9 @@ #endif ALIGN_4 -.L40: +L(40): testq $2, N - jle .L70 + jle L(70) #if defined(LT) || defined(RN) movq A, AO @@ -889,10 +889,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L80 + jle L(80) ALIGN_4 -.L71: +L(71): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -927,10 +927,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -973,10 +973,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_3 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -985,10 +985,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addps %xmm1, %xmm8 pshufd $0x00, %xmm3, %xmm1 mulps %xmm0, %xmm1 @@ -1003,10 +1003,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_3 -.L78: +L(78): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1210,13 +1210,13 @@ decq I BRANCH - jg .L71 + jg L(71) ALIGN_4 -.L80: +L(80): testq $2, M BRANCH - jle .L90 + jle L(90) #ifdef LN movq K, %rax @@ -1249,10 +1249,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): addps %xmm1, %xmm8 movsd -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -1282,10 +1282,10 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) ALIGN_3 -.L85: +L(85): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1294,10 +1294,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): addps %xmm1, %xmm8 movsd -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -1309,10 +1309,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_3 -.L88: +L(88): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1439,10 +1439,10 @@ #endif ALIGN_4 -.L90: +L(90): testq $1, M BRANCH - jle .L99 + jle L(99) #ifdef LN movq K, %rax @@ -1473,10 +1473,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_3 -.L92: +L(92): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movsd -32 * SIZE(BO), %xmm2 @@ -1504,11 +1504,11 @@ subq $1, %rax BRANCH - jg .L92 + jg L(92) addps %xmm9, %xmm8 ALIGN_3 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1517,10 +1517,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_3 -.L96: +L(96): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -1532,10 +1532,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_3 -.L98: +L(98): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1641,7 +1641,7 @@ #endif ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -1659,9 +1659,9 @@ #endif ALIGN_4 -.L70: +L(70): testq $4, N - jle .L100 + jle L(100) #if defined(LT) || defined(RN) movq A, AO @@ -1698,10 +1698,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L50 + jle L(50) ALIGN_4 -.L41: +L(41): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -1741,10 +1741,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -1811,10 +1811,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1823,10 +1823,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): addps %xmm1, %xmm8 movaps -32 * SIZE(BO), %xmm1 addps %xmm2, %xmm9 @@ -1847,10 +1847,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2152,13 +2152,13 @@ decq I BRANCH - jg .L41 + jg L(41) ALIGN_4 -.L50: +L(50): testq $2, M BRANCH - jle .L60 + jle L(60) #ifdef LN movq K, %rax @@ -2191,10 +2191,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2236,10 +2236,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2248,10 +2248,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2266,10 +2266,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2473,10 +2473,10 @@ #endif ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L69 + jle L(69) #ifdef LN movq K, %rax @@ -2507,10 +2507,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movaps -32 * SIZE(BO), %xmm2 @@ -2538,11 +2538,11 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) addps %xmm9, %xmm8 ALIGN_3 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2551,10 +2551,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -2566,10 +2566,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2743,7 +2743,7 @@ #endif ALIGN_4 -.L69: +L(69): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -2761,14 +2761,14 @@ #endif ALIGN_4 -.L100: +L(100): movq N, J sarq $3, J NOBRANCH - jle .L999 + jle L(999) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -2808,10 +2808,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + BASE_SHIFT, %rax @@ -2865,10 +2865,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm12 @@ -2995,10 +2995,10 @@ subq $-16 * SIZE, AO subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3007,10 +3007,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addps %xmm1, %xmm12 movaps -32 * SIZE(BO), %xmm1 addps %xmm2, %xmm13 @@ -3046,10 +3046,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3669,13 +3669,13 @@ decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) #ifdef LN movq K, %rax @@ -3713,10 +3713,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -3794,10 +3794,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3806,10 +3806,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -3833,10 +3833,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -4286,10 +4286,10 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M BRANCH - jle .L39 + jle L(39) #ifdef LN movq K, %rax @@ -4321,10 +4321,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movaps -32 * SIZE(BO), %xmm2 @@ -4364,10 +4364,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4376,10 +4376,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -4394,10 +4394,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -4796,7 +4796,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 8), B @@ -4815,10 +4815,10 @@ subq $1, J BRANCH - jg .L10 + jg L(10) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/trsm_kernel_RT_8x4_sse.S b/kernel/x86_64/trsm_kernel_RT_8x4_sse.S index c854b93088..831c301d45 100644 --- a/kernel/x86_64/trsm_kernel_RT_8x4_sse.S +++ b/kernel/x86_64/trsm_kernel_RT_8x4_sse.S @@ -186,7 +186,7 @@ #endif testq $1, N - je .L50 + je L(50) #ifdef LN movq OFFSET, %rax @@ -222,10 +222,10 @@ subq KK, %rax #endif sarq $3, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movsd 0 * SIZE(B), %xmm3 movhps 2 * SIZE(B), %xmm3 movsd 4 * SIZE(B), %xmm7 @@ -254,10 +254,10 @@ addq $32 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -266,10 +266,10 @@ #endif andq $7, %rax BRANCH - jle .L110 + jle L(110) ALIGN_4 -.L104: +L(104): movss 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -279,10 +279,10 @@ addq $ 1 * SIZE, B addq $ 4 * SIZE, BO decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L110: +L(110): #if defined(LT) || defined(RN) movq A, AO #else @@ -300,10 +300,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L120 + jle L(120) ALIGN_4 -.L111: +L(111): #ifdef LN movq K, %rax salq $3 + BASE_SHIFT, %rax @@ -349,10 +349,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulps %xmm9, %xmm8 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -425,10 +425,10 @@ addq $64 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -437,10 +437,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulps %xmm9, %xmm8 mulps 4 * SIZE(AO), %xmm9 addps %xmm8, %xmm0 @@ -451,10 +451,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -878,12 +878,12 @@ #endif decq I # i -- - jg .L111 + jg L(111) ALIGN_4 -.L120: +L(120): testq $4, M - je .L130 + je L(130) #ifdef LN movq K, %rax @@ -924,10 +924,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L125 + je L(125) ALIGN_4 -.L122: +L(122): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -965,10 +965,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L122 + jne L(122) ALIGN_4 -.L125: +L(125): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -977,10 +977,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): mulps %xmm8, %xmm9 movaps 4 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -989,10 +989,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): addps %xmm1, %xmm0 addps %xmm3, %xmm2 addps %xmm2, %xmm0 @@ -1191,9 +1191,9 @@ #endif ALIGN_4 -.L130: +L(130): testq $2, M - je .L140 + je L(140) #ifdef LN movq K, %rax @@ -1236,10 +1236,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L135 + je L(135) ALIGN_4 -.L132: +L(132): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1285,10 +1285,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L132 + jne L(132) ALIGN_4 -.L135: +L(135): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1297,10 +1297,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): mulps %xmm8, %xmm9 movsd 2 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -1309,10 +1309,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): addps %xmm1, %xmm0 #if defined(LN) || defined(RT) @@ -1441,9 +1441,9 @@ #endif ALIGN_4 -.L140: +L(140): testq $1, M - je .L149 + je L(149) #ifdef LN movq K, %rax @@ -1483,10 +1483,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L145 + je L(145) ALIGN_4 -.L142: +L(142): mulss %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1520,10 +1520,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L142 + jne L(142) ALIGN_4 -.L145: +L(145): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1532,10 +1532,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): mulss %xmm8, %xmm9 movss 1 * SIZE(AO), %xmm8 addss %xmm9, %xmm0 @@ -1544,10 +1544,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): addss %xmm1, %xmm0 addss %xmm3, %xmm2 addss %xmm2, %xmm0 @@ -1632,7 +1632,7 @@ #endif ALIGN_4 -.L149: +L(149): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 1), B @@ -1654,9 +1654,9 @@ #endif ALIGN_4 -.L50: +L(50): testq $2, N - je .L100 + je L(100) #ifdef LN movq OFFSET, %rax @@ -1692,10 +1692,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 @@ -1722,10 +1722,10 @@ addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1734,10 +1734,10 @@ #endif andq $3, %rax BRANCH - jle .L60 + jle L(60) ALIGN_4 -.L54: +L(54): movsd 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -1749,10 +1749,10 @@ addq $2 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L54 + jne L(54) ALIGN_4 -.L60: +L(60): #if defined(LT) || defined(RN) movq A, AO #else @@ -1771,10 +1771,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L70 + jle L(70) ALIGN_4 -.L61: +L(61): #ifdef LN movq K, %rax salq $3 + BASE_SHIFT, %rax @@ -1821,10 +1821,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1944,10 +1944,10 @@ addq $64 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1956,10 +1956,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -1976,10 +1976,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2497,12 +2497,12 @@ #endif decq I # i -- - jg .L61 + jg L(61) ALIGN_4 -.L70: +L(70): testq $4, M - je .L80 + je L(80) #ifdef LN movq K, %rax @@ -2545,10 +2545,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2612,10 +2612,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2624,10 +2624,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -2638,10 +2638,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -2889,9 +2889,9 @@ #endif ALIGN_4 -.L80: +L(80): testq $2, M - je .L90 + je L(90) #ifdef LN movq K, %rax @@ -2952,10 +2952,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L85 + je L(85) ALIGN_4 -.L82: +L(82): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -3026,10 +3026,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L85: +L(85): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3038,10 +3038,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -3053,10 +3053,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -3226,9 +3226,9 @@ #endif ALIGN_4 -.L90: +L(90): testq $1, M - je .L99 + je L(99) #ifdef LN movq K, %rax @@ -3270,10 +3270,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -3344,10 +3344,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3356,10 +3356,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -3371,10 +3371,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): addss %xmm2, %xmm0 addss %xmm3, %xmm1 @@ -3500,7 +3500,7 @@ #endif ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -3522,12 +3522,12 @@ #endif ALIGN_4 -.L100: +L(100): movq N, J sarq $2, J # j = (n >> 2) - jle .L999 + jle L(999) -.L01: +L(01): /* Copying to Sub Buffer */ #ifdef LN @@ -3564,10 +3564,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 movaps 8 * SIZE(B), %xmm11 @@ -3615,10 +3615,10 @@ addq $64 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3627,10 +3627,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movaps 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -3646,10 +3646,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -3669,10 +3669,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $3 + BASE_SHIFT, %rax @@ -3724,10 +3724,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 0 * SIZE(BO), %xmm9 @@ -3828,10 +3828,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3840,9 +3840,9 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -3871,10 +3871,10 @@ addq $8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -4537,12 +4537,12 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M - je .L30 + je L(30) #ifdef LN movq K, %rax @@ -4585,10 +4585,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -4702,10 +4702,10 @@ addq $ 32 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -4714,10 +4714,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -4734,10 +4734,10 @@ addq $ 4 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -5061,9 +5061,9 @@ #endif ALIGN_4 -.L30: +L(30): testq $2, M - je .L40 + je L(40) #ifdef LN movq K, %rax @@ -5112,10 +5112,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -5234,10 +5234,10 @@ addq $ 16 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -5246,10 +5246,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -5267,10 +5267,10 @@ addq $ 2 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -5515,9 +5515,9 @@ #endif ALIGN_4 -.L40: +L(40): testq $1, M - je .L49 + je L(49) #ifdef LN movq K, %rax @@ -5559,10 +5559,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): mulss %xmm8, %xmm9 addss %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -5681,10 +5681,10 @@ addq $ 8 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -5693,10 +5693,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -5714,10 +5714,10 @@ addq $ 1 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -5918,7 +5918,7 @@ #endif ALIGN_4 -.L49: +L(49): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -5940,11 +5940,11 @@ #endif decq J # j -- - jg .L01 + jg L(01) -.L999: +L(999): movq %rbx, %rsp EMMS movq 0(%rsp), %rbx diff --git a/kernel/x86_64/xdot.S b/kernel/x86_64/xdot.S index c4b4734947..d41846d34d 100644 --- a/kernel/x86_64/xdot.S +++ b/kernel/x86_64/xdot.S @@ -77,7 +77,7 @@ movl STACK_INCY, INCY testl N, N - jle .L88 + jle L(88) sall $ZBASE_SHIFT, INCX sall $ZBASE_SHIFT, INCY @@ -88,16 +88,16 @@ fldz cmpl $2 * SIZE, INCX - jne .L14 + jne L(14) cmpl $2 * SIZE, INCY - jne .L14 + jne L(14) movl N, %eax sarl $1, %eax - jle .L15 + jle L(15) ALIGN_3 -.L16: +L(16): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -146,16 +146,16 @@ addl $4 * SIZE, X addl $4 * SIZE, Y decl %eax - jg .L16 + jg L(16) ALIGN_3 -.L15: +L(15): movl N, %eax andl $1, %eax - jle .L27 + jle L(27) ALIGN_3 -.L22: +L(22): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -174,17 +174,17 @@ FLD 1 * SIZE(Y) fmulp %st, %st(1) faddp %st, %st(4) - jmp .L27 + jmp L(27) ALIGN_3 -.L14: +L(14): movl N, %eax sarl $1, %eax - jle .L30 + jle L(30) ALIGN_3 -.L31: +L(31): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -228,16 +228,16 @@ addl INCY, Y decl %eax - jg .L31 + jg L(31) ALIGN_3 -.L30: +L(30): movl N, %eax andl $1, %eax - jle .L27 + jle L(27) ALIGN_3 -.L37: +L(37): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -258,7 +258,7 @@ faddp %st, %st(4) ALIGN_3 -.L27: +L(27): movl RESULT, %eax #ifndef CONJ @@ -278,7 +278,7 @@ ret ALIGN_3 -.L88: +L(88): movl RESULT, %eax fldz diff --git a/kernel/x86_64/xgemm3m_kernel_2x2.S b/kernel/x86_64/xgemm3m_kernel_2x2.S index 1d0b23c402..1d6c16e081 100644 --- a/kernel/x86_64/xgemm3m_kernel_2x2.S +++ b/kernel/x86_64/xgemm3m_kernel_2x2.S @@ -106,10 +106,10 @@ movq N, %rax sarq $1, %rax movq %rax, J - je .L30 + je L(30) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -123,10 +123,10 @@ movq M, I sarq $1, I - je .L20 + je L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -167,10 +167,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -251,20 +251,20 @@ addq $8 * SIZE,BO decq %rax - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif and $3, %rax - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -287,10 +287,10 @@ addq $2 * SIZE,BO decq %rax - jne .L16 + jne L(16) ALIGN_4 -.L18: +L(18): #ifndef TRMMKERNEL FLD ALPHA_I FLD ALPHA_R @@ -364,16 +364,16 @@ addq $4 * SIZE, CO decq I - jne .L11 + jne L(11) ALIGN_4 -.L20: +L(20): movq M, %rax andq $1, %rax - je .L29 + je L(29) ALIGN_4 -.L21: +L(21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -404,10 +404,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -454,20 +454,20 @@ addq $8 * SIZE,BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif and $3, %rax - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -482,10 +482,10 @@ addq $2 * SIZE,BO decq %rax - jne .L26 + jne L(26) ALIGN_4 -.L28: +L(28): #ifndef TRMMKERNEL FLD ALPHA_I FLD ALPHA_R @@ -534,20 +534,20 @@ addq $1 * SIZE, CO ALIGN_4 -.L29: +L(29): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif movq BO, B decq J - jne .L01 + jne L(01) ALIGN_4 -.L30: +L(30): movq N, %rax testq $1, %rax - je .L999 + je L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -561,10 +561,10 @@ movq M, I sarq $1, I - je .L40 + je L(40) ALIGN_4 -.L31: +L(31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -601,10 +601,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(BO) @@ -647,20 +647,20 @@ addq $4 * SIZE,BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif and $3, %rax - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): FLD -8 * SIZE(BO) FLD -8 * SIZE(AO) @@ -675,10 +675,10 @@ addq $1 * SIZE,BO decq %rax - jne .L36 + jne L(36) ALIGN_4 -.L38: +L(38): #ifndef TRMMKERNEL FLD ALPHA_I FLD ALPHA_R @@ -726,16 +726,16 @@ addq $4 * SIZE, CO decq I - jne .L31 + jne L(31) ALIGN_4 -.L40: +L(40): movq M, %rax andq $1, %rax - je .L49 + je L(49) ALIGN_4 -.L41: +L(41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -765,10 +765,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -795,20 +795,20 @@ addq $4 * SIZE,BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif and $3, %rax - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -819,10 +819,10 @@ addq $1 * SIZE,BO decq %rax - jne .L46 + jne L(46) ALIGN_4 -.L48: +L(48): #ifndef TRMMKERNEL FLD ALPHA_I FLD ALPHA_R @@ -858,7 +858,7 @@ addq $1 * SIZE, CO ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addq $1, KK #endif @@ -866,7 +866,7 @@ movq BO, B ALIGN_4 -.L999: +L(999): EMMS movq 0(%rsp), %rbx diff --git a/kernel/x86_64/xgemm_kernel_1x1.S b/kernel/x86_64/xgemm_kernel_1x1.S index ee67d8d430..9b196717c2 100644 --- a/kernel/x86_64/xgemm_kernel_1x1.S +++ b/kernel/x86_64/xgemm_kernel_1x1.S @@ -123,15 +123,15 @@ salq $ZBASE_SHIFT, LDC cmpq $0, M - jle .L999 + jle L(999) movq N, %rax movq %rax, J testq %rax, %rax - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -145,7 +145,7 @@ movq M, I ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -184,10 +184,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -268,20 +268,20 @@ addq $8 * SIZE,BO decq %rax - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif and $3, %rax - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -304,10 +304,10 @@ addq $2 * SIZE,BO decq %rax - jne .L16 + jne L(16) ALIGN_4 -.L18: +L(18): faddp %st, %st(3) faddp %st, %st(1) @@ -352,7 +352,7 @@ addq $2 * SIZE, CO decq I - jne .L11 + jne L(11) #if defined(TRMMKERNEL) && !defined(LEFT) addq $1, KK @@ -360,10 +360,10 @@ movq BO, B decq J - jne .L01 + jne L(01) ALIGN_4 -.L999: +L(999): EMMS movq 0(%rsp), %rbx diff --git a/kernel/x86_64/xgemv_n.S b/kernel/x86_64/xgemv_n.S index b66f28d586..ede06c7677 100644 --- a/kernel/x86_64/xgemv_n.S +++ b/kernel/x86_64/xgemv_n.S @@ -101,9 +101,9 @@ movq $0, IS test M, M - jle .L79 + jle L(79) test N, N - jle .L79 + jle L(79) movq LDA, %rax imulq $P, %rax # P * lda @@ -114,7 +114,7 @@ salq $ZBASE_SHIFT, LDA ALIGN_2 -.L32: +L(32): movq $P, %rax movq N, MIN_N subq IS, MIN_N @@ -126,17 +126,17 @@ leaq (X,XP, 1), XP cmpq $2 * SIZE, INCX - je .L34 + je L(34) movq BUFFER, XP movq XP, X1 movq MIN_N, I sarq $1, I - jle .L35 + jle L(35) ALIGN_2 -.L36: +L(36): FLD 0 * SIZE(X) FLD 1 * SIZE(X) addq INCX,X # x += incx @@ -151,13 +151,13 @@ addq $4 * SIZE, X1 # xp += 4 decq I - jg .L36 + jg L(36) ALIGN_3 -.L35: +L(35): movq MIN_N, I andq $1, I - jle .L34 + jle L(34) FLD 0 * SIZE(X) FLD 1 * SIZE(X) @@ -167,12 +167,12 @@ ALIGN_3 /* Main Routine */ -.L34: +L(34): movq Y, Y1 # c_offset movq M, J # j = m ALIGN_3 -.L61: +L(61): movq A, A1 # a_offset = a addq $2 * SIZE, A # a++ @@ -186,10 +186,10 @@ movq MIN_N, I sarq $1, I - jle .L64 + jle L(64) ALIGN_3 -.L65: +L(65): FLD 0 * SIZE(A1) # at1 = *(a_offset + 0) fmul %st(1) # at1 *= bt1 faddp %st, %st(2) # ct1 += at1 @@ -241,15 +241,15 @@ addq LDA, A1 # a_offset += lda decq I - jg .L65 + jg L(65) -.L64: +L(64): movq MIN_N, I andq $1, I - jle .L70 + jle L(70) ALIGN_2 -.L71: +L(71): FLD 0 * SIZE(A1) # at1 = *(a_offset + 0) fmul %st(1) # at1 *= bt1 faddp %st, %st(2) # ct1 += at1 @@ -273,7 +273,7 @@ fldz ALIGN_2 -.L70: +L(70): ffreep %st(0) #ifndef XCONJ @@ -315,15 +315,15 @@ addq INCY, Y1 decq J - jg .L61 + jg L(61) -.L60: +L(60): addq PLDA_M, A addq $P, IS cmpq N, IS - jl .L32 + jl L(32) -.L79: +L(79): ffreep %st ffreep %st diff --git a/kernel/x86_64/xgemv_t.S b/kernel/x86_64/xgemv_t.S index d6d37010d1..0844f966a2 100644 --- a/kernel/x86_64/xgemv_t.S +++ b/kernel/x86_64/xgemv_t.S @@ -100,9 +100,9 @@ movq $0, IS test M, M - jle .L79 # goto END + jle L(79) # goto END test N, N - jle .L79 # goto END + jle L(79) # goto END movq N, %rax imulq LDA, %rax @@ -113,7 +113,7 @@ salq $ZBASE_SHIFT, LDA ALIGN_2 -.L32: +L(32): movq $P, %rax movq M, MIN_M subq IS , MIN_M @@ -127,17 +127,17 @@ movq X1, XP cmpq $2 * SIZE, INCX - je .L34 + je L(34) movq BUFFER, X1 movq X1, XP movq MIN_M, I sarq $1, I - jle .L35 + jle L(35) ALIGN_3 -.L36: +L(36): FLD 0 * SIZE(X) FLD 1 * SIZE(X) addq INCX,X # x += incx @@ -152,13 +152,13 @@ addq $4 * SIZE, X1 # xp += 4 decq I - jg .L36 + jg L(36) ALIGN_3 -.L35: +L(35): movq MIN_M, I andq $1,I - jle .L34 + jle L(34) FLD 0 * SIZE(X) FLD 1 * SIZE(X) @@ -169,13 +169,13 @@ /* Main Routine */ -.L34: +L(34): movq Y, Y1 # coffset = y movq N, J ALIGN_2 -.L61: +L(61): movq A, A1 # a_offset = a fldz # ct1 = ZERO fldz # ct1 = ZERO @@ -190,10 +190,10 @@ movq MIN_M, I sarq $1, I - jle .L64 + jle L(64) ALIGN_3 -.L65: +L(65): FLD 0 * SIZE(A1) # at1 = *(a_offset + 0) fmul %st(1) # at1 *= bt1 faddp %st, %st(2) # ct1 += at1 @@ -241,16 +241,16 @@ addq $4 * SIZE, X1 addq $4 * SIZE, A1 decq I - jg .L65 + jg L(65) ALIGN_3 -.L64: +L(64): movq MIN_M, I andq $1, I - jle .L70 + jle L(70) ALIGN_3 -.L71: +L(71): FLD 0 * SIZE(A1) # at1 = *(a_offset + 0) fmul %st(1) # at1 *= bt1 faddp %st, %st(2) # ct1 += at1 @@ -274,7 +274,7 @@ fldz ALIGN_3 -.L70: +L(70): ffreep %st(0) #ifndef XCONJ @@ -316,18 +316,18 @@ addq INCY, Y1 decq J - jg .L61 + jg L(61) ALIGN_3 -.L60: +L(60): addq NLDA, A addq $P, IS cmpq M, IS - jl .L32 + jl L(32) ALIGN_3 -.L79: +L(79): ffreep %st ffreep %st diff --git a/kernel/x86_64/xtrsm_kernel_LT_1x1.S b/kernel/x86_64/xtrsm_kernel_LT_1x1.S index 875206363f..3f08dd0a55 100644 --- a/kernel/x86_64/xtrsm_kernel_LT_1x1.S +++ b/kernel/x86_64/xtrsm_kernel_LT_1x1.S @@ -141,15 +141,15 @@ #endif cmpq $0, M - jle .L999 + jle L(999) movq N, %rax movq %rax, J testq %rax, %rax - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -183,7 +183,7 @@ movq M, I ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -218,10 +218,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) FLD -8 * SIZE(AO) @@ -302,10 +302,10 @@ addq $8 * SIZE,BO decq %rax - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -313,10 +313,10 @@ subq KK, %rax #endif and $3, %rax - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): FLD -8 * SIZE(AO) FLD -8 * SIZE(BO) @@ -339,10 +339,10 @@ addq $2 * SIZE,BO decq %rax - jne .L16 + jne L(16) ALIGN_4 -.L18: +L(18): faddp %st, %st(3) faddp %st, %st(1) @@ -453,7 +453,7 @@ #endif decq I - jne .L11 + jne L(11) #ifdef LN movq K, %rax @@ -474,10 +474,10 @@ #endif decq J - jne .L01 + jne L(01) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zamax.S b/kernel/x86_64/zamax.S index 5cb2f60198..9c8f576020 100644 --- a/kernel/x86_64/zamax.S +++ b/kernel/x86_64/zamax.S @@ -64,9 +64,9 @@ fldz testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) ffreep %st @@ -77,17 +77,17 @@ faddp %st, %st(1) addq INCX, X decq M - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX - jne .L40 + jne L(40) movq M, I sarq $2, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -135,17 +135,17 @@ addq $8 * SIZE, X decq I - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): movq M, I andq $3, I - jle .L999 + jle L(999) ALIGN_4 -.L21: +L(21): FLD 0 * SIZE(X) fabs FLD 1 * SIZE(X) @@ -158,17 +158,17 @@ addq $2 * SIZE, X decq I - jg .L21 - jmp .L999 + jg L(21) + jmp L(999) ALIGN_4 -.L40: +L(40): movq M, I sarq $2, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): FLD 0 * SIZE(X) fabs FLD 1 * SIZE(X) @@ -214,16 +214,16 @@ ffreep %st decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): movq M, I andq $3, I - jle .L999 + jle L(999) ALIGN_4 -.L61: +L(61): FLD 0 * SIZE(X) fabs FLD 1 * SIZE(X) @@ -236,10 +236,10 @@ addq INCX, X decq I - jg .L61 + jg L(61) ALIGN_4 -.L999: +L(999): ret EPILOGUE diff --git a/kernel/x86_64/zamax_atom.S b/kernel/x86_64/zamax_atom.S index 8b4e144f05..c23b8df848 100644 --- a/kernel/x86_64/zamax_atom.S +++ b/kernel/x86_64/zamax_atom.S @@ -60,10 +60,10 @@ salq $ZBASE_SHIFT, INCX testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) pcmpeqb %xmm15, %xmm15 psrlq $1, %xmm15 @@ -77,16 +77,16 @@ addsd %xmm4, %xmm0 decq M - jle .L999 + jle L(999) movaps %xmm0, %xmm1 cmpq $2 * SIZE, INCX - jne .L20 + jne L(20) movq M, I sarq $2, I - jle .L15 + jle L(15) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -101,10 +101,10 @@ addsd %xmm4, %xmm5 movsd 7 * SIZE(X), %xmm11 decq I - jle .L13 + jle L(13) ALIGN_4 -.L12: +L(12): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -139,10 +139,10 @@ addq $8 * SIZE, X decq I - jg .L12 + jg L(12) ALIGN_4 -.L13: +L(13): andps %xmm15, %xmm6 andps %xmm15, %xmm7 addsd %xmm6, %xmm7 @@ -163,9 +163,9 @@ addq $8 * SIZE, X ALIGN_4 -.L15: +L(15): testq $2, M - jle .L17 + jle L(17) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -185,9 +185,9 @@ maxsd %xmm7, %xmm1 ALIGN_3 -.L17: +L(17): testq $1, M - jle .L998 + jle L(998) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -197,13 +197,13 @@ addsd %xmm4, %xmm5 maxsd %xmm5, %xmm0 - jmp .L998 + jmp L(998) ALIGN_3 -.L20: +L(20): movq M, I sarq $2, I - jle .L25 + jle L(25) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -223,10 +223,10 @@ addq INCX, X decq I - jle .L23 + jle L(23) ALIGN_4 -.L22: +L(22): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -264,10 +264,10 @@ addsd %xmm4, %xmm5 decq I - jg .L22 + jg L(22) ALIGN_4 -.L23: +L(23): andps %xmm15, %xmm6 andps %xmm15, %xmm7 addsd %xmm6, %xmm7 @@ -286,9 +286,9 @@ maxsd %xmm11, %xmm1 ALIGN_4 -.L25: +L(25): testq $2, M - jle .L27 + jle L(27) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -309,9 +309,9 @@ maxsd %xmm7, %xmm1 ALIGN_3 -.L27: +L(27): testq $1, M - jle .L998 + jle L(998) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -323,11 +323,11 @@ maxsd %xmm5, %xmm0 ALIGN_3 -.L998: +L(998): maxsd %xmm1, %xmm0 ALIGN_4 -.L999: +L(999): RESTOREREGISTERS diff --git a/kernel/x86_64/zamax_sse.S b/kernel/x86_64/zamax_sse.S index 5f8a1f1c6d..b3caefd666 100644 --- a/kernel/x86_64/zamax_sse.S +++ b/kernel/x86_64/zamax_sse.S @@ -61,7 +61,7 @@ salq $ZBASE_SHIFT, INCX testq M, M - jle .L999 + jle L(999) pcmpeqb %xmm15, %xmm15 psrld $1, %xmm15 @@ -76,15 +76,15 @@ shufps $0, %xmm0, %xmm0 movaps %xmm0, %xmm1 cmpq $2 * SIZE, INCX - jne .L40 + jne L(40) -.L30: +L(30): movq M, I sarq $3, I - jle .L35 + jle L(35) ALIGN_4 -.L31: +L(31): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -120,15 +120,15 @@ addq $16 * SIZE, X decq I - jg .L31 + jg L(31) ALIGN_4 -.L35: +L(35): andq $7, M - jle .L998 + jle L(998) testq $4, M - je .L36 + je L(36) movsd 0 * SIZE(X), %xmm4 movhps 2 * SIZE(X), %xmm4 @@ -147,9 +147,9 @@ addq $8 * SIZE, X ALIGN_3 -.L36: +L(36): testq $2, M - je .L37 + je L(37) movss 0 * SIZE(X), %xmm4 movss 1 * SIZE(X), %xmm5 @@ -166,9 +166,9 @@ addq $4 * SIZE, X ALIGN_3 -.L37: +L(37): testq $1, M - je .L998 + je L(998) movss 0 * SIZE(X), %xmm4 movss 1 * SIZE(X), %xmm5 @@ -176,17 +176,17 @@ andps %xmm15, %xmm5 addps %xmm5, %xmm4 maxss %xmm4, %xmm0 - jmp .L998 + jmp L(998) ALIGN_4 -.L40: +L(40): movq M, I sarq $3, I - jle .L45 + jle L(45) ALIGN_4 -.L41: +L(41): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -229,15 +229,15 @@ maxps %xmm7, %xmm0 decq I - jg .L41 + jg L(41) ALIGN_4 -.L45: +L(45): andq $7, M - jle .L998 + jle L(998) testq $4, M - je .L46 + je L(46) movsd 0 * SIZE(X), %xmm4 addq INCX, X @@ -258,9 +258,9 @@ maxps %xmm4, %xmm0 ALIGN_3 -.L46: +L(46): testq $2, M - je .L47 + je L(47) movss 0 * SIZE(X), %xmm4 movss 1 * SIZE(X), %xmm5 @@ -278,9 +278,9 @@ maxss %xmm6, %xmm1 ALIGN_3 -.L47: +L(47): testq $1, M - je .L998 + je L(998) movss 0 * SIZE(X), %xmm4 movss 1 * SIZE(X), %xmm5 @@ -288,10 +288,10 @@ andps %xmm15, %xmm5 addps %xmm5, %xmm4 maxss %xmm4, %xmm0 - jmp .L998 + jmp L(998) ALIGN_4 -.L998: +L(998): maxps %xmm1, %xmm0 movaps %xmm0, %xmm1 movhlps %xmm0, %xmm0 @@ -301,7 +301,7 @@ maxss %xmm1, %xmm0 ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/zamax_sse2.S b/kernel/x86_64/zamax_sse2.S index bde290be98..4732f0b5c0 100644 --- a/kernel/x86_64/zamax_sse2.S +++ b/kernel/x86_64/zamax_sse2.S @@ -59,9 +59,9 @@ pxor %xmm0, %xmm0 testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) salq $ZBASE_SHIFT, INCX @@ -81,15 +81,15 @@ movapd %xmm0, %xmm3 cmpq $2 * SIZE, INCX - jne .L40 + jne L(40) -.L30: +L(30): movq M, I sarq $3, I - jle .L35 + jle L(35) ALIGN_4 -.L31: +L(31): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -138,15 +138,15 @@ addq $16 * SIZE, X decq I - jg .L31 + jg L(31) ALIGN_4 -.L35: +L(35): andq $7, M - jle .L998 + jle L(998) testq $4, M - je .L36 + je L(36) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -170,9 +170,9 @@ addq $8 * SIZE, X ALIGN_3 -.L36: +L(36): testq $2, M - je .L37 + je L(37) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -186,9 +186,9 @@ maxpd %xmm4, %xmm0 ALIGN_3 -.L37: +L(37): testq $1, M - je .L998 + je L(998) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -196,17 +196,17 @@ andpd %xmm15, %xmm5 addpd %xmm5, %xmm4 maxsd %xmm4, %xmm2 - jmp .L998 + jmp L(998) ALIGN_4 -.L40: +L(40): movq M, I sarq $3, I - jle .L45 + jle L(45) ALIGN_4 -.L41: +L(41): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -262,15 +262,15 @@ maxpd %xmm6, %xmm3 decq I - jg .L41 + jg L(41) ALIGN_4 -.L45: +L(45): andq $7, M - jle .L998 + jle L(998) testq $4, M - je .L46 + je L(46) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -295,9 +295,9 @@ maxpd %xmm6, %xmm1 ALIGN_3 -.L46: +L(46): testq $2, M - je .L47 + je L(47) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -311,9 +311,9 @@ maxpd %xmm4, %xmm2 ALIGN_3 -.L47: +L(47): testq $1, M - je .L998 + je L(998) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -321,10 +321,10 @@ andpd %xmm15, %xmm5 addpd %xmm5, %xmm4 maxsd %xmm4, %xmm3 - jmp .L998 + jmp L(998) ALIGN_4 -.L998: +L(998): maxpd %xmm1, %xmm0 maxpd %xmm3, %xmm2 maxpd %xmm2, %xmm0 @@ -333,7 +333,7 @@ maxsd %xmm1, %xmm0 ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/zasum.S b/kernel/x86_64/zasum.S index 3460fcea30..7b57747eba 100644 --- a/kernel/x86_64/zasum.S +++ b/kernel/x86_64/zasum.S @@ -56,9 +56,9 @@ fldz testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) salq $ZBASE_SHIFT, INCX @@ -66,14 +66,14 @@ fldz fldz cmpq $SIZE * 2, INCX - jne .L40 + jne L(40) movq M, I sarq $2, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -109,16 +109,16 @@ faddp %st, %st(1) decq I - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): andq $3, M - jle .L998 + jle L(998) ALIGN_4 -.L21: +L(21): FLD 0 * SIZE(X) fabs FLD 1 * SIZE(X) @@ -127,17 +127,17 @@ faddp %st,%st(1) addq $2 * SIZE, X decq M - jg .L21 - jmp .L998 + jg L(21) + jmp L(998) ALIGN_4 -.L40: +L(40): movq M, I sarq $2, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): FLD 0 * SIZE(X) fabs FLD 1 * SIZE(X) @@ -171,16 +171,16 @@ faddp %st, %st(1) decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): andq $3, M - jle .L998 + jle L(998) ALIGN_4 -.L61: +L(61): FLD 0 * SIZE(X) fabs FLD 1 * SIZE(X) @@ -189,16 +189,16 @@ faddp %st,%st(3) faddp %st,%st(1) decq M - jg .L61 + jg L(61) ALIGN_4 -.L998: +L(998): faddp %st,%st(2) faddp %st,%st(1) faddp %st,%st(1) ALIGN_4 -.L999: +L(999): ret EPILOGUE diff --git a/kernel/x86_64/zasum_atom.S b/kernel/x86_64/zasum_atom.S index 888dbbb80c..1f63f5b294 100644 --- a/kernel/x86_64/zasum_atom.S +++ b/kernel/x86_64/zasum_atom.S @@ -55,9 +55,9 @@ xorps %xmm0, %xmm0 testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) xorps %xmm1, %xmm1 xorps %xmm2, %xmm2 @@ -69,12 +69,12 @@ xorps %xmm13, %xmm13 cmpq $2 * SIZE, INCX - jne .L20 + jne L(20) addq M, M testq $SIZE, X - je .L05 + je L(05) movsd (X), %xmm0 addq $SIZE, X @@ -82,12 +82,12 @@ decq M ALIGN_3 -.L05: +L(05): subq $-16 * SIZE, X movq M, I sarq $4, I - jle .L12 + jle L(12) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -100,10 +100,10 @@ movaps -2 * SIZE(X), %xmm11 decq I - jle .L11 + jle L(11) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -162,10 +162,10 @@ subq $-16 * SIZE, X decq I - jg .L10 + jg L(10) ALIGN_4 -.L11: +L(11): andps %xmm15, %xmm4 addsd %xmm13, %xmm3 pshufd $0x4e, %xmm4, %xmm12 @@ -210,12 +210,12 @@ subq $-16 * SIZE, X ALIGN_3 -.L12: +L(12): andq $15, M - jle .L998 + jle L(998) testq $8, M - je .L13 + je L(13) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -241,9 +241,9 @@ addsd %xmm13, %xmm3 ALIGN_3 -.L13: +L(13): testq $4, M - je .L14 + je L(14) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -259,9 +259,9 @@ addsd %xmm13, %xmm3 ALIGN_3 -.L14: +L(14): testq $2, M - je .L15 + je L(15) movaps -16 * SIZE(X), %xmm4 addq $2 * SIZE, X @@ -272,20 +272,20 @@ addsd %xmm5, %xmm3 ALIGN_3 -.L15: +L(15): testq $1, M - je .L998 + je L(998) movsd -16 * SIZE(X), %xmm4 andps %xmm15, %xmm4 addsd %xmm4, %xmm0 - jmp .L998 + jmp L(998) ALIGN_3 -.L20: +L(20): movq M, I sarq $2, I - jle .L25 + jle L(25) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -301,10 +301,10 @@ movsd 1 * SIZE(X), %xmm11 decq I - jle .L23 + jle L(23) ALIGN_4 -.L22: +L(22): andps %xmm15, %xmm4 addq INCX, X addsd %xmm4, %xmm0 @@ -336,10 +336,10 @@ movsd 1 * SIZE(X), %xmm11 decq I - jg .L22 + jg L(22) ALIGN_4 -.L23: +L(23): andps %xmm15, %xmm4 addq INCX, X addsd %xmm4, %xmm0 @@ -360,9 +360,9 @@ addsd %xmm11, %xmm3 ALIGN_3 -.L25: +L(25): testq $2, M - je .L26 + je L(26) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -381,9 +381,9 @@ addsd %xmm7, %xmm3 ALIGN_3 -.L26: +L(26): testq $1, M - je .L998 + je L(998) movsd 0 * SIZE(X), %xmm4 movsd 1 * SIZE(X), %xmm5 @@ -396,13 +396,13 @@ addsd %xmm5, %xmm1 ALIGN_3 -.L998: +L(998): addsd %xmm1, %xmm0 addsd %xmm3, %xmm2 addsd %xmm2, %xmm0 ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/zasum_sse.S b/kernel/x86_64/zasum_sse.S index 44d6da5617..3ce7ee275b 100644 --- a/kernel/x86_64/zasum_sse.S +++ b/kernel/x86_64/zasum_sse.S @@ -54,9 +54,9 @@ pxor %xmm0, %xmm0 testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) pxor %xmm1, %xmm1 pxor %xmm2, %xmm2 @@ -68,26 +68,26 @@ salq $ZBASE_SHIFT, INCX cmpq $2 * SIZE, INCX - jne .L100 + jne L(100) subq $-32 * SIZE, X addq M, M cmpq $3, M - jle .L18 + jle L(18) testq $4, X - je .L05 + je L(05) movss -32 * SIZE(X), %xmm0 andps %xmm15, %xmm0 addq $SIZE, X decq M - jle .L998 + jle L(998) ALIGN_3 -.L05: +L(05): testq $8, X - je .L10 + je L(10) #ifdef movsd xorps %xmm1, %xmm1 @@ -96,13 +96,13 @@ andps %xmm15, %xmm1 addq $2 * SIZE, X subq $2, M - jle .L998 + jle L(998) ALIGN_3 -.L10: +L(10): movq M, I sarq $5, I - jle .L14 + jle L(14) movaps -32 * SIZE(X), %xmm4 movaps -28 * SIZE(X), %xmm5 @@ -114,10 +114,10 @@ movaps -8 * SIZE(X), %xmm10 movaps -4 * SIZE(X), %xmm11 decq I - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -160,10 +160,10 @@ subq $-32 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): andps %xmm15, %xmm4 addps %xmm4, %xmm0 andps %xmm15, %xmm5 @@ -187,13 +187,13 @@ addq $32 * SIZE, X ALIGN_3 -.L14: +L(14): testq $31, M - jle .L998 + jle L(998) -.L15: +L(15): testq $16, M - je .L16 + je L(16) movaps -32 * SIZE(X), %xmm4 andps %xmm15, %xmm4 @@ -214,9 +214,9 @@ addq $16 * SIZE, X ALIGN_3 -.L16: +L(16): testq $8, M - je .L17 + je L(17) movaps -32 * SIZE(X), %xmm4 andps %xmm15, %xmm4 @@ -229,9 +229,9 @@ addq $8 * SIZE, X ALIGN_3 -.L17: +L(17): testq $4, M - je .L18 + je L(18) movaps -32 * SIZE(X), %xmm6 andps %xmm15, %xmm6 @@ -239,9 +239,9 @@ addq $4 * SIZE, X ALIGN_3 -.L18: +L(18): testq $2, M - je .L19 + je L(19) #ifdef movsd xorps %xmm7, %xmm7 @@ -252,23 +252,23 @@ addq $2 * SIZE, X ALIGN_3 -.L19: +L(19): testq $1, M - je .L998 + je L(998) movss -32 * SIZE(X), %xmm6 andps %xmm15, %xmm6 addps %xmm6, %xmm2 - jmp .L998 + jmp L(998) ALIGN_4 -.L100: +L(100): movq M, I sarq $2, I - jle .L105 + jle L(105) ALIGN_4 -.L101: +L(101): movsd (X), %xmm4 addq INCX, X movhps (X), %xmm4 @@ -286,27 +286,27 @@ addps %xmm5, %xmm1 decq I - jg .L101 + jg L(101) ALIGN_4 -.L105: +L(105): #ifdef movsd xorps %xmm4, %xmm4 #endif andq $3, M - jle .L998 + jle L(998) ALIGN_4 -.L106: +L(106): movsd (X), %xmm4 andps %xmm15, %xmm4 addps %xmm4, %xmm0 addq INCX, X decq M - jg .L106 + jg L(106) ALIGN_4 -.L998: +L(998): addps %xmm1, %xmm0 addps %xmm3, %xmm2 addps %xmm2, %xmm0 @@ -324,7 +324,7 @@ #endif ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/zasum_sse2.S b/kernel/x86_64/zasum_sse2.S index d1e076c311..d8e424cdb5 100644 --- a/kernel/x86_64/zasum_sse2.S +++ b/kernel/x86_64/zasum_sse2.S @@ -54,9 +54,9 @@ xorps %xmm0, %xmm0 testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) xorps %xmm1, %xmm1 xorps %xmm2, %xmm2 @@ -68,13 +68,13 @@ salq $ZBASE_SHIFT, INCX cmpq $2 * SIZE, INCX - jne .L40 + jne L(40) subq $-16 * SIZE, X addq M, M testq $SIZE, X - je .L05 + je L(05) #ifdef movsd xorps %xmm0, %xmm0 @@ -84,13 +84,13 @@ andps %xmm15, %xmm0 subq $1, M - jle .L999 + jle L(999) ALIGN_3 -.L05: +L(05): movq M, I sarq $4, I - jle .L20 + jle L(20) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -103,10 +103,10 @@ movaps -2 * SIZE(X), %xmm11 decq I - jle .L11 + jle L(11) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -149,10 +149,10 @@ subq $-16 * SIZE, X decq I - jg .L10 + jg L(10) ALIGN_4 -.L11: +L(11): andps %xmm15, %xmm4 andps %xmm15, %xmm5 andps %xmm15, %xmm6 @@ -176,12 +176,12 @@ subq $-16 * SIZE, X ALIGN_3 -.L20: +L(20): andq $15, M - jle .L998 + jle L(998) testq $8, M - je .L21 + je L(21) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -200,9 +200,9 @@ addq $8 * SIZE, X ALIGN_3 -.L21: +L(21): testq $4, M - je .L22 + je L(22) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -215,18 +215,18 @@ addq $4 * SIZE, X ALIGN_3 -.L22: +L(22): testq $2, M - je .L23 + je L(23) movaps -16 * SIZE(X), %xmm6 andps %xmm15, %xmm6 addpd %xmm6, %xmm3 addq $2 * SIZE, X -.L23: +L(23): testq $1, M - je .L998 + je L(998) #ifdef movsd xorps %xmm4, %xmm4 @@ -234,17 +234,17 @@ movsd -16 * SIZE(X), %xmm4 andps %xmm15, %xmm4 addsd %xmm4, %xmm0 - jmp .L998 + jmp L(998) ALIGN_3 -.L40: +L(40): movq M, I sarq $2, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) prefetcht0 PREFETCHSIZE * SIZE(X) #endif @@ -278,26 +278,26 @@ addpd %xmm7, %xmm3 decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): andq $3, M - jle .L998 + jle L(998) ALIGN_4 -.L61: +L(61): movsd 0 * SIZE(X), %xmm4 movhpd 1 * SIZE(X), %xmm4 andpd %xmm15, %xmm4 addpd %xmm4, %xmm0 addq INCX, X decq M - jg .L61 + jg L(61) ALIGN_4 -.L998: +L(998): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -310,7 +310,7 @@ #endif ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/zaxpy.S b/kernel/x86_64/zaxpy.S index 1758ca9431..e0f800db05 100644 --- a/kernel/x86_64/zaxpy.S +++ b/kernel/x86_64/zaxpy.S @@ -70,19 +70,19 @@ salq $ZBASE_SHIFT, INCY testq M, M - jle .L40 + jle L(40) cmpq $2 * SIZE, INCX - jne .L14 + jne L(14) cmpq $2 * SIZE, INCY - jne .L14 + jne L(14) movq M, %rax sarq $2, %rax - jle .L15 + jle L(15) ALIGN_3 -.L16: +L(16): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -166,16 +166,16 @@ addq $8 * SIZE, X addq $8 * SIZE, Y decq %rax - jg .L16 + jg L(16) ALIGN_3 -.L15: +L(15): movq M, %rax andq $3, %rax - jle .L40 + jle L(40) ALIGN_3 -.L22: +L(22): FLD 0 * SIZE(X) fmul %st(1), %st FLD 1 * SIZE(X) @@ -197,17 +197,17 @@ addq $2 * SIZE, X addq $2 * SIZE, Y decq %rax - jg .L22 - jmp .L40 + jg L(22) + jmp L(40) ALIGN_3 -.L14: +L(14): movq M, %rax sarq $2, %rax - jle .L28 + jle L(28) ALIGN_3 -.L29: +L(29): FLD 0 * SIZE(X) fmul %st(1), %st FLD 1 * SIZE(X) @@ -293,16 +293,16 @@ addq INCY, Y decq %rax - jg .L29 + jg L(29) ALIGN_3 -.L28: +L(28): movq M, %rax andq $3, %rax - jle .L40 + jle L(40) ALIGN_3 -.L35: +L(35): FLD 0 * SIZE(X) fmul %st(1), %st FLD 1 * SIZE(X) @@ -325,10 +325,10 @@ addq INCY, Y decq %rax - jg .L35 + jg L(35) ALIGN_3 -.L40: +L(40): ffreep %st(0) ffreep %st(0) ret diff --git a/kernel/x86_64/zaxpy_atom.S b/kernel/x86_64/zaxpy_atom.S index 2fe2756fb9..8073c3c5fd 100644 --- a/kernel/x86_64/zaxpy_atom.S +++ b/kernel/x86_64/zaxpy_atom.S @@ -94,16 +94,16 @@ movaps %xmm1, ALPHA_I testq M, M - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX - jne .L20 + jne L(20) cmpq $2 * SIZE, INCY - jne .L20 + jne L(20) movq M, %rax sarq $2, %rax - jle .L15 + jle L(15) movsd 0 * SIZE(X), %xmm0 movsd 1 * SIZE(X), %xmm1 @@ -133,10 +133,10 @@ movsd 5 * SIZE(X), %xmm1 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -228,10 +228,10 @@ addq $8 * SIZE, X addq $8 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): mulsd ALPHA_I, %xmm7 movsd 4 * SIZE(Y), %xmm12 ADD1 %xmm3, %xmm8 @@ -299,10 +299,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L15: +L(15): movq M, %rax andq $2, %rax - jle .L17 + jle L(17) movsd 0 * SIZE(X), %xmm0 movsd 1 * SIZE(X), %xmm1 @@ -348,10 +348,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L17: +L(17): movq M, %rax andq $1, %rax - jle .L999 + jle L(999) movsd 0 * SIZE(X), %xmm0 movsd 1 * SIZE(X), %xmm1 @@ -372,15 +372,15 @@ movsd %xmm8, 0 * SIZE(Y) movsd %xmm9, 1 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L20: +L(20): movq Y, YY movq M, %rax sarq $2, %rax - jle .L25 + jle L(25) movsd 0 * SIZE(X), %xmm0 movsd 1 * SIZE(X), %xmm1 @@ -415,10 +415,10 @@ addq INCX, X decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): mulsd ALPHA_I, %xmm7 movsd 0 * SIZE(Y), %xmm12 ADD1 %xmm3, %xmm8 @@ -512,10 +512,10 @@ addq INCX, X decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): mulsd ALPHA_I, %xmm7 movsd 0 * SIZE(Y), %xmm12 ADD1 %xmm3, %xmm8 @@ -587,10 +587,10 @@ addq INCY, YY ALIGN_3 -.L25: +L(25): movq M, %rax andq $2, %rax - jle .L27 + jle L(27) movsd 0 * SIZE(X), %xmm0 movsd 1 * SIZE(X), %xmm1 @@ -639,10 +639,10 @@ addq INCY, YY ALIGN_3 -.L27: +L(27): movq M, %rax andq $1, %rax - jle .L999 + jle L(999) movsd 0 * SIZE(X), %xmm0 movsd 1 * SIZE(X), %xmm1 @@ -665,7 +665,7 @@ movsd %xmm9, 1 * SIZE(YY) ALIGN_3 -.L999: +L(999): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/zaxpy_sse.S b/kernel/x86_64/zaxpy_sse.S index 0a12e244a5..f83d7d0982 100644 --- a/kernel/x86_64/zaxpy_sse.S +++ b/kernel/x86_64/zaxpy_sse.S @@ -81,12 +81,12 @@ salq $ZBASE_SHIFT, INCY testq M, M - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX - jne .L100 + jne L(100) cmpq $2 * SIZE, INCY - jne .L100 + jne L(100) pcmpeqb %xmm7, %xmm7 psllq $63, %xmm7 @@ -105,7 +105,7 @@ subq $-32 * SIZE, Y testq $2 * SIZE, Y - je .L10 + je L(10) movsd -32 * SIZE(X), %xmm0 movsd -32 * SIZE(Y), %xmm1 @@ -122,19 +122,19 @@ addq $2 * SIZE, X addq $2 * SIZE, Y decq M - jle .L999 + jle L(999) ALIGN_2 -.L10: +L(10): testq $SIZE, Y - jne .L50 + jne L(50) testq $3 * SIZE, X - jne .L20 + jne L(20) movq M, %rax sarq $4, %rax - jle .L15 + jle L(15) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -142,10 +142,10 @@ movaps -20 * SIZE(X), %xmm3 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): movaps -16 * SIZE(X), %xmm4 movaps -12 * SIZE(X), %xmm5 @@ -233,10 +233,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movaps -16 * SIZE(X), %xmm4 movaps -12 * SIZE(X), %xmm5 @@ -303,9 +303,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $8, M - jle .L16 + jle L(16) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -345,9 +345,9 @@ addq $16 * SIZE, Y ALIGN_2 -.L16: +L(16): testq $4, M - jle .L17 + jle L(17) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -370,9 +370,9 @@ addq $8 * SIZE, Y ALIGN_2 -.L17: +L(17): testq $2, M - jle .L18 + jle L(18) movaps -32 * SIZE(X), %xmm0 @@ -388,9 +388,9 @@ addq $4 * SIZE, Y ALIGN_2 -.L18: +L(18): testq $1, M - jle .L999 + jle L(999) movsd -32 * SIZE(X), %xmm0 @@ -402,14 +402,14 @@ addps %xmm1, %xmm0 addps %xmm8, %xmm0 movlps %xmm0, -32 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L20: +L(20): #ifdef ALIGNED_ACCESS testq $2 * SIZE, X - jne .L30 + jne L(30) subq $1 * SIZE, X @@ -417,7 +417,7 @@ movq M, %rax sarq $4, %rax - jle .L25 + jle L(25) movaps -28 * SIZE(X), %xmm1 movaps -24 * SIZE(X), %xmm2 @@ -425,10 +425,10 @@ movaps -16 * SIZE(X), %xmm4 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): movaps -12 * SIZE(X), %xmm5 movaps -8 * SIZE(X), %xmm6 @@ -532,10 +532,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): movaps -12 * SIZE(X), %xmm5 movaps -8 * SIZE(X), %xmm6 @@ -618,9 +618,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $8, M - jle .L26 + jle L(26) movaps -28 * SIZE(X), %xmm1 movaps -24 * SIZE(X), %xmm2 @@ -668,9 +668,9 @@ addq $16 * SIZE, Y ALIGN_2 -.L26: +L(26): testq $4, M - jle .L27 + jle L(27) movaps -28 * SIZE(X), %xmm1 movaps -24 * SIZE(X), %xmm2 @@ -699,9 +699,9 @@ addq $8 * SIZE, Y ALIGN_2 -.L27: +L(27): testq $2, M - jle .L28 + jle L(28) movaps -28 * SIZE(X), %xmm1 @@ -721,9 +721,9 @@ addq $4 * SIZE, Y ALIGN_2 -.L28: +L(28): testq $1, M - jle .L999 + jle L(999) pshufd $0x06, %xmm0, %xmm8 pshufd $0x09, %xmm0, %xmm0 @@ -736,17 +736,17 @@ movlps %xmm0, -32 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L30: +L(30): testq $1 * SIZE, X - jne .L40 + jne L(40) #endif movq M, %rax sarq $4, %rax - jle .L35 + jle L(35) movsd -32 * SIZE(X), %xmm0 movhps -30 * SIZE(X), %xmm0 @@ -758,10 +758,10 @@ movhps -18 * SIZE(X), %xmm3 decq %rax - jle .L32 + jle L(32) ALIGN_3 -.L31: +L(31): movsd -16 * SIZE(X), %xmm4 movhps -14 * SIZE(X), %xmm4 @@ -861,10 +861,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L32: +L(32): movsd -16 * SIZE(X), %xmm4 movhps -14 * SIZE(X), %xmm4 @@ -937,9 +937,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L35: +L(35): testq $8, M - jle .L36 + jle L(36) movsd -32 * SIZE(X), %xmm0 movhps -30 * SIZE(X), %xmm0 @@ -983,9 +983,9 @@ addq $16 * SIZE, Y ALIGN_2 -.L36: +L(36): testq $4, M - jle .L37 + jle L(37) movsd -32 * SIZE(X), %xmm0 movhps -30 * SIZE(X), %xmm0 @@ -1010,9 +1010,9 @@ addq $8 * SIZE, Y ALIGN_2 -.L37: +L(37): testq $2, M - jle .L38 + jle L(38) movsd -32 * SIZE(X), %xmm0 movhps -30 * SIZE(X), %xmm0 @@ -1029,9 +1029,9 @@ addq $4 * SIZE, Y ALIGN_2 -.L38: +L(38): testq $1, M - jle .L999 + jle L(999) movsd -32 * SIZE(X), %xmm0 @@ -1044,19 +1044,19 @@ addps %xmm1, %xmm0 addps %xmm8, %xmm0 movlps %xmm0, -32 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 #ifdef ALIGNED_ACCESS -.L40: +L(40): subq $3 * SIZE, X movaps -32 * SIZE(X), %xmm0 movq M, %rax sarq $4, %rax - jle .L45 + jle L(45) movaps -28 * SIZE(X), %xmm1 movaps -24 * SIZE(X), %xmm2 @@ -1064,10 +1064,10 @@ movaps -16 * SIZE(X), %xmm4 decq %rax - jle .L42 + jle L(42) ALIGN_3 -.L41: +L(41): movaps -12 * SIZE(X), %xmm5 movaps -8 * SIZE(X), %xmm6 @@ -1171,10 +1171,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L42: +L(42): movaps -12 * SIZE(X), %xmm5 movaps -8 * SIZE(X), %xmm6 @@ -1257,9 +1257,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L45: +L(45): testq $8, M - jle .L46 + jle L(46) movaps -28 * SIZE(X), %xmm1 movaps -24 * SIZE(X), %xmm2 @@ -1307,9 +1307,9 @@ addq $16 * SIZE, Y ALIGN_2 -.L46: +L(46): testq $4, M - jle .L47 + jle L(47) movaps -28 * SIZE(X), %xmm1 movaps -24 * SIZE(X), %xmm2 @@ -1338,9 +1338,9 @@ addq $8 * SIZE, Y ALIGN_2 -.L47: +L(47): testq $2, M - jle .L48 + jle L(48) movaps -28 * SIZE(X), %xmm1 @@ -1360,9 +1360,9 @@ addq $4 * SIZE, Y ALIGN_2 -.L48: +L(48): testq $1, M - jle .L999 + jle L(999) movaps -28 * SIZE(X), %xmm1 movsd -32 * SIZE(Y), %xmm2 @@ -1377,21 +1377,21 @@ addps %xmm2, %xmm0 movlps %xmm0, -32 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 #endif -.L50: +L(50): xorps %xmm0, %xmm0 subq $1 * SIZE, Y testq $3 * SIZE, X - jne .L60 + jne L(60) movq M, %rax sarq $4, %rax - jle .L55 + jle L(55) movaps -32 * SIZE(X), %xmm1 movaps -28 * SIZE(X), %xmm2 @@ -1399,10 +1399,10 @@ movaps -20 * SIZE(X), %xmm4 decq %rax - jle .L52 + jle L(52) ALIGN_3 -.L51: +L(51): movaps -16 * SIZE(X), %xmm5 movaps -12 * SIZE(X), %xmm6 @@ -1506,10 +1506,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L51 + jg L(51) ALIGN_3 -.L52: +L(52): movaps -16 * SIZE(X), %xmm5 movaps -12 * SIZE(X), %xmm6 @@ -1592,9 +1592,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L55: +L(55): testq $8, M - jle .L56 + jle L(56) movaps -32 * SIZE(X), %xmm1 movaps -28 * SIZE(X), %xmm2 @@ -1642,9 +1642,9 @@ addq $16 * SIZE, Y ALIGN_2 -.L56: +L(56): testq $4, M - jle .L57 + jle L(57) movaps -32 * SIZE(X), %xmm1 movaps -28 * SIZE(X), %xmm2 @@ -1673,9 +1673,9 @@ addq $8 * SIZE, Y ALIGN_2 -.L57: +L(57): testq $2, M - jle .L58 + jle L(58) movaps -32 * SIZE(X), %xmm1 @@ -1694,9 +1694,9 @@ addq $4 * SIZE, Y ALIGN_2 -.L58: +L(58): testq $1, M - jle .L59 + jle L(59) #ifdef movsd xorps %xmm1, %xmm1 @@ -1712,22 +1712,22 @@ addps -32 * SIZE(Y), %xmm0 movaps %xmm0, -32 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L59: +L(59): shufps $0x93, %xmm0, %xmm0 addss -32 * SIZE(Y), %xmm0 movss %xmm0, -32 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L60: +L(60): #ifdef ALIGNED_ACCESS testq $2 * SIZE, X - jne .L70 + jne L(70) subq $1 * SIZE, X @@ -1735,16 +1735,16 @@ movq M, %rax sarq $4, %rax - jle .L65 + jle L(65) movaps -28 * SIZE(X), %xmm2 movaps -24 * SIZE(X), %xmm3 decq %rax - jle .L62 + jle L(62) ALIGN_3 -.L61: +L(61): movaps -20 * SIZE(X), %xmm4 movaps -16 * SIZE(X), %xmm5 @@ -1865,10 +1865,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L61 + jg L(61) ALIGN_3 -.L62: +L(62): movaps -20 * SIZE(X), %xmm4 movaps -16 * SIZE(X), %xmm5 @@ -1970,9 +1970,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L65: +L(65): testq $8, M - jle .L66 + jle L(66) movaps -28 * SIZE(X), %xmm2 movaps -24 * SIZE(X), %xmm3 @@ -2028,9 +2028,9 @@ addq $16 * SIZE, Y ALIGN_2 -.L66: +L(66): testq $4, M - jle .L67 + jle L(67) movaps -28 * SIZE(X), %xmm2 movaps -24 * SIZE(X), %xmm3 @@ -2064,9 +2064,9 @@ addq $8 * SIZE, Y ALIGN_2 -.L67: +L(67): testq $2, M - jle .L68 + jle L(68) movaps -28 * SIZE(X), %xmm2 @@ -2088,9 +2088,9 @@ addq $4 * SIZE, Y ALIGN_2 -.L68: +L(68): testq $1, M - jle .L69 + jle L(69) movaps -28 * SIZE(X), %xmm2 @@ -2107,24 +2107,24 @@ movlps %xmm0, -32 * SIZE(Y) movhlps %xmm0, %xmm0 movss %xmm0, -30 * SIZE(Y) - jmp .L999 + jmp L(999) -.L69: +L(69): shufps $0x93, %xmm0, %xmm0 addss -32 * SIZE(Y), %xmm0 movss %xmm0, -32 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L70: +L(70): testq $1 * SIZE, X - jne .L80 + jne L(80) #endif movq M, %rax sarq $4, %rax - jle .L75 + jle L(75) movsd -32 * SIZE(X), %xmm1 movhps -30 * SIZE(X), %xmm1 @@ -2136,10 +2136,10 @@ movhps -18 * SIZE(X), %xmm4 decq %rax - jle .L72 + jle L(72) ALIGN_3 -.L71: +L(71): movsd -16 * SIZE(X), %xmm5 movhps -14 * SIZE(X), %xmm5 movsd -12 * SIZE(X), %xmm6 @@ -2251,10 +2251,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L71 + jg L(71) ALIGN_3 -.L72: +L(72): movsd -16 * SIZE(X), %xmm5 movhps -14 * SIZE(X), %xmm5 movsd -12 * SIZE(X), %xmm6 @@ -2341,9 +2341,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L75: +L(75): testq $8, M - jle .L76 + jle L(76) movsd -32 * SIZE(X), %xmm1 movhps -30 * SIZE(X), %xmm1 @@ -2395,9 +2395,9 @@ addq $16 * SIZE, Y ALIGN_2 -.L76: +L(76): testq $4, M - jle .L77 + jle L(77) movsd -32 * SIZE(X), %xmm1 movhps -30 * SIZE(X), %xmm1 @@ -2429,9 +2429,9 @@ addq $8 * SIZE, Y ALIGN_2 -.L77: +L(77): testq $2, M - jle .L78 + jle L(78) movsd -32 * SIZE(X), %xmm1 movhps -30 * SIZE(X), %xmm1 @@ -2451,9 +2451,9 @@ addq $4 * SIZE, Y ALIGN_2 -.L78: +L(78): testq $1, M - jle .L79 + jle L(79) #ifdef movsd xorps %xmm1, %xmm1 @@ -2469,36 +2469,36 @@ addps -32 * SIZE(Y), %xmm0 movaps %xmm0, -32 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L79: +L(79): shufps $0x93, %xmm0, %xmm0 addss -32 * SIZE(Y), %xmm0 movss %xmm0, -32 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 #ifdef ALIGNED_ACCESS -.L80: +L(80): subq $3 * SIZE, X movaps -32 * SIZE(X), %xmm1 movq M, %rax sarq $4, %rax - jle .L85 + jle L(85) movaps -28 * SIZE(X), %xmm2 movaps -24 * SIZE(X), %xmm3 decq %rax - jle .L82 + jle L(82) ALIGN_3 -.L81: +L(81): movaps -20 * SIZE(X), %xmm4 movaps -16 * SIZE(X), %xmm5 @@ -2618,10 +2618,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L81 + jg L(81) ALIGN_3 -.L82: +L(82): movaps -20 * SIZE(X), %xmm4 movaps -16 * SIZE(X), %xmm5 @@ -2723,9 +2723,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L85: +L(85): testq $8, M - jle .L86 + jle L(86) movaps -28 * SIZE(X), %xmm2 movaps -24 * SIZE(X), %xmm3 @@ -2781,9 +2781,9 @@ addq $16 * SIZE, Y ALIGN_2 -.L86: +L(86): testq $4, M - jle .L87 + jle L(87) movaps -28 * SIZE(X), %xmm2 movaps -24 * SIZE(X), %xmm3 @@ -2817,9 +2817,9 @@ addq $8 * SIZE, Y ALIGN_2 -.L87: +L(87): testq $2, M - jle .L88 + jle L(88) movaps -28 * SIZE(X), %xmm2 @@ -2841,9 +2841,9 @@ addq $4 * SIZE, Y ALIGN_2 -.L88: +L(88): testq $1, M - jle .L89 + jle L(89) movaps -28 * SIZE(X), %xmm2 @@ -2860,18 +2860,18 @@ movlps %xmm0, -32 * SIZE(Y) movhlps %xmm0, %xmm0 movss %xmm0, -30 * SIZE(Y) - jmp .L999 + jmp L(999) -.L89: +L(89): shufps $0x93, %xmm0, %xmm0 addss -32 * SIZE(Y), %xmm0 movss %xmm0, -32 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 #endif -.L100: +L(100): #ifndef CONJ pshufd $0, %xmm0, %xmm14 pshufd $0, %xmm1, %xmm15 @@ -2895,18 +2895,18 @@ //If incx==0 || incy==0, avoid unloop and jump to end. cmpq $0, INCX - je .L200 + je L(200) cmpq $0, INCY - je .L200 + je L(200) movq Y, YY movq M, %rax sarq $3, %rax - jle .L105 + jle L(105) ALIGN_3 -.L102: +L(102): movsd (X), %xmm0 addq INCX, X movhps (X), %xmm0 @@ -2997,12 +2997,12 @@ addq INCY, YY decq %rax - jg .L102 + jg L(102) ALIGN_3 -.L105: +L(105): testq $4, M - jle .L106 + jle L(106) movsd (X), %xmm0 addq INCX, X @@ -3054,9 +3054,9 @@ addq INCY, YY ALIGN_3 -.L106: +L(106): testq $2, M - jle .L107 + jle L(107) movsd (X), %xmm0 addq INCX, X @@ -3088,9 +3088,9 @@ addq INCY, YY ALIGN_3 -.L107: +L(107): testq $1, M - jle .L999 + jle L(999) movsd (X), %xmm0 @@ -3111,16 +3111,16 @@ addps %xmm1, %xmm8 movsd %xmm8, (Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L200: +L(200): movq M, %rax cmpq $0, %rax - jle .L999 + jle L(999) ALIGN_3 -.L201: +L(201): movsd (X), %xmm0 addq INCX, X @@ -3144,10 +3144,10 @@ addq INCY, Y decq %rax - jg .L201 + jg L(201) ALIGN_3 -.L999: +L(999): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/zaxpy_sse2.S b/kernel/x86_64/zaxpy_sse2.S index 3776c8910d..2d185a65b5 100644 --- a/kernel/x86_64/zaxpy_sse2.S +++ b/kernel/x86_64/zaxpy_sse2.S @@ -90,12 +90,12 @@ salq $ZBASE_SHIFT, INCY testq M, M - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX - jne .L50 + jne L(50) cmpq $2 * SIZE, INCY - jne .L50 + jne L(50) subq $-16 * SIZE, X subq $-16 * SIZE, Y @@ -120,14 +120,14 @@ #endif testq $SIZE, Y - jne .L30 + jne L(30) testq $SIZE, X - jne .L20 + jne L(20) movq M, %rax sarq $3, %rax - jle .L15 + jle L(15) movaps -16 * SIZE(X), %xmm0 movaps -14 * SIZE(X), %xmm1 @@ -135,10 +135,10 @@ movaps -10 * SIZE(X), %xmm3 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): movaps -8 * SIZE(X), %xmm4 movaps -6 * SIZE(X), %xmm5 @@ -266,10 +266,10 @@ subq $-16 * SIZE, X subq $-16 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movaps -8 * SIZE(X), %xmm4 movaps -6 * SIZE(X), %xmm5 @@ -336,10 +336,10 @@ subq $-16 * SIZE, Y ALIGN_3 -.L15: +L(15): movq M, %rax andq $4, %rax - jle .L16 + jle L(16) movaps -16 * SIZE(X), %xmm0 movaps -14 * SIZE(X), %xmm1 @@ -378,10 +378,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L16: +L(16): movq M, %rax andq $2, %rax - jle .L17 + jle L(17) movaps -16 * SIZE(X), %xmm0 movaps -14 * SIZE(X), %xmm1 @@ -404,10 +404,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L17: +L(17): movq M, %rax andq $1, %rax - jle .L999 + jle L(999) movaps -16 * SIZE(X), %xmm0 @@ -417,13 +417,13 @@ addpd -16 * SIZE(Y), %xmm0 addpd %xmm8, %xmm0 movaps %xmm0, -16 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L20: +L(20): movq M, %rax sarq $3, %rax - jle .L25 + jle L(25) movsd -16 * SIZE(X), %xmm0 movhps -15 * SIZE(X), %xmm0 @@ -435,10 +435,10 @@ movhps -9 * SIZE(X), %xmm3 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): movsd -8 * SIZE(X), %xmm4 movhps -7 * SIZE(X), %xmm4 @@ -538,10 +538,10 @@ subq $-16 * SIZE, X subq $-16 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): movsd -8 * SIZE(X), %xmm4 movhps -7 * SIZE(X), %xmm4 @@ -614,10 +614,10 @@ subq $-16 * SIZE, Y ALIGN_3 -.L25: +L(25): movq M, %rax andq $4, %rax - jle .L26 + jle L(26) movsd -16 * SIZE(X), %xmm0 movhps -15 * SIZE(X), %xmm0 @@ -661,10 +661,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L26: +L(26): movq M, %rax andq $2, %rax - jle .L27 + jle L(27) movsd -16 * SIZE(X), %xmm0 movhps -15 * SIZE(X), %xmm0 @@ -690,10 +690,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L27: +L(27): movq M, %rax andq $1, %rax - jle .L999 + jle L(999) movsd -16 * SIZE(X), %xmm0 movhps -15 * SIZE(X), %xmm0 @@ -704,12 +704,12 @@ addpd -16 * SIZE(Y), %xmm0 addpd %xmm8, %xmm0 movaps %xmm0, -16 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L30: +L(30): testq $SIZE, X - jne .L40 + jne L(40) movaps -16 * SIZE(X), %xmm1 @@ -730,21 +730,21 @@ addq $2 * SIZE, X addq $1 * SIZE, Y decq M - jle .L39 + jle L(39) movq M, %rax sarq $3, %rax - jle .L35 + jle L(35) movaps -16 * SIZE(X), %xmm1 movaps -14 * SIZE(X), %xmm2 movaps -12 * SIZE(X), %xmm3 decq %rax - jle .L32 + jle L(32) ALIGN_3 -.L31: +L(31): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -844,10 +844,10 @@ subq $-16 * SIZE, X subq $-16 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L32: +L(32): pshufd $0x4e, %xmm1, %xmm8 mulpd ALPHA_R, %xmm1 mulpd ALPHA_I, %xmm8 @@ -929,10 +929,10 @@ subq $-16 * SIZE, Y ALIGN_3 -.L35: +L(35): movq M, %rax andq $4, %rax - jle .L36 + jle L(36) movaps -16 * SIZE(X), %xmm1 movaps -14 * SIZE(X), %xmm2 @@ -980,10 +980,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L36: +L(36): movq M, %rax andq $2, %rax - jle .L37 + jle L(37) movaps -16 * SIZE(X), %xmm1 movaps -14 * SIZE(X), %xmm2 @@ -1011,10 +1011,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L37: +L(37): movq M, %rax andq $1, %rax - jle .L39 + jle L(39) movaps -16 * SIZE(X), %xmm1 @@ -1032,15 +1032,15 @@ addq $2 * SIZE, Y ALIGN_3 -.L39: +L(39): SHUFPD_1 %xmm0, %xmm0 addsd -16 * SIZE(Y), %xmm0 movlps %xmm0, -16 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L40: +L(40): movsd -16 * SIZE(X), %xmm1 movhps -15 * SIZE(X), %xmm1 @@ -1061,11 +1061,11 @@ addq $2 * SIZE, X addq $1 * SIZE, Y decq M - jle .L49 + jle L(49) movq M, %rax sarq $3, %rax - jle .L45 + jle L(45) movsd -16 * SIZE(X), %xmm1 movhps -15 * SIZE(X), %xmm1 @@ -1075,10 +1075,10 @@ movhps -11 * SIZE(X), %xmm3 decq %rax - jle .L42 + jle L(42) ALIGN_3 -.L41: +L(41): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -1186,10 +1186,10 @@ subq $-16 * SIZE, X subq $-16 * SIZE, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L42: +L(42): pshufd $0x4e, %xmm1, %xmm8 mulpd ALPHA_R, %xmm1 mulpd ALPHA_I, %xmm8 @@ -1276,10 +1276,10 @@ subq $-16 * SIZE, Y ALIGN_3 -.L45: +L(45): movq M, %rax andq $4, %rax - jle .L46 + jle L(46) movsd -16 * SIZE(X), %xmm1 movhps -15 * SIZE(X), %xmm1 @@ -1331,10 +1331,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L46: +L(46): movq M, %rax andq $2, %rax - jle .L47 + jle L(47) movsd -16 * SIZE(X), %xmm1 movhps -15 * SIZE(X), %xmm1 @@ -1364,10 +1364,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L47: +L(47): movq M, %rax andq $1, %rax - jle .L49 + jle L(49) movsd -16 * SIZE(X), %xmm1 movhps -15 * SIZE(X), %xmm1 @@ -1386,15 +1386,15 @@ addq $2 * SIZE, Y ALIGN_3 -.L49: +L(49): SHUFPD_1 %xmm0, %xmm0 addsd -16 * SIZE(Y), %xmm0 movlps %xmm0, -16 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L50: +L(50): #ifndef CONJ movaps %xmm0, %xmm14 # a 0 @@ -1418,12 +1418,12 @@ movq M, %rax //If incx==0 || incy==0, avoid unloop and jump to end. cmpq $0, INCX - jne .L59 + jne L(59) cmpq $0, INCY - je .L58 -.L59: + je L(58) +L(59): sarq $3, %rax - jle .L55 + jle L(55) MOVDDUP( 0 * SIZE, X, %xmm0) MOVDDUP( 1 * SIZE, X, %xmm1) @@ -1457,10 +1457,10 @@ mulpd %xmm14, %xmm6 decq %rax - jle .L52 + jle L(52) ALIGN_3 -.L51: +L(51): addpd %xmm0, %xmm8 mulpd %xmm15, %xmm1 addpd %xmm2, %xmm9 @@ -1578,10 +1578,10 @@ addq INCY, Y decq %rax - jg .L51 + jg L(51) ALIGN_3 -.L52: +L(52): addpd %xmm0, %xmm8 mulpd %xmm15, %xmm1 addpd %xmm2, %xmm9 @@ -1668,10 +1668,10 @@ addq INCY, YY ALIGN_3 -.L55: +L(55): movq M, %rax andq $4, %rax - jle .L56 + jle L(56) MOVDDUP( 0 * SIZE, X, %xmm0) MOVDDUP( 1 * SIZE, X, %xmm1) @@ -1733,10 +1733,10 @@ addq INCY, YY ALIGN_3 -.L56: +L(56): movq M, %rax andq $2, %rax - jle .L57 + jle L(57) MOVDDUP( 0 * SIZE, X, %xmm0) MOVDDUP( 1 * SIZE, X, %xmm1) @@ -1770,12 +1770,12 @@ addq INCY, YY ALIGN_3 -.L57: +L(57): movq M, %rax andq $1, %rax - jle .L999 + jle L(999) -.L58: +L(58): MOVDDUP( 0 * SIZE, X, %xmm0) MOVDDUP( 1 * SIZE, X, %xmm1) @@ -1790,10 +1790,10 @@ movhpd %xmm8, 1 * SIZE(YY) decq %rax - jg .L58 + jg L(58) ALIGN_3 -.L999: +L(999): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/zcopy.S b/kernel/x86_64/zcopy.S index 3cc4e185b4..35b9f2221a 100644 --- a/kernel/x86_64/zcopy.S +++ b/kernel/x86_64/zcopy.S @@ -66,19 +66,19 @@ salq $ZBASE_SHIFT, INCY testq N, N # if m == 0 goto End - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX # if incx != 1 - jne .L100 + jne L(100) cmpq $2 * SIZE, INCY # if incy != 1 - jne .L100 + jne L(100) movq N, %rax # i = m sarq $2, %rax - jle .L20 + jle L(20) ALIGN_2 -.L11: +L(11): #ifdef XDOUBLE #ifdef PREFETCH @@ -201,16 +201,16 @@ addq $8 * SIZE, X addq $8 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_2 -.L20: +L(20): movq N, %rax # i = m andq $3, %rax - jle .L99 + jle L(99) ALIGN_2 -.L21: +L(21): #ifdef XDOUBLE movq 0(X), %mm0 movq %mm0, 0(Y) @@ -233,21 +233,21 @@ addq $2 * SIZE, X addq $2 * SIZE, Y decq %rax - jg .L21 + jg L(21) -.L99: +L(99): xorq %rax,%rax EMMS ret ALIGN_3 -.L100: +L(100): movq N, %rax sarq $2, %rax - jle .L120 + jle L(120) ALIGN_2 -.L111: +L(111): #ifdef XDOUBLE movq 0(X), %mm0 movq %mm0, 0(Y) @@ -343,15 +343,15 @@ #endif decq %rax - jg .L111 + jg L(111) -.L120: +L(120): movq N, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_2 -.L121: +L(121): #ifdef XDOUBLE movq 0(X), %mm0 movq %mm0, 0(Y) @@ -378,9 +378,9 @@ #endif decq %rax - jg .L121 + jg L(121) -.L999: +L(999): xorq %rax,%rax EMMS ret diff --git a/kernel/x86_64/zcopy_sse.S b/kernel/x86_64/zcopy_sse.S index 018a56ff23..a36ce0339a 100644 --- a/kernel/x86_64/zcopy_sse.S +++ b/kernel/x86_64/zcopy_sse.S @@ -70,19 +70,19 @@ salq $ZBASE_SHIFT, INCY cmpq $2 * SIZE, INCX - jne .L100 + jne L(100) cmpq $2 * SIZE, INCY - jne .L100 + jne L(100) cmpq $3, M - jle .L106 + jle L(106) subq $-32 * SIZE, X subq $-32 * SIZE, Y addq M, M testq $SIZE, Y - je .L05 + je L(05) movss -32 * SIZE(X), %xmm0 movss %xmm0, -32 * SIZE(Y) @@ -91,25 +91,25 @@ decq M ALIGN_4 -.L05: +L(05): testq $2 * SIZE, Y - je .L10 + je L(10) movsd -32 * SIZE(X), %xmm0 movlps %xmm0, -32 * SIZE(Y) addq $2 * SIZE, X addq $2 * SIZE, Y subq $2, M - jle .L19 + jle L(19) ALIGN_4 -.L10: +L(10): testq $3 * SIZE, X - jne .L20 + jne L(20) movq M, %rax sarq $5, %rax - jle .L13 + jle L(13) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -121,10 +121,10 @@ movaps -4 * SIZE(X), %xmm7 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -164,10 +164,10 @@ subq $-32 * SIZE, Y subq $-32 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movaps %xmm0, -32 * SIZE(Y) movaps %xmm1, -28 * SIZE(Y) movaps %xmm2, -24 * SIZE(Y) @@ -181,9 +181,9 @@ subq $-32 * SIZE, X ALIGN_3 -.L13: +L(13): testq $16, M - jle .L14 + jle L(14) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -199,9 +199,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $8, M - jle .L15 + jle L(15) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -213,9 +213,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $4, M - jle .L16 + jle L(16) movaps -32 * SIZE(X), %xmm0 movaps %xmm0, -32 * SIZE(Y) @@ -224,9 +224,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $2, M - jle .L17 + jle L(17) movsd -32 * SIZE(X), %xmm0 movlps %xmm0, -32 * SIZE(Y) @@ -235,15 +235,15 @@ addq $2 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $1, M - jle .L19 + jle L(19) movss -32 * SIZE(X), %xmm0 movss %xmm0, -32 * SIZE(Y) ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -252,15 +252,15 @@ ALIGN_3 -.L20: +L(20): testq $SIZE, X - jne .L30 + jne L(30) movhps -32 * SIZE(X), %xmm0 movq M, %rax sarq $5, %rax - jle .L23 + jle L(23) movaps -30 * SIZE(X), %xmm1 movaps -26 * SIZE(X), %xmm2 @@ -271,10 +271,10 @@ movaps -6 * SIZE(X), %xmm7 decq %rax - jle .L22 + jle L(22) ALIGN_4 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) @@ -327,10 +327,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): shufps $0x4e, %xmm1, %xmm0 movaps %xmm0, -32 * SIZE(Y) movaps -2 * SIZE(X), %xmm0 @@ -360,9 +360,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L23: +L(23): testq $16, M - jle .L24 + jle L(24) ALIGN_3 movaps -30 * SIZE(X), %xmm1 @@ -385,9 +385,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $8, M - jle .L25 + jle L(25) ALIGN_3 movaps -30 * SIZE(X), %xmm1 @@ -404,9 +404,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $4, M - jle .L26 + jle L(26) ALIGN_3 movaps -30 * SIZE(X), %xmm1 @@ -417,9 +417,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $2, M - jle .L27 + jle L(27) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -430,9 +430,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $1, M - jle .L29 + jle L(29) ALIGN_3 movss -32 * SIZE(X), %xmm0 @@ -440,7 +440,7 @@ addq $SIZE, Y ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -448,15 +448,15 @@ ret ALIGN_3 -.L30: +L(30): testq $2 * SIZE, X - jne .L40 + jne L(40) movaps -33 * SIZE(X), %xmm0 movq M, %rax sarq $5, %rax - jle .L33 + jle L(33) movaps -29 * SIZE(X), %xmm1 movaps -25 * SIZE(X), %xmm2 @@ -467,10 +467,10 @@ movaps -5 * SIZE(X), %xmm7 decq %rax - jle .L32 + jle L(32) ALIGN_4 -.L31: +L(31): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -530,10 +530,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L32: +L(32): movss %xmm1, %xmm0 shufps $0x39, %xmm0, %xmm0 movaps %xmm0, -32 * SIZE(Y) @@ -571,9 +571,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L33: +L(33): testq $16, M - jle .L34 + jle L(34) ALIGN_3 movaps -29 * SIZE(X), %xmm1 @@ -603,9 +603,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L34: +L(34): testq $8, M - jle .L35 + jle L(35) ALIGN_3 movaps -29 * SIZE(X), %xmm1 @@ -624,9 +624,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L35: +L(35): testq $4, M - jle .L36 + jle L(36) ALIGN_3 movaps -29 * SIZE(X), %xmm1 @@ -640,9 +640,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L36: +L(36): testq $2, M - jle .L37 + jle L(37) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -652,9 +652,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L37: +L(37): testq $1, M - jle .L39 + jle L(39) ALIGN_3 movss -32 * SIZE(X), %xmm0 @@ -662,7 +662,7 @@ addq $SIZE, Y ALIGN_3 -.L39: +L(39): xorq %rax,%rax RESTOREREGISTERS @@ -670,12 +670,12 @@ ret ALIGN_3 -.L40: +L(40): movaps -35 * SIZE(X), %xmm0 movq M, %rax sarq $5, %rax - jle .L43 + jle L(43) movaps -31 * SIZE(X), %xmm1 movaps -27 * SIZE(X), %xmm2 @@ -686,10 +686,10 @@ movaps -7 * SIZE(X), %xmm7 decq %rax - jle .L42 + jle L(42) ALIGN_4 -.L41: +L(41): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -749,10 +749,10 @@ subq $-32 * SIZE, X subq $-32 * SIZE, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L42: +L(42): movss %xmm1, %xmm0 shufps $0x93, %xmm1, %xmm0 movaps %xmm0, -32 * SIZE(Y) @@ -790,9 +790,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L43: +L(43): testq $16, M - jle .L44 + jle L(44) ALIGN_3 movaps -31 * SIZE(X), %xmm1 @@ -822,9 +822,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L44: +L(44): testq $8, M - jle .L45 + jle L(45) ALIGN_3 movaps -31 * SIZE(X), %xmm1 @@ -844,9 +844,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L45: +L(45): testq $4, M - jle .L46 + jle L(46) ALIGN_3 movaps -31 * SIZE(X), %xmm1 @@ -860,9 +860,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L46: +L(46): testq $2, M - jle .L47 + jle L(47) ALIGN_3 movsd -32 * SIZE(X), %xmm0 @@ -872,9 +872,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L47: +L(47): testq $1, M - jle .L49 + jle L(49) ALIGN_3 movss -32 * SIZE(X), %xmm0 @@ -882,7 +882,7 @@ addq $SIZE, Y ALIGN_3 -.L49: +L(49): xorq %rax,%rax RESTOREREGISTERS @@ -890,13 +890,13 @@ ret ALIGN_4 -.L100: +L(100): movq M, %rax sarq $3, %rax - jle .L105 + jle L(105) ALIGN_3 -.L102: +L(102): movsd (X), %xmm0 addq INCX, X movhps (X), %xmm0 @@ -932,12 +932,12 @@ addq INCY, Y decq %rax - jg .L102 + jg L(102) ALIGN_3 -.L105: +L(105): testq $4, M - jle .L106 + jle L(106) movsd (X), %xmm0 addq INCX, X @@ -958,9 +958,9 @@ addq INCY, Y ALIGN_3 -.L106: +L(106): testq $2, M - jle .L107 + jle L(107) movsd (X), %xmm0 addq INCX, X @@ -973,15 +973,15 @@ addq INCY, Y ALIGN_3 -.L107: +L(107): testq $1, M - jle .L999 + jle L(999) movsd (X), %xmm0 movsd %xmm0, (Y) ALIGN_3 -.L999: +L(999): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/zcopy_sse2.S b/kernel/x86_64/zcopy_sse2.S index c3a99a57b3..c06e65e1d4 100644 --- a/kernel/x86_64/zcopy_sse2.S +++ b/kernel/x86_64/zcopy_sse2.S @@ -70,9 +70,9 @@ salq $ZBASE_SHIFT, INCY cmpq $2 * SIZE, INCX - jne .L50 + jne L(50) cmpq $2 * SIZE, INCY - jne .L50 + jne L(50) addq M, M @@ -81,17 +81,17 @@ #else testq $SIZE, X #endif - je .L10 + je L(10) movsd (X), %xmm0 movsd %xmm0, (Y) addq $1 * SIZE, X addq $1 * SIZE, Y decq M - jle .L19 + jle L(19) ALIGN_4 -.L10: +L(10): subq $-16 * SIZE, X subq $-16 * SIZE, Y @@ -100,11 +100,11 @@ #else testq $SIZE, Y #endif - jne .L20 + jne L(20) movq M, %rax sarq $4, %rax - jle .L13 + jle L(13) movaps -16 * SIZE(X), %xmm0 movaps -14 * SIZE(X), %xmm1 @@ -116,10 +116,10 @@ movaps -2 * SIZE(X), %xmm7 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -159,10 +159,10 @@ subq $-16 * SIZE, Y subq $-16 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movaps %xmm0, -16 * SIZE(Y) movaps %xmm1, -14 * SIZE(Y) movaps %xmm2, -12 * SIZE(Y) @@ -176,9 +176,9 @@ subq $-16 * SIZE, X ALIGN_3 -.L13: +L(13): testq $8, M - jle .L14 + jle L(14) ALIGN_3 movaps -16 * SIZE(X), %xmm0 @@ -195,9 +195,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $4, M - jle .L15 + jle L(15) ALIGN_3 movaps -16 * SIZE(X), %xmm0 @@ -210,9 +210,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $2, M - jle .L16 + jle L(16) ALIGN_3 movaps -16 * SIZE(X), %xmm0 @@ -222,16 +222,16 @@ addq $2 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $1, M - jle .L19 + jle L(19) ALIGN_3 movsd -16 * SIZE(X), %xmm0 movsd %xmm0, -16 * SIZE(Y) ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -239,14 +239,14 @@ ret ALIGN_3 -.L20: +L(20): #ifdef ALIGNED_ACCESS movhps -16 * SIZE(X), %xmm0 movq M, %rax sarq $4, %rax - jle .L23 + jle L(23) movaps -15 * SIZE(X), %xmm1 movaps -13 * SIZE(X), %xmm2 @@ -257,10 +257,10 @@ movaps -3 * SIZE(X), %xmm7 decq %rax - jle .L22 + jle L(22) ALIGN_4 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -312,10 +312,10 @@ subq $-16 * SIZE, X subq $-16 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): SHUFPD_1 %xmm1, %xmm0 movaps %xmm0, -16 * SIZE(Y) LOAD(-1 * SIZE, X, %xmm0) @@ -342,9 +342,9 @@ subq $-16 * SIZE, Y ALIGN_3 -.L23: +L(23): testq $8, M - jle .L24 + jle L(24) ALIGN_3 movaps -15 * SIZE(X), %xmm1 @@ -370,9 +370,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $4, M - jle .L25 + jle L(25) ALIGN_3 movaps -15 * SIZE(X), %xmm1 @@ -389,9 +389,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $2, M - jle .L26 + jle L(26) ALIGN_3 movaps -15 * SIZE(X), %xmm1 @@ -403,16 +403,16 @@ addq $2 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $1, M - jle .L29 + jle L(29) ALIGN_3 movsd -16 * SIZE(X), %xmm0 movsd %xmm0, -16 * SIZE(Y) ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -424,7 +424,7 @@ movq M, %rax sarq $4, %rax - jle .L23 + jle L(23) movaps -16 * SIZE(X), %xmm0 movaps -14 * SIZE(X), %xmm1 @@ -436,10 +436,10 @@ movaps -2 * SIZE(X), %xmm7 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -487,10 +487,10 @@ subq $-16 * SIZE, Y subq $-16 * SIZE, X decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): movlps %xmm0, -16 * SIZE(Y) movhps %xmm0, -15 * SIZE(Y) movlps %xmm1, -14 * SIZE(Y) @@ -512,9 +512,9 @@ subq $-16 * SIZE, X ALIGN_3 -.L23: +L(23): testq $8, M - jle .L24 + jle L(24) ALIGN_3 movaps -16 * SIZE(X), %xmm0 @@ -534,9 +534,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $4, M - jle .L25 + jle L(25) ALIGN_3 movaps -16 * SIZE(X), %xmm0 @@ -550,9 +550,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $2, M - jle .L26 + jle L(26) ALIGN_3 movaps -16 * SIZE(X), %xmm0 @@ -563,16 +563,16 @@ addq $2 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $1, M - jle .L29 + jle L(29) ALIGN_3 movsd -16 * SIZE(X), %xmm0 movsd %xmm0, -16 * SIZE(Y) ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -582,13 +582,13 @@ #endif -.L50: +L(50): movq M, %rax sarq $2, %rax - jle .L55 + jle L(55) ALIGN_3 -.L51: +L(51): movsd 0 * SIZE(X), %xmm0 movhps 1 * SIZE(X), %xmm0 addq INCX, X @@ -623,16 +623,16 @@ addq INCY, Y decq %rax - jg .L51 + jg L(51) ALIGN_3 -.L55: +L(55): movq M, %rax andq $3, %rax - jle .L57 + jle L(57) ALIGN_3 -.L56: +L(56): movsd 0 * SIZE(X), %xmm0 movhps 1 * SIZE(X), %xmm0 addq INCX, X @@ -642,10 +642,10 @@ addq INCY, Y decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L57: +L(57): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/zdot.S b/kernel/x86_64/zdot.S index 87c08d7c80..fcae4bf1d7 100644 --- a/kernel/x86_64/zdot.S +++ b/kernel/x86_64/zdot.S @@ -61,7 +61,7 @@ #endif testq N, N - jle .L88 + jle L(88) salq $ZBASE_SHIFT, INCX salq $ZBASE_SHIFT, INCY @@ -72,16 +72,16 @@ fldz cmpq $2 * SIZE, INCX - jne .L14 + jne L(14) cmpq $2 * SIZE, INCY - jne .L14 + jne L(14) movq N, %rax sarq $1, %rax - jle .L15 + jle L(15) ALIGN_3 -.L16: +L(16): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -130,16 +130,16 @@ addq $4 * SIZE, X addq $4 * SIZE, Y decq %rax - jg .L16 + jg L(16) ALIGN_3 -.L15: +L(15): movq N, %rax andq $1, %rax - jle .L27 + jle L(27) ALIGN_3 -.L22: +L(22): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -158,17 +158,17 @@ FLD 1 * SIZE(Y) fmulp %st, %st(1) faddp %st, %st(4) - jmp .L27 + jmp L(27) ALIGN_3 -.L14: +L(14): movq N, %rax sarq $1, %rax - jle .L30 + jle L(30) ALIGN_3 -.L31: +L(31): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -212,16 +212,16 @@ addq INCY, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L30: +L(30): movq N, %rax andq $1, %rax - jle .L27 + jle L(27) ALIGN_3 -.L37: +L(37): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -242,7 +242,7 @@ faddp %st, %st(4) ALIGN_3 -.L27: +L(27): #ifndef CONJ fsubp %st, %st(3) faddp %st, %st(1) @@ -255,7 +255,7 @@ ret ALIGN_3 -.L88: +L(88): fldz fldz diff --git a/kernel/x86_64/zdot_atom.S b/kernel/x86_64/zdot_atom.S index 9a8239c8d4..4884903e21 100644 --- a/kernel/x86_64/zdot_atom.S +++ b/kernel/x86_64/zdot_atom.S @@ -68,16 +68,16 @@ pxor %xmm2, %xmm2 cmpq $0, N pxor %xmm3, %xmm3 - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX - jne .L20 + jne L(20) cmpq $2 * SIZE, INCY - jne .L20 + jne L(20) movq N, %rax sarq $2, %rax - jle .L15 + jle L(15) movsd 0 * SIZE(X), %xmm4 movsd 0 * SIZE(Y), %xmm6 @@ -96,10 +96,10 @@ movsd 3 * SIZE(Y), %xmm13 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -172,10 +172,10 @@ addq $8 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): addsd %xmm4, %xmm0 movaps %xmm10, %xmm14 mulsd %xmm11, %xmm10 @@ -226,10 +226,10 @@ addq $8 * SIZE, Y ALIGN_3 -.L15: +L(15): movq N, %rax andq $2, %rax - jle .L17 + jle L(17) movsd 0 * SIZE(X), %xmm4 movsd 0 * SIZE(Y), %xmm6 @@ -267,10 +267,10 @@ addq $4 * SIZE, Y ALIGN_3 -.L17: +L(17): movq N, %rax andq $1, %rax - jle .L999 + jle L(999) movsd 0 * SIZE(X), %xmm4 movsd 0 * SIZE(Y), %xmm6 @@ -288,16 +288,16 @@ addsd %xmm8, %xmm1 addsd %xmm5, %xmm2 addsd %xmm9, %xmm3 - jmp .L999 + jmp L(999) ALIGN_3 -.L20: +L(20): movq N, %rax sarq $2, %rax - jle .L25 + jle L(25) ALIGN_3 -.L23: +L(23): movsd 0 * SIZE(X), %xmm4 movsd 0 * SIZE(Y), %xmm6 movsd 1 * SIZE(X), %xmm5 @@ -372,16 +372,16 @@ addsd %xmm9, %xmm3 decq %rax - jg .L23 + jg L(23) ALIGN_3 -.L25: +L(25): testq $3, N - je .L999 + je L(999) movq N, %rax andq $2, %rax - jle .L27 + jle L(27) movsd 0 * SIZE(X), %xmm4 movsd 0 * SIZE(Y), %xmm6 @@ -422,10 +422,10 @@ ALIGN_3 -.L27: +L(27): movq N, %rax andq $1, %rax - jle .L999 + jle L(999) movsd 0 * SIZE(X), %xmm4 movsd 0 * SIZE(Y), %xmm6 @@ -445,7 +445,7 @@ addsd %xmm9, %xmm3 ALIGN_3 -.L999: +L(999): #ifndef CONJ subsd %xmm2, %xmm0 addsd %xmm3, %xmm1 diff --git a/kernel/x86_64/zdot_sse.S b/kernel/x86_64/zdot_sse.S index f53e04cfe6..92e9ab7807 100644 --- a/kernel/x86_64/zdot_sse.S +++ b/kernel/x86_64/zdot_sse.S @@ -69,22 +69,22 @@ xorps %xmm3, %xmm3 testq N, N - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX - jne .L200 + jne L(200) cmpq $2 * SIZE, INCY - jne .L200 + jne L(200) subq $-32 * SIZE, X subq $-32 * SIZE, Y testq $SIZE, X - jne .L50 + jne L(50) -.L0x: +L(0x): testq $2 * SIZE, X - je .L10 + je L(10) #ifdef movsd xorps %xmm4, %xmm4 @@ -100,13 +100,13 @@ decq N ALIGN_3 -.L10: +L(10): testq $3 * SIZE, Y - jne .L20 + jne L(20) movq N, %rax sarq $4, %rax - jle .L15 + jle L(15) movaps -32 * SIZE(X), %xmm4 movaps -28 * SIZE(X), %xmm5 @@ -118,10 +118,10 @@ movaps -20 * SIZE(Y), %xmm11 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -206,10 +206,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): pshufd $0xb1, %xmm8, %xmm12 mulps %xmm4, %xmm8 addps %xmm8, %xmm0 @@ -270,9 +270,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $8, N - jle .L16 + jle L(16) movaps -32 * SIZE(X), %xmm4 movaps -32 * SIZE(Y), %xmm8 @@ -314,9 +314,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $4, N - jle .L17 + jle L(17) movaps -32 * SIZE(X), %xmm4 movaps -32 * SIZE(Y), %xmm8 @@ -339,9 +339,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $2, N - jle .L18 + jle L(18) movaps -32 * SIZE(X), %xmm4 movaps -32 * SIZE(Y), %xmm8 @@ -356,9 +356,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L18: +L(18): testq $1, N - jle .L98 + jle L(98) #ifdef movsd xorps %xmm4, %xmm4 @@ -374,14 +374,14 @@ addps %xmm8, %xmm0 mulps %xmm4, %xmm12 addps %xmm12, %xmm1 - jmp .L98 + jmp L(98) ALIGN_3 -.L20: +L(20): #ifdef ALIGNED_ACCESS testq $2 * SIZE, Y - jne .L30 + jne L(30) movaps -33 * SIZE(Y), %xmm8 addq $3 * SIZE, Y @@ -390,7 +390,7 @@ movq N, %rax sarq $4, %rax - jle .L25 + jle L(25) movaps -32 * SIZE(X), %xmm4 movaps -32 * SIZE(Y), %xmm9 @@ -401,10 +401,10 @@ movaps -20 * SIZE(X), %xmm7 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -505,10 +505,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): movss %xmm9, %xmm8 pshufd $0xb1, %xmm4, %xmm12 shufps $0x39, %xmm8, %xmm8 @@ -586,9 +586,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $8, N - jle .L26 + jle L(26) movaps -32 * SIZE(X), %xmm4 movaps -32 * SIZE(Y), %xmm9 @@ -637,9 +637,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $4, N - jle .L27 + jle L(27) movaps -32 * SIZE(X), %xmm4 movaps -32 * SIZE(Y), %xmm9 @@ -669,9 +669,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $2, N - jle .L28 + jle L(28) movaps -32 * SIZE(X), %xmm4 movaps -32 * SIZE(Y), %xmm9 @@ -689,9 +689,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L28: +L(28): testq $1, N - jle .L29 + jle L(29) #ifdef movsd xorps %xmm4, %xmm4 @@ -706,21 +706,21 @@ addps %xmm12, %xmm1 ALIGN_3 -.L29: +L(29): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm3, %xmm3 - jmp .L98 + jmp L(98) ALIGN_3 -.L30: +L(30): testq $SIZE, Y - jne .L40 + jne L(40) #endif movq N, %rax sarq $4, %rax - jle .L35 + jle L(35) movaps -32 * SIZE(X), %xmm4 movsd -32 * SIZE(Y), %xmm8 @@ -737,10 +737,10 @@ movhps -18 * SIZE(Y), %xmm11 decq %rax - jle .L32 + jle L(32) ALIGN_3 -.L31: +L(31): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -833,10 +833,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L32: +L(32): pshufd $0xb1, %xmm8, %xmm12 mulps %xmm4, %xmm8 addps %xmm8, %xmm0 @@ -901,9 +901,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L35: +L(35): testq $8, N - jle .L36 + jle L(36) movaps -32 * SIZE(X), %xmm4 movsd -32 * SIZE(Y), %xmm8 @@ -949,9 +949,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L36: +L(36): testq $4, N - jle .L37 + jle L(37) movaps -32 * SIZE(X), %xmm4 movsd -32 * SIZE(Y), %xmm8 @@ -977,9 +977,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L37: +L(37): testq $2, N - jle .L38 + jle L(38) movaps -32 * SIZE(X), %xmm4 movsd -32 * SIZE(Y), %xmm8 @@ -995,9 +995,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L38: +L(38): testq $1, N - jle .L98 + jle L(98) #ifdef movsd xorps %xmm4, %xmm4 @@ -1013,11 +1013,11 @@ addps %xmm8, %xmm0 mulps %xmm4, %xmm12 addps %xmm12, %xmm1 - jmp .L98 + jmp L(98) ALIGN_3 #ifdef ALIGNED_ACCESS -.L40: +L(40): movaps -35 * SIZE(Y), %xmm8 addq $1 * SIZE, Y @@ -1025,7 +1025,7 @@ movq N, %rax sarq $4, %rax - jle .L45 + jle L(45) movaps -32 * SIZE(X), %xmm4 movaps -32 * SIZE(Y), %xmm9 @@ -1036,10 +1036,10 @@ movaps -20 * SIZE(X), %xmm7 decq %rax - jle .L42 + jle L(42) ALIGN_3 -.L41: +L(41): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -1140,10 +1140,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L42: +L(42): movss %xmm9, %xmm8 pshufd $0xb1, %xmm4, %xmm12 shufps $0x93, %xmm9, %xmm8 @@ -1221,9 +1221,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L45: +L(45): testq $8, N - jle .L46 + jle L(46) movaps -32 * SIZE(X), %xmm4 movaps -32 * SIZE(Y), %xmm9 @@ -1272,9 +1272,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L46: +L(46): testq $4, N - jle .L47 + jle L(47) movaps -32 * SIZE(X), %xmm4 movaps -32 * SIZE(Y), %xmm9 @@ -1304,9 +1304,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L47: +L(47): testq $2, N - jle .L48 + jle L(48) movaps -32 * SIZE(X), %xmm4 movaps -32 * SIZE(Y), %xmm9 @@ -1324,9 +1324,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L48: +L(48): testq $1, N - jle .L49 + jle L(49) #ifdef movsd xorps %xmm4, %xmm4 @@ -1343,21 +1343,21 @@ addps %xmm12, %xmm1 ALIGN_3 -.L49: +L(49): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm3, %xmm3 - jmp .L98 + jmp L(98) ALIGN_3 #endif -.L50: +L(50): testq $SIZE, Y - jne .L70 + jne L(70) #ifdef ALIGNED_ACCESS testq $2 * SIZE, Y - je .L50x + je L(50x) #ifdef movsd xorps %xmm0, %xmm0 @@ -1377,9 +1377,9 @@ decq N ALIGN_3 -.L50x: +L(50x): testq $2 * SIZE, X - jne .L60 + jne L(60) movaps -33 * SIZE(X), %xmm8 addq $3 * SIZE, X @@ -1388,7 +1388,7 @@ movq N, %rax sarq $4, %rax - jle .L55 + jle L(55) movaps -32 * SIZE(Y), %xmm4 movaps -32 * SIZE(X), %xmm9 @@ -1399,10 +1399,10 @@ movaps -20 * SIZE(Y), %xmm7 decq %rax - jle .L52 + jle L(52) ALIGN_3 -.L51: +L(51): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -1503,10 +1503,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L51 + jg L(51) ALIGN_3 -.L52: +L(52): movss %xmm9, %xmm8 pshufd $0xb1, %xmm4, %xmm12 shufps $0x39, %xmm8, %xmm8 @@ -1584,9 +1584,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L55: +L(55): testq $8, N - jle .L56 + jle L(56) movaps -32 * SIZE(Y), %xmm4 movaps -32 * SIZE(X), %xmm9 @@ -1635,9 +1635,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L56: +L(56): testq $4, N - jle .L57 + jle L(57) movaps -32 * SIZE(Y), %xmm4 movaps -32 * SIZE(X), %xmm9 @@ -1667,9 +1667,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L57: +L(57): testq $2, N - jle .L58 + jle L(58) movaps -32 * SIZE(Y), %xmm4 movaps -32 * SIZE(X), %xmm9 @@ -1687,9 +1687,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L58: +L(58): testq $1, N - jle .L98 + jle L(98) #ifdef movsd xorps %xmm4, %xmm4 @@ -1702,10 +1702,10 @@ addps %xmm4, %xmm0 mulps %xmm8, %xmm12 addps %xmm12, %xmm1 - jmp .L98 + jmp L(98) ALIGN_3 -.L60: +L(60): movaps -35 * SIZE(X), %xmm8 addq $1 * SIZE, X @@ -1713,7 +1713,7 @@ movq N, %rax sarq $4, %rax - jle .L65 + jle L(65) movaps -32 * SIZE(Y), %xmm4 movaps -32 * SIZE(X), %xmm9 @@ -1724,10 +1724,10 @@ movaps -20 * SIZE(Y), %xmm7 decq %rax - jle .L62 + jle L(62) ALIGN_3 -.L61: +L(61): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -1828,10 +1828,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L61 + jg L(61) ALIGN_3 -.L62: +L(62): movss %xmm9, %xmm8 pshufd $0xb1, %xmm4, %xmm12 shufps $0x93, %xmm9, %xmm8 @@ -1909,9 +1909,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L65: +L(65): testq $8, N - jle .L66 + jle L(66) movaps -32 * SIZE(Y), %xmm4 movaps -32 * SIZE(X), %xmm9 @@ -1960,9 +1960,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L66: +L(66): testq $4, N - jle .L67 + jle L(67) movaps -32 * SIZE(Y), %xmm4 movaps -32 * SIZE(X), %xmm9 @@ -1992,9 +1992,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L67: +L(67): testq $2, N - jle .L68 + jle L(68) movaps -32 * SIZE(Y), %xmm4 movaps -32 * SIZE(X), %xmm9 @@ -2012,9 +2012,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L68: +L(68): testq $1, N - jle .L98 + jle L(98) #ifdef movsd xorps %xmm4, %xmm4 @@ -2029,13 +2029,13 @@ addps %xmm4, %xmm0 mulps %xmm8, %xmm12 addps %xmm12, %xmm1 - jmp .L98 + jmp L(98) ALIGN_3 #else testq $2 * SIZE, Y - je .L50x + je L(50x) #ifdef movsd xorps %xmm0, %xmm0 @@ -2055,10 +2055,10 @@ decq N ALIGN_3 -.L50x: +L(50x): movq N, %rax sarq $4, %rax - jle .L55 + jle L(55) movaps -32 * SIZE(Y), %xmm4 movlps -32 * SIZE(X), %xmm8 @@ -2075,10 +2075,10 @@ movhps -18 * SIZE(X), %xmm11 decq %rax - jle .L52 + jle L(52) ALIGN_3 -.L51: +L(51): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -2171,10 +2171,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L51 + jg L(51) ALIGN_3 -.L52: +L(52): pshufd $0xb1, %xmm4, %xmm12 mulps %xmm8, %xmm4 addps %xmm4, %xmm0 @@ -2239,9 +2239,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L55: +L(55): testq $8, N - jle .L56 + jle L(56) movaps -32 * SIZE(Y), %xmm4 movlps -32 * SIZE(X), %xmm8 @@ -2287,9 +2287,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L56: +L(56): testq $4, N - jle .L57 + jle L(57) movaps -32 * SIZE(Y), %xmm4 movlps -32 * SIZE(X), %xmm8 @@ -2315,9 +2315,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L57: +L(57): testq $2, N - jle .L58 + jle L(58) movaps -32 * SIZE(Y), %xmm4 movlps -32 * SIZE(X), %xmm8 @@ -2334,9 +2334,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L58: +L(58): testq $1, N - jle .L98 + jle L(98) #ifdef movsd xorps %xmm4, %xmm4 @@ -2352,13 +2352,13 @@ addps %xmm4, %xmm0 mulps %xmm8, %xmm12 addps %xmm12, %xmm1 - jmp .L98 + jmp L(98) ALIGN_3 #endif -.L70: +L(70): testq $2 * SIZE, Y - je .L70x + je L(70x) #ifdef movsd xorps %xmm4, %xmm4 @@ -2379,9 +2379,9 @@ decq N ALIGN_3 -.L70x: +L(70x): testq $2 * SIZE, X - jne .L80 + jne L(80) movaps -33 * SIZE(X), %xmm4 addq $3 * SIZE, X @@ -2390,7 +2390,7 @@ movq N, %rax sarq $4, %rax - jle .L75 + jle L(75) movaps -32 * SIZE(X), %xmm5 movaps -32 * SIZE(Y), %xmm9 @@ -2400,10 +2400,10 @@ movaps -24 * SIZE(Y), %xmm11 decq %rax - jle .L72 + jle L(72) ALIGN_3 -.L71: +L(71): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -2504,10 +2504,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L71 + jg L(71) ALIGN_3 -.L72: +L(72): movss %xmm9, %xmm8 pshufd $0x1b, %xmm8, %xmm12 movss %xmm5, %xmm4 @@ -2586,9 +2586,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L75: +L(75): testq $8, N - jle .L76 + jle L(76) movaps -32 * SIZE(X), %xmm5 movaps -32 * SIZE(Y), %xmm9 @@ -2638,9 +2638,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L76: +L(76): testq $4, N - jle .L77 + jle L(77) movaps -32 * SIZE(X), %xmm5 movaps -32 * SIZE(Y), %xmm9 @@ -2670,9 +2670,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L77: +L(77): testq $2, N - jle .L78 + jle L(78) movaps -32 * SIZE(X), %xmm5 movaps -32 * SIZE(Y), %xmm9 @@ -2689,9 +2689,9 @@ movaps %xmm9, %xmm8 ALIGN_3 -.L78: +L(78): testq $1, N - jle .L79 + jle L(79) xorps %xmm5, %xmm5 movss %xmm5, %xmm4 @@ -2707,15 +2707,15 @@ addps %xmm12, %xmm1 ALIGN_3 -.L79: +L(79): shufps $0x39, %xmm0, %xmm0 shufps $0x39, %xmm1, %xmm1 shufps $0x39, %xmm2, %xmm2 shufps $0x39, %xmm3, %xmm3 - jmp .L98 + jmp L(98) ALIGN_3 -.L80: +L(80): movsd -33 * SIZE(X), %xmm4 movhps -31 * SIZE(X), %xmm4 addq $3 * SIZE, X @@ -2724,7 +2724,7 @@ movq N, %rax sarq $4, %rax - jle .L85 + jle L(85) movsd -32 * SIZE(X), %xmm5 movhps -30 * SIZE(X), %xmm5 @@ -2739,10 +2739,10 @@ movaps -24 * SIZE(Y), %xmm11 decq %rax - jle .L82 + jle L(82) ALIGN_3 -.L81: +L(81): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -2851,10 +2851,10 @@ subq $-32 * SIZE, Y decq %rax - jg .L81 + jg L(81) ALIGN_3 -.L82: +L(82): movss %xmm9, %xmm8 pshufd $0x1b, %xmm8, %xmm12 movss %xmm5, %xmm4 @@ -2938,9 +2938,9 @@ subq $-32 * SIZE, Y ALIGN_3 -.L85: +L(85): testq $8, N - jle .L86 + jle L(86) movsd -32 * SIZE(X), %xmm5 movhps -30 * SIZE(X), %xmm5 @@ -2994,9 +2994,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L86: +L(86): testq $4, N - jle .L87 + jle L(87) movsd -32 * SIZE(X), %xmm5 movhps -30 * SIZE(X), %xmm5 @@ -3029,9 +3029,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L87: +L(87): testq $2, N - jle .L88 + jle L(88) movsd -32 * SIZE(X), %xmm5 movhps -30 * SIZE(X), %xmm5 @@ -3049,9 +3049,9 @@ movaps %xmm9, %xmm8 ALIGN_3 -.L88: +L(88): testq $1, N - jle .L89 + jle L(89) xorps %xmm5, %xmm5 movss %xmm5, %xmm4 @@ -3067,18 +3067,18 @@ addps %xmm12, %xmm1 ALIGN_3 -.L89: +L(89): shufps $0x39, %xmm0, %xmm0 shufps $0x39, %xmm1, %xmm1 shufps $0x39, %xmm2, %xmm2 shufps $0x39, %xmm3, %xmm3 - jmp .L98 + jmp L(98) ALIGN_3 -.L200: +L(200): movq N, %rax sarq $4, %rax - jle .L205 + jle L(205) movsd (X), %xmm4 addq INCX, X @@ -3117,10 +3117,10 @@ addq INCY, Y decq %rax - jle .L204 + jle L(204) ALIGN_3 -.L203: +L(203): pshufd $0xb1, %xmm8, %xmm12 mulps %xmm4, %xmm8 addps %xmm8, %xmm0 @@ -3235,10 +3235,10 @@ addps %xmm12, %xmm3 decq %rax - jg .L203 + jg L(203) ALIGN_3 -.L204: +L(204): pshufd $0xb1, %xmm8, %xmm12 mulps %xmm4, %xmm8 addps %xmm8, %xmm0 @@ -3320,9 +3320,9 @@ addps %xmm12, %xmm3 ALIGN_3 -.L205: +L(205): testq $8, N - jle .L206 + jle L(206) movsd (X), %xmm4 addq INCX, X @@ -3385,9 +3385,9 @@ addps %xmm12, %xmm3 ALIGN_3 -.L206: +L(206): testq $4, N - jle .L207 + jle L(207) movsd (X), %xmm4 addq INCX, X @@ -3420,9 +3420,9 @@ addps %xmm12, %xmm3 ALIGN_3 -.L207: +L(207): testq $2, N - jle .L208 + jle L(208) movsd (X), %xmm4 addq INCX, X @@ -3440,9 +3440,9 @@ addps %xmm12, %xmm1 ALIGN_3 -.L208: +L(208): testq $1, N - jle .L98 + jle L(98) #ifdef movsd xorps %xmm4, %xmm4 @@ -3460,7 +3460,7 @@ addps %xmm12, %xmm1 ALIGN_3 -.L98: +L(98): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -3474,7 +3474,7 @@ pshufd $1, %xmm1, %xmm3 ALIGN_3 -.L999: +L(999): #ifndef CONJ subss %xmm2, %xmm0 addss %xmm3, %xmm1 diff --git a/kernel/x86_64/zdot_sse2.S b/kernel/x86_64/zdot_sse2.S index 3ab6f45174..2be2fb0f66 100644 --- a/kernel/x86_64/zdot_sse2.S +++ b/kernel/x86_64/zdot_sse2.S @@ -84,25 +84,25 @@ xorps %xmm3, %xmm3 cmpq $0, N - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX - jne .L50 + jne L(50) cmpq $2 * SIZE, INCY - jne .L50 + jne L(50) subq $-16 * SIZE, X subq $-16 * SIZE, Y testq $SIZE, Y - jne .L30 + jne L(30) testq $SIZE, X - jne .L20 + jne L(20) movq N, %rax sarq $3, %rax - jle .L15 + jle L(15) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -114,10 +114,10 @@ movaps -10 * SIZE(Y), %xmm11 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -202,10 +202,10 @@ subq $-16 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): pshufd $0x4e, %xmm8, %xmm12 mulpd %xmm4, %xmm8 addpd %xmm8, %xmm0 @@ -266,9 +266,9 @@ subq $-16 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $4, N - jle .L16 + jle L(16) movaps -16 * SIZE(X), %xmm4 movaps -16 * SIZE(Y), %xmm8 @@ -308,9 +308,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $2, N - jle .L17 + jle L(17) movaps -16 * SIZE(X), %xmm4 movaps -16 * SIZE(Y), %xmm8 @@ -333,9 +333,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $1, N - jle .L98 + jle L(98) movaps -16 * SIZE(X), %xmm4 movaps -16 * SIZE(Y), %xmm8 @@ -345,13 +345,13 @@ addpd %xmm8, %xmm0 mulpd %xmm4, %xmm12 addpd %xmm12, %xmm1 - jmp .L98 + jmp L(98) ALIGN_3 -.L20: +L(20): movq N, %rax sarq $3, %rax - jle .L25 + jle L(25) MOVLPS -16 * SIZE(X), %xmm4 movhps -15 * SIZE(X), %xmm4 @@ -367,10 +367,10 @@ movaps -10 * SIZE(Y), %xmm11 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -463,10 +463,10 @@ subq $-16 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): pshufd $0x4e, %xmm8, %xmm12 mulpd %xmm4, %xmm8 @@ -532,9 +532,9 @@ subq $-16 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $4, N - jle .L26 + jle L(26) MOVLPS -16 * SIZE(X), %xmm4 movhps -15 * SIZE(X), %xmm4 @@ -580,9 +580,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $2, N - jle .L27 + jle L(27) MOVLPS -16 * SIZE(X), %xmm4 movhps -15 * SIZE(X), %xmm4 @@ -608,9 +608,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $1, N - jle .L98 + jle L(98) MOVLPS -16 * SIZE(X), %xmm4 movhps -15 * SIZE(X), %xmm4 @@ -621,16 +621,16 @@ addpd %xmm8, %xmm0 mulpd %xmm4, %xmm12 addpd %xmm12, %xmm1 - jmp .L98 + jmp L(98) ALIGN_3 -.L30: +L(30): testq $SIZE, X - jne .L40 + jne L(40) movq N, %rax sarq $3, %rax - jle .L35 + jle L(35) MOVLPS -16 * SIZE(Y), %xmm4 movhps -15 * SIZE(Y), %xmm4 @@ -646,10 +646,10 @@ movaps -10 * SIZE(X), %xmm11 decq %rax - jle .L32 + jle L(32) ALIGN_3 -.L31: +L(31): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -742,10 +742,10 @@ subq $-16 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L32: +L(32): pshufd $0x4e, %xmm8, %xmm12 mulpd %xmm4, %xmm8 @@ -811,9 +811,9 @@ subq $-16 * SIZE, Y ALIGN_3 -.L35: +L(35): testq $4, N - jle .L36 + jle L(36) MOVLPS -16 * SIZE(Y), %xmm4 movhps -15 * SIZE(Y), %xmm4 @@ -859,9 +859,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L36: +L(36): testq $2, N - jle .L37 + jle L(37) MOVLPS -16 * SIZE(Y), %xmm4 movhps -15 * SIZE(Y), %xmm4 @@ -887,12 +887,12 @@ addq $4 * SIZE, Y ALIGN_3 -.L37: +L(37): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm3, %xmm3 testq $1, N - jle .L98 + jle L(98) MOVLPS -16 * SIZE(Y), %xmm4 movhps -15 * SIZE(Y), %xmm4 @@ -904,10 +904,10 @@ mulpd %xmm4, %xmm12 SHUFPD_1 %xmm12, %xmm12 addpd %xmm12, %xmm1 - jmp .L98 + jmp L(98) ALIGN_3 -.L40: +L(40): movhps -16 * SIZE(X), %xmm4 addq $SIZE, X movhps -16 * SIZE(Y), %xmm8 @@ -915,7 +915,7 @@ movq N, %rax sarq $3, %rax - jle .L45 + jle L(45) movaps -16 * SIZE(X), %xmm5 movaps -16 * SIZE(Y), %xmm9 @@ -924,10 +924,10 @@ movaps -12 * SIZE(X), %xmm7 movaps -12 * SIZE(Y), %xmm11 decq %rax - jle .L42 + jle L(42) ALIGN_3 -.L41: +L(41): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -1028,10 +1028,10 @@ subq $-16 * SIZE, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L42: +L(42): movsd %xmm9, %xmm8 pshufd $0x4e, %xmm8, %xmm12 movsd %xmm5, %xmm4 @@ -1110,9 +1110,9 @@ subq $-16 * SIZE, Y ALIGN_3 -.L45: +L(45): testq $4, N - jle .L46 + jle L(46) movaps -16 * SIZE(X), %xmm5 movaps -16 * SIZE(Y), %xmm9 @@ -1161,9 +1161,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L46: +L(46): testq $2, N - jle .L47 + jle L(47) movaps -16 * SIZE(X), %xmm5 movaps -16 * SIZE(Y), %xmm9 @@ -1194,9 +1194,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L47: +L(47): testq $1, N - jle .L48 + jle L(48) movlps -16 * SIZE(X), %xmm4 movlps -16 * SIZE(Y), %xmm8 @@ -1208,18 +1208,18 @@ addpd %xmm12, %xmm1 ALIGN_3 -.L48: +L(48): SHUFPD_1 %xmm0, %xmm0 SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm2, %xmm2 SHUFPD_1 %xmm3, %xmm3 - jmp .L98 + jmp L(98) ALIGN_3 -.L50: +L(50): movq N, %rax sarq $3, %rax - jle .L55 + jle L(55) MOVLPS 0 * SIZE(X), %xmm4 movhps 1 * SIZE(X), %xmm4 @@ -1250,10 +1250,10 @@ addq INCY, Y decq %rax - jle .L54 + jle L(54) ALIGN_3 -.L53: +L(53): pshufd $0x4e, %xmm8, %xmm12 mulpd %xmm4, %xmm8 addpd %xmm8, %xmm0 @@ -1353,10 +1353,10 @@ addpd %xmm12, %xmm3 decq %rax - jg .L53 + jg L(53) ALIGN_3 -.L54: +L(54): pshufd $0x4e, %xmm8, %xmm12 mulpd %xmm4, %xmm8 addpd %xmm8, %xmm0 @@ -1430,9 +1430,9 @@ addpd %xmm12, %xmm3 ALIGN_3 -.L55: +L(55): testq $4, N - jle .L56 + jle L(56) MOVLPS 0 * SIZE(X), %xmm4 movhps 1 * SIZE(X), %xmm4 @@ -1487,9 +1487,9 @@ addpd %xmm12, %xmm3 ALIGN_3 -.L56: +L(56): testq $2, N - jle .L57 + jle L(57) MOVLPS 0 * SIZE(X), %xmm4 movhps 1 * SIZE(X), %xmm4 @@ -1518,9 +1518,9 @@ addpd %xmm12, %xmm3 ALIGN_3 -.L57: +L(57): testq $1, N - jle .L98 + jle L(98) MOVLPS 0 * SIZE(X), %xmm4 movhps 1 * SIZE(X), %xmm4 @@ -1534,14 +1534,14 @@ addpd %xmm12, %xmm1 ALIGN_3 -.L98: +L(98): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 pshufd $0x4e, %xmm0, %xmm2 pshufd $0x4e, %xmm1, %xmm3 -.L999: +L(999): #ifndef CONJ subsd %xmm2, %xmm0 addsd %xmm3, %xmm1 diff --git a/kernel/x86_64/zgemm3m_kernel_2x8_nehalem.S b/kernel/x86_64/zgemm3m_kernel_2x8_nehalem.S index 0069066ad7..8be6472ff1 100644 --- a/kernel/x86_64/zgemm3m_kernel_2x8_nehalem.S +++ b/kernel/x86_64/zgemm3m_kernel_2x8_nehalem.S @@ -169,10 +169,10 @@ movq N, J sarq $3, J NOBRANCH - jle .L30 + jle L(30) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -189,10 +189,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -253,10 +253,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm12 @@ -379,10 +379,10 @@ subq $-8 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -392,10 +392,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addpd %xmm1, %xmm12 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm13 @@ -431,10 +431,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): addpd %xmm1, %xmm12 addpd %xmm2, %xmm13 addpd %xmm3, %xmm14 @@ -605,13 +605,13 @@ decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M BRANCH - jle .L29 + jle L(29) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -652,10 +652,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -717,10 +717,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -730,10 +730,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -753,10 +753,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): leaq (LDC, LDC, 2), %rax movsd 0 * SIZE(CO1), %xmm0 @@ -832,7 +832,7 @@ movhps %xmm3, 1 * SIZE(CO2, %rax) ALIGN_4 -.L29: +L(29): #if defined(TRMMKERNEL) && !defined(LEFT) addq $8, KK #endif @@ -843,12 +843,12 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L30: +L(30): testq $4, N - jle .L50 + jle L(50) ALIGN_4 #if defined(TRMMKERNEL) && defined(LEFT) @@ -863,10 +863,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L31: +L(31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -913,10 +913,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -988,10 +988,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -1001,10 +1001,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): addpd %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm9 @@ -1026,10 +1026,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): addpd %xmm1, %xmm8 addpd %xmm2, %xmm9 addpd %xmm3, %xmm10 @@ -1119,13 +1119,13 @@ addq $4 * SIZE, CO2 decq I BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L40: +L(40): testq $1, M BRANCH - jle .L49 + jle L(49) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1166,10 +1166,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1207,10 +1207,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -1220,10 +1220,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movaps -14 * SIZE(BO), %xmm1 @@ -1237,10 +1237,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1281,7 +1281,7 @@ movhps %xmm3, 1 * SIZE(CO2, LDC) ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif @@ -1291,9 +1291,9 @@ leaq (C, LDC, 4), C ALIGN_4 -.L50: +L(50): testq $2, N - jle .L70 + jle L(70) ALIGN_4 #if defined(TRMMKERNEL) && defined(LEFT) @@ -1308,10 +1308,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1354,10 +1354,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -1397,13 +1397,13 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 ALIGN_3 -.L55: +L(55): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -1413,10 +1413,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addpd %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 addpd %xmm2, %xmm9 @@ -1430,10 +1430,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): addpd %xmm1, %xmm8 addpd %xmm2, %xmm9 @@ -1481,13 +1481,13 @@ addq $4 * SIZE, CO2 decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L69 + jle L(69) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1526,10 +1526,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): mulpd %xmm0, %xmm1 movddup -15 * SIZE(AO), %xmm0 addpd %xmm1, %xmm8 @@ -1555,10 +1555,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -1568,10 +1568,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): mulpd %xmm0, %xmm1 movddup -15 * SIZE(AO), %xmm0 addpd %xmm1, %xmm8 @@ -1582,10 +1582,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addpd %xmm9, %xmm8 movsd 0 * SIZE(CO1), %xmm0 @@ -1607,7 +1607,7 @@ movhps %xmm1, 1 * SIZE(CO2) ALIGN_4 -.L69: +L(69): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -1617,9 +1617,9 @@ leaq (C, LDC, 2), C ALIGN_4 -.L70: +L(70): testq $1, N - jle .L999 + jle L(999) ALIGN_4 #if defined(TRMMKERNEL) && defined(LEFT) @@ -1633,10 +1633,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L80 + jle L(80) ALIGN_4 -.L71: +L(71): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1678,10 +1678,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm1, %xmm8 @@ -1709,12 +1709,12 @@ subq $1, %rax BRANCH - jg .L72 + jg L(72) addpd %xmm9, %xmm8 ALIGN_3 -.L75: +L(75): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -1724,10 +1724,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addpd %xmm1, %xmm8 movddup -16 * SIZE(BO), %xmm1 mulpd %xmm0, %xmm1 @@ -1738,10 +1738,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addpd %xmm1, %xmm8 movsd 0 * SIZE(CO1), %xmm0 @@ -1765,13 +1765,13 @@ addq $4 * SIZE, CO1 decq I BRANCH - jg .L71 + jg L(71) ALIGN_4 -.L80: +L(80): testq $1, M BRANCH - jle .L999 + jle L(999) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1819,10 +1819,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): mulpd %xmm0, %xmm1 #ifndef TRMMKERNEL movapd -14 * SIZE(AO), %xmm0 @@ -1858,12 +1858,12 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) addpd %xmm9, %xmm8 ALIGN_3 -.L85: +L(85): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -1873,10 +1873,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): mulsd %xmm0, %xmm1 movsd -15 * SIZE(AO), %xmm0 addsd %xmm1, %xmm8 @@ -1887,10 +1887,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): haddpd %xmm8, %xmm8 movsd 0 * SIZE(CO1), %xmm0 @@ -1904,7 +1904,7 @@ movhps %xmm0, 1 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm3m_kernel_4x2_atom.S b/kernel/x86_64/zgemm3m_kernel_4x2_atom.S index 1049c01312..49b449746c 100644 --- a/kernel/x86_64/zgemm3m_kernel_4x2_atom.S +++ b/kernel/x86_64/zgemm3m_kernel_4x2_atom.S @@ -141,10 +141,10 @@ movq N, J sarq $1, J - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): movq C, CO1 leaq (C, LDC, 1), CO2 leaq (C, LDC, 2), C @@ -157,10 +157,10 @@ movq M, I sarq $2, I - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): movq B, BO prefetcht0 0 * SIZE(BB) @@ -190,10 +190,10 @@ movq K, %rax sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): addsd %xmm2, %xmm13 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps %xmm0, %xmm2 @@ -336,18 +336,18 @@ mulsd %xmm3, %xmm6 movsd 1 * SIZE(BO), %xmm3 - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): movq K, %rax andq $3, %rax BRANCH BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): addsd %xmm2, %xmm13 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -386,10 +386,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): movsd ALPHA_R, %xmm4 addsd %xmm2, %xmm13 movsd ALPHA_I, %xmm5 @@ -464,12 +464,12 @@ addq $8 * SIZE, CO2 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M - jle .L30 + jle L(30) movq B, BO @@ -490,10 +490,10 @@ movq K, %rax sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addsd %xmm2, %xmm9 movaps %xmm0, %xmm2 @@ -571,10 +571,10 @@ addq $8 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): movq K, %rax movsd ALPHA_R, %xmm5 movsd ALPHA_I, %xmm7 @@ -582,10 +582,10 @@ andq $3, %rax BRANCH BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): addsd %xmm2, %xmm9 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -608,10 +608,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): addsd %xmm2, %xmm9 addsd %xmm6, %xmm11 @@ -653,9 +653,9 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_4 movq B, BO @@ -672,10 +672,10 @@ movq K, %rax sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): addsd %xmm5, %xmm8 movsd 2 * SIZE(BO), %xmm5 mulsd %xmm0, %xmm1 @@ -717,10 +717,10 @@ addq $8 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): movq K, %rax addsd %xmm5, %xmm8 @@ -732,10 +732,10 @@ andq $3, %rax BRANCH BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulsd %xmm0, %xmm1 addq $2 * SIZE, BO mulsd %xmm0, %xmm3 @@ -749,10 +749,10 @@ addq $1 * SIZE, AO decq %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): movaps %xmm8, %xmm10 movaps %xmm9, %xmm11 @@ -772,15 +772,15 @@ movsd %xmm11, 1 * SIZE(CO2) ALIGN_4 -.L39: +L(39): movq BO, B decq J # j -- - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $1, N - je .L999 + je L(999) movq C, CO1 addq LDC, C @@ -789,10 +789,10 @@ movq M, I sarq $2, I - jle .L50 + jle L(50) ALIGN_4 -.L41: +L(41): movq B, BO movsd 0 * SIZE(AO), %xmm0 @@ -814,10 +814,10 @@ movq K, %rax sarq $2, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): addsd %xmm9, %xmm8 movsd 4 * SIZE(AO), %xmm9 mulsd %xmm4, %xmm0 @@ -891,10 +891,10 @@ mulsd %xmm5, %xmm15 movsd 1 * SIZE(BO), %xmm5 - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): movq K, %rax movsd ALPHA_R, %xmm6 @@ -908,10 +908,10 @@ andq $3, %rax BRANCH BRANCH - je .L49 + je L(49) ALIGN_4 -.L46: +L(46): mulsd %xmm4, %xmm0 mulsd %xmm4, %xmm1 mulsd %xmm4, %xmm2 @@ -931,10 +931,10 @@ addq $1 * SIZE, BO decq %rax BRANCH - jg .L46 + jg L(46) ALIGN_4 -.L49: +L(49): movaps %xmm8, %xmm9 movaps %xmm10, %xmm11 movaps %xmm12, %xmm13 @@ -970,12 +970,12 @@ addq $8 * SIZE, CO1 decq I # i -- - jg .L41 + jg L(41) ALIGN_4 -.L50: +L(50): testq $2, M - jle .L60 + jle L(60) movq B, BO @@ -991,10 +991,10 @@ movq K, %rax sarq $2, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): addsd %xmm2, %xmm8 movsd 2 * SIZE(AO), %xmm2 mulsd %xmm4, %xmm0 @@ -1035,10 +1035,10 @@ mulsd %xmm5, %xmm3 movsd 1 * SIZE(BO), %xmm5 - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): movq K, %rax movsd ALPHA_R, %xmm6 movsd ALPHA_I, %xmm7 @@ -1049,10 +1049,10 @@ andq $3, %rax BRANCH BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): mulsd %xmm4, %xmm0 mulsd %xmm4, %xmm1 movsd 1 * SIZE(BO), %xmm4 @@ -1066,10 +1066,10 @@ addq $1 * SIZE, BO decq %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): movaps %xmm8, %xmm9 movaps %xmm10, %xmm11 @@ -1091,9 +1091,9 @@ addq $4 * SIZE, CO1 ALIGN_4 -.L60: +L(60): testq $1, M - je .L999 + je L(999) ALIGN_4 movq B, BO @@ -1112,10 +1112,10 @@ movq K, %rax sarq $2, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): addsd %xmm5, %xmm8 movsd 2 * SIZE(BO), %xmm5 mulsd %xmm0, %xmm1 @@ -1140,13 +1140,13 @@ addq $4 * SIZE, BO decq %rax - jne .L62 + jne L(62) addsd %xmm5, %xmm8 addsd %xmm7, %xmm9 ALIGN_4 -.L65: +L(65): movq K, %rax movsd ALPHA_R, %xmm6 movsd ALPHA_I, %xmm7 @@ -1154,10 +1154,10 @@ andq $3, %rax BRANCH BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): movsd 0 * SIZE(AO), %xmm0 movsd 0 * SIZE(BO), %xmm1 @@ -1169,10 +1169,10 @@ decq %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addsd %xmm9, %xmm8 movaps %xmm8, %xmm9 @@ -1186,7 +1186,7 @@ movsd %xmm9, 1 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm3m_kernel_4x4_barcelona.S b/kernel/x86_64/zgemm3m_kernel_4x4_barcelona.S index 76ed76d811..e833ad69ae 100644 --- a/kernel/x86_64/zgemm3m_kernel_4x4_barcelona.S +++ b/kernel/x86_64/zgemm3m_kernel_4x4_barcelona.S @@ -487,10 +487,10 @@ #endif movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #ifdef BUFFERED leaq 16 * SIZE + BUFFER, BO #endif @@ -505,10 +505,10 @@ #ifdef BUFFERED movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_3 -.L02: +L(02): prefetch (RPREFETCHSIZE + 0) * SIZE(B) movaps (B), %xmm0 @@ -545,17 +545,17 @@ subq $-16 * SIZE, B subq $1, %rax - jne .L02 + jne L(02) ALIGN_3 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_3 -.L04: +L(04): movaps (B), %xmm0 movaps %xmm0, -16 * SIZE(BO) @@ -565,20 +565,20 @@ addq $4 * SIZE, B addq $4 * SIZE, BO subq $1, %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): #endif movq A, AO # aoffset = a movq B, BB movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -643,10 +643,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -656,7 +656,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -666,7 +666,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -676,7 +676,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -686,7 +686,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -696,7 +696,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -706,7 +706,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -716,7 +716,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -726,10 +726,10 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) BRANCH - jl .L12 + jl L(12) ALIGN_4 -.L15: +L(15): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -738,7 +738,7 @@ movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_4 @@ -751,14 +751,14 @@ subq $-16 * SIZE, AO ALIGN_4 -.L16: +L(16): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L19 + je L(19) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -766,7 +766,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -793,10 +793,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L17 + jl L(17) ALIGN_4 -.L19: +L(19): movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 movsd 2 * SIZE(CO1), %xmm1 @@ -942,18 +942,18 @@ decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $3, M - je .L39 + je L(39) testq $2, M - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1003,10 +1003,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L26 + je L(26) ALIGN_4 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -1062,10 +1062,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L22 + jl L(22) ALIGN_4 -.L26: +L(26): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1074,7 +1074,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L29 + je L(29) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -1082,7 +1082,7 @@ negq %rax ALIGN_4 -.L27: +L(27): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -1098,10 +1098,10 @@ movddup -11 * SIZE(BO, %rax, 4), %xmm5 addq $SIZE, %rax - jl .L27 + jl L(27) ALIGN_4 -.L29: +L(29): movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 movsd 2 * SIZE(CO1), %xmm1 @@ -1178,9 +1178,9 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1231,10 +1231,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L36 + je L(36) ALIGN_4 -.L32: +L(32): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO, %rax, 4), %xmm0 addpd %xmm1, %xmm8 @@ -1262,10 +1262,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L32 + jl L(32) ALIGN_4 -.L36: +L(36): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1274,7 +1274,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L38 + je L(38) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -1282,7 +1282,7 @@ negq %rax ALIGN_4 -.L37: +L(37): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO, %rax, 4), %xmm0 addpd %xmm1, %xmm8 @@ -1291,10 +1291,10 @@ movddup -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L37 + jl L(37) ALIGN_4 -.L38: +L(38): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1341,7 +1341,7 @@ ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif @@ -1352,18 +1352,18 @@ leaq (C, LDC, 4), C # c += 4 * ldc decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $3, N - je .L999 + je L(999) testq $2, N - je .L80 + je L(80) ALIGN_4 -.L41: +L(41): #ifdef BUFFERED leaq 16 * SIZE + BUFFER, BO #endif @@ -1376,10 +1376,10 @@ #ifdef BUFFERED movq K, %rax sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): prefetchnta (RPREFETCHSIZE + 0) * SIZE(B) movaps (B), %xmm0 @@ -1400,27 +1400,27 @@ subq $-8 * SIZE, B subq $1, %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): movq K, %rax andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movaps (B), %xmm0 movaps %xmm0, -16 * SIZE(BO) addq $2 * SIZE, B addq $2 * SIZE, BO subq $1, %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): #endif movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc @@ -1428,10 +1428,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1484,10 +1484,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L56 + je L(56) ALIGN_4 -.L52: +L(52): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -1539,10 +1539,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L52 + jl L(52) ALIGN_4 -.L56: +L(56): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1551,7 +1551,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L59 + je L(59) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -1559,7 +1559,7 @@ negq %rax ALIGN_4 -.L57: +L(57): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -1574,10 +1574,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L57 + jl L(57) ALIGN_4 -.L59: +L(59): movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 movsd 2 * SIZE(CO1), %xmm1 @@ -1652,15 +1652,15 @@ addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1709,10 +1709,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L66 + je L(66) ALIGN_4 -.L62: +L(62): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -1744,10 +1744,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L62 + jl L(62) ALIGN_4 -.L66: +L(66): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1756,7 +1756,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L69 + je L(69) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -1764,7 +1764,7 @@ negq %rax ALIGN_4 -.L67: +L(67): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -1774,10 +1774,10 @@ movddup -13 * SIZE(BO, %rax, 2), %xmm3 addq $SIZE, %rax - jl .L67 + jl L(67) ALIGN_4 -.L69: +L(69): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1821,12 +1821,12 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1875,10 +1875,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L76 + je L(76) ALIGN_4 -.L72: +L(72): mulpd -16 * SIZE(BO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -12 * SIZE(AO, %rax, 1), %xmm0 @@ -1897,10 +1897,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L72 + jl L(72) ALIGN_4 -.L76: +L(76): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1909,7 +1909,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L78 + je L(78) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -1917,16 +1917,16 @@ negq %rax ALIGN_4 -.L77: +L(77): mulpd -16 * SIZE(BO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L77 + jl L(77) ALIGN_4 -.L78: +L(78): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -1952,7 +1952,7 @@ movhpd %xmm0, 1 * SIZE(CO2) ALIGN_4 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -1964,12 +1964,12 @@ leaq (C, LDC, 2), C ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L81: +L(81): #ifdef BUFFERED leaq 16 * SIZE + BUFFER, BO #endif @@ -1982,10 +1982,10 @@ #ifdef BUFFERED movq K, %rax sarq $3, %rax - jle .L83 + jle L(83) ALIGN_4 -.L82: +L(82): prefetchnta (RPREFETCHSIZE + 0) * SIZE(B) movaps (B), %xmm0 @@ -2006,37 +2006,37 @@ subq $-8 * SIZE, B subq $1, %rax - jne .L82 + jne L(82) ALIGN_4 -.L83: +L(83): movq K, %rax andq $7, %rax BRANCH - jle .L90 + jle L(90) ALIGN_4 -.L84: +L(84): movsd (B), %xmm0 movlpd %xmm0, -16 * SIZE(BO) addq $1 * SIZE, B addq $1 * SIZE, BO decq %rax - jne .L84 + jne L(84) ALIGN_4 -.L90: +L(90): #endif movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2088,10 +2088,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L96 + je L(96) ALIGN_4 -.L92: +L(92): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -2119,10 +2119,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L92 + jl L(92) ALIGN_4 -.L96: +L(96): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2131,7 +2131,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L99 + je L(99) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -2139,7 +2139,7 @@ negq %rax ALIGN_4 -.L97: +L(97): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 addpd %xmm0, %xmm8 @@ -2148,10 +2148,10 @@ movddup -15 * SIZE(BO, %rax, 1), %xmm1 addq $SIZE, %rax - jl .L97 + jl L(97) ALIGN_4 -.L99: +L(99): addpd %xmm9, %xmm8 addpd %xmm13, %xmm12 @@ -2193,15 +2193,15 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2250,10 +2250,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L106 + je L(106) ALIGN_4 -.L102: +L(102): mulpd -16 * SIZE(AO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 movddup -12 * SIZE(BO, %rax, 1), %xmm0 @@ -2272,10 +2272,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L102 + jl L(102) ALIGN_4 -.L106: +L(106): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2284,7 +2284,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L109 + je L(109) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -2292,16 +2292,16 @@ negq %rax ALIGN_4 -.L107: +L(107): movddup -16 * SIZE(BO, %rax, 1), %xmm0 mulpd -16 * SIZE(AO, %rax, 2), %xmm0 addpd %xmm0, %xmm8 addq $SIZE, %rax - jl .L107 + jl L(107) ALIGN_4 -.L109: +L(109): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -2327,12 +2327,12 @@ addq $4 * SIZE, CO1 ALIGN_4 -.L110: +L(110): testq $1, M - je .L999 + je L(999) ALIGN_4 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2377,10 +2377,10 @@ leaq (BO, %rax, 1), BO negq %rax NOBRANCH - je .L116 + je L(116) ALIGN_4 -.L112: +L(112): mulpd -16 * SIZE(BO, %rax, 1), %xmm0 addpd %xmm0, %xmm8 movapd -12 * SIZE(AO, %rax, 1), %xmm0 @@ -2391,10 +2391,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L112 + jl L(112) ALIGN_4 -.L116: +L(116): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2403,7 +2403,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L118 + je L(118) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO @@ -2411,16 +2411,16 @@ negq %rax ALIGN_4 -.L117: +L(117): mulsd -16 * SIZE(BO, %rax, 1), %xmm0 addsd %xmm0, %xmm8 movsd -15 * SIZE(AO, %rax, 1), %xmm0 addq $SIZE, %rax - jl .L117 + jl L(117) ALIGN_4 -.L118: +L(118): addpd %xmm9, %xmm8 haddpd %xmm8, %xmm8 @@ -2436,7 +2436,7 @@ movhpd %xmm0, 1 * SIZE(CO1) ALIGN_3 -.L999: +L(999): movq %rbx, %rsp movq (%rsp), %rbx diff --git a/kernel/x86_64/zgemm3m_kernel_4x4_core2.S b/kernel/x86_64/zgemm3m_kernel_4x4_core2.S index a78890dea6..fdef5f9a4f 100644 --- a/kernel/x86_64/zgemm3m_kernel_4x4_core2.S +++ b/kernel/x86_64/zgemm3m_kernel_4x4_core2.S @@ -165,10 +165,10 @@ movq N, J sarq $2, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): /* Copying to Sub Buffer */ leaq 16 * SIZE + BUFFER, BO @@ -180,10 +180,10 @@ movq K, %rax sarq $2, %rax NOBRANCH - jle .L05 + jle L(05) ALIGN_4 -.L02: +L(02): movapd -16 * SIZE(B), %xmm0 prefetchnta (PREFETCH_R + 0) * SIZE(B) movapd -14 * SIZE(B), %xmm1 @@ -240,18 +240,18 @@ subq $-32 * SIZE, BO decq %rax BRANCH - jne .L02 + jne L(02) ALIGN_4 -.L05: +L(05): movq K, %rax andq $3, %rax BRANCH BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L06: +L(06): movapd -16 * SIZE(B), %xmm0 movapd -14 * SIZE(B), %xmm1 @@ -269,10 +269,10 @@ addq $8 * SIZE, BO decq %rax BRANCH - jne .L06 + jne L(06) ALIGN_4 -.L10: +L(10): leaq (PREFETCH_R + 0) * SIZE(B), BB movq C, CO1 # coffset1 = c @@ -282,10 +282,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -344,10 +344,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): PADDING; addpd %xmm2, %xmm10 movaps -16 * SIZE(BO), %xmm2 @@ -476,10 +476,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): prefetcht2 -8 * SIZE(BB) #ifndef TRMMKERNEL @@ -489,10 +489,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): addpd %xmm2, %xmm10 movaps -16 * SIZE(BO), %xmm2 addpd %xmm3, %xmm14 @@ -527,10 +527,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): movapd ALPHA, %xmm7 addpd %xmm2, %xmm10 @@ -682,13 +682,13 @@ addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -731,10 +731,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_4 -.L21: +L(21): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm2, %xmm8 movapd -16 * SIZE(BO), %xmm2 @@ -796,10 +796,10 @@ subq $-32 * SIZE, BO subq $1, %rax BRANCH - jg .L21 + jg L(21) ALIGN_4 -.L25: +L(25): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -809,10 +809,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): addpd %xmm2, %xmm8 movapd -16 * SIZE(BO), %xmm2 mulpd %xmm0, %xmm2 @@ -831,10 +831,10 @@ addq $8 * SIZE, BO subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): addpd %xmm2, %xmm8 addpd %xmm3, %xmm9 addpd %xmm4, %xmm10 @@ -916,10 +916,10 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $1, M BRANCH - jle .L39 + jle L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -963,10 +963,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_4 -.L31: +L(31): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addsd %xmm2, %xmm8 movsd -16 * SIZE(BO), %xmm2 @@ -1028,10 +1028,10 @@ subq $-32 * SIZE, BO subq $1, %rax BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L35: +L(35): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1041,10 +1041,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): addsd %xmm2, %xmm8 movsd -16 * SIZE(BO), %xmm2 mulsd %xmm0, %xmm2 @@ -1063,10 +1063,10 @@ addq $8 * SIZE, BO subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): addsd %xmm2, %xmm8 addsd %xmm3, %xmm9 addsd %xmm4, %xmm10 @@ -1113,7 +1113,7 @@ movhpd %xmm0, 1 * SIZE(CO2, LDC, 2) ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif @@ -1121,16 +1121,16 @@ leaq (C, LDC, 4), C subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $2, N BRANCH - jle .L80 + jle L(80) ALIGN_4 -.L41: +L(41): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -1141,12 +1141,12 @@ movq K, %rax sarq $3, %rax - jle .L43 + jle L(43) addq %rax, %rax ALIGN_4 -.L42: +L(42): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 movddup -14 * SIZE(B), %xmm10 @@ -1169,17 +1169,17 @@ addq $16 * SIZE, BO subq $1, %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): movq K, %rax andq $7, %rax BRANCH - jle .L45 + jle L(45) ALIGN_4 -.L44: +L(44): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 @@ -1189,20 +1189,20 @@ addq $2 * SIZE, B addq $4 * SIZE, BO subq $1, %rax - jne .L44 + jne L(44) ALIGN_4 -.L45: +L(45): movq C, CO1 leaq (C, LDC, 1), CO2 movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1249,10 +1249,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L55 + jle L(55) ALIGN_4 -.L51: +L(51): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm2, %xmm8 @@ -1320,10 +1320,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L51 + jg L(51) ALIGN_4 -.L55: +L(55): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1332,10 +1332,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): addpd %xmm2, %xmm8 movapd -16 * SIZE(BO), %xmm2 addpd %xmm3, %xmm12 @@ -1355,10 +1355,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): addpd %xmm2, %xmm8 addpd %xmm3, %xmm12 addpd %xmm4, %xmm9 @@ -1437,12 +1437,12 @@ addq $8 * SIZE, CO1 addq $8 * SIZE, CO2 subq $1, I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): testq $2, M - jle .L70 + jle L(70) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1486,10 +1486,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L65 + jle L(65) ALIGN_4 -.L61: +L(61): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm2, %xmm8 @@ -1525,10 +1525,10 @@ subq $ -8 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L61 + jg L(61) ALIGN_4 -.L65: +L(65): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1537,10 +1537,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): addpd %xmm2, %xmm8 movapd -16 * SIZE(BO), %xmm2 mulpd %xmm0, %xmm2 @@ -1552,10 +1552,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addpd %xmm2, %xmm8 addpd %xmm3, %xmm9 addpd %xmm4, %xmm10 @@ -1605,9 +1605,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L70: +L(70): testq $1, M - jle .L79 + jle L(79) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1650,10 +1650,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L75 + jle L(75) ALIGN_4 -.L71: +L(71): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addsd %xmm2, %xmm8 @@ -1689,10 +1689,10 @@ subq $ -4 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L71 + jg L(71) ALIGN_4 -.L75: +L(75): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1701,10 +1701,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): addsd %xmm2, %xmm8 movsd -16 * SIZE(BO), %xmm2 mulsd %xmm0, %xmm2 @@ -1716,10 +1716,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addsd %xmm2, %xmm8 addsd %xmm3, %xmm9 addsd %xmm4, %xmm10 @@ -1749,7 +1749,7 @@ movhpd %xmm0, 1 * SIZE(CO2) ALIGN_4 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -1757,13 +1757,13 @@ leaq (C, LDC, 2), C ALIGN_4 -.L80: +L(80): testq $1, N BRANCH - jle .L999 + jle L(999) ALIGN_4 -.L81: +L(81): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -1774,12 +1774,12 @@ movq K, %rax sarq $4, %rax - jle .L83 + jle L(83) addq %rax, %rax ALIGN_4 -.L82: +L(82): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 movddup -14 * SIZE(B), %xmm10 @@ -1801,17 +1801,17 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO subq $1, %rax - jne .L82 + jne L(82) ALIGN_4 -.L83: +L(83): movq K, %rax andq $15, %rax BRANCH - jle .L85 + jle L(85) ALIGN_4 -.L84: +L(84): movddup -16 * SIZE(B), %xmm8 movapd %xmm8, 0 * SIZE(BO) @@ -1819,19 +1819,19 @@ addq $1 * SIZE, B addq $2 * SIZE, BO subq $1, %rax - jne .L84 + jne L(84) ALIGN_4 -.L85: +L(85): movq C, CO1 movq A, AO movq M, I sarq $2, I - jle .L100 + jle L(100) ALIGN_4 -.L90: +L(90): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1876,10 +1876,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L95 + jle L(95) ALIGN_4 -.L91: +L(91): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm0, %xmm8 @@ -1916,10 +1916,10 @@ subq $-16 * SIZE, AO subq $ -8 * SIZE, BO subq $1, %rax - jg .L91 + jg L(91) ALIGN_4 -.L95: +L(95): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1928,10 +1928,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): addpd %xmm0, %xmm8 movapd -16 * SIZE(AO), %xmm0 mulpd %xmm4, %xmm0 @@ -1943,10 +1943,10 @@ addq $4 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): addpd %xmm0, %xmm8 addpd %xmm1, %xmm12 addpd %xmm2, %xmm9 @@ -1993,12 +1993,12 @@ addq $8 * SIZE, CO1 # coffset += 4 subq $1, I - jg .L90 + jg L(90) ALIGN_4 -.L100: +L(100): testq $2, M - jle .L110 + jle L(110) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2043,10 +2043,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L105 + jle L(105) ALIGN_4 -.L101: +L(101): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm0, %xmm8 @@ -2069,10 +2069,10 @@ subq $-8 * SIZE, AO subq $-8 * SIZE, BO subq $1, %rax - jg .L101 + jg L(101) ALIGN_4 -.L105: +L(105): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2081,10 +2081,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L108 + je L(108) ALIGN_4 -.L106: +L(106): addpd %xmm0, %xmm8 movapd -16 * SIZE(AO), %xmm0 mulpd %xmm4, %xmm0 @@ -2093,10 +2093,10 @@ addq $2 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L106 + jg L(106) ALIGN_4 -.L108: +L(108): addpd %xmm0, %xmm8 addpd %xmm1, %xmm9 addpd %xmm2, %xmm10 @@ -2129,9 +2129,9 @@ addq $4 * SIZE, CO1 ALIGN_4 -.L110: +L(110): testq $1, M - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2176,10 +2176,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L115 + jle L(115) ALIGN_4 -.L111: +L(111): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm0, %xmm8 @@ -2202,10 +2202,10 @@ subq $-4 * SIZE, AO subq $-8 * SIZE, BO subq $1, %rax - jg .L111 + jg L(111) ALIGN_4 -.L115: +L(115): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2214,10 +2214,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): addsd %xmm0, %xmm8 movsd -16 * SIZE(AO), %xmm0 mulsd %xmm4, %xmm0 @@ -2226,10 +2226,10 @@ addq $1 * SIZE, AO addq $2 * SIZE, BO subq $1, %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addsd %xmm0, %xmm8 addsd %xmm1, %xmm9 addsd %xmm2, %xmm10 @@ -2251,7 +2251,7 @@ movhpd %xmm0, 1 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/zgemm3m_kernel_4x4_penryn.S b/kernel/x86_64/zgemm3m_kernel_4x4_penryn.S index b000dc5677..39447d91fb 100644 --- a/kernel/x86_64/zgemm3m_kernel_4x4_penryn.S +++ b/kernel/x86_64/zgemm3m_kernel_4x4_penryn.S @@ -181,10 +181,10 @@ movq N, J sarq $2, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -201,10 +201,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -263,10 +263,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm3, %xmm11 movaps -15 * SIZE(BO), %xmm3 @@ -394,10 +394,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): PREFETCHB -8 * SIZE(BB) #ifndef TRMMKERNEL @@ -407,10 +407,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addpd %xmm3, %xmm11 movaps -15 * SIZE(BO), %xmm3 addpd %xmm4, %xmm15 @@ -446,10 +446,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): movups ALPHA_R, %xmm7 addpd %xmm3, %xmm11 @@ -622,13 +622,13 @@ decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -676,10 +676,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_4 -.L22: +L(22): addpd %xmm3, %xmm11 movaps -15 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -745,10 +745,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -756,10 +756,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): addpd %xmm3, %xmm11 movaps -15 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -780,10 +780,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): movups ALPHA_R, %xmm7 addpd %xmm3, %xmm11 @@ -873,10 +873,10 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $1, M BRANCH - jle .L39 + jle L(39) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -919,10 +919,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) shufps $0x44, %xmm0, %xmm0 @@ -969,10 +969,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -980,10 +980,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): shufps $0x44, %xmm0, %xmm0 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm3 @@ -999,10 +999,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): movups ALPHA_R, %xmm7 addpd %xmm10, %xmm8 @@ -1045,7 +1045,7 @@ movhps %xmm1, 1 * SIZE(CO2, LDC, 2) ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif @@ -1056,13 +1056,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $2, N BRANCH - jle .L80 + jle L(80) movq C, CO1 leaq (C, LDC, 1), CO2 @@ -1080,10 +1080,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1129,10 +1129,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_4 -.L52: +L(52): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 @@ -1208,10 +1208,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_4 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1219,10 +1219,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -1245,10 +1245,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): movups ALPHA_R, %xmm7 movaps %xmm8, %xmm0 @@ -1335,13 +1335,13 @@ addq $8 * SIZE, CO2 decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M BRANCH - jle .L70 + jle L(70) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1382,10 +1382,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x4e, %xmm2, %xmm7 @@ -1428,10 +1428,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1439,10 +1439,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm7 @@ -1457,10 +1457,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): movups ALPHA_R, %xmm7 addpd %xmm10, %xmm8 @@ -1510,10 +1510,10 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L70: +L(70): testq $1, M BRANCH - jle .L79 + jle L(79) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1553,10 +1553,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_4 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) shufps $0x44, %xmm0, %xmm0 @@ -1587,10 +1587,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -1598,10 +1598,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): shufps $0x44, %xmm0, %xmm0 mulpd %xmm0, %xmm2 movsd -15 * SIZE(AO), %xmm0 @@ -1613,10 +1613,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): movups ALPHA_R, %xmm7 addpd %xmm9, %xmm8 @@ -1640,7 +1640,7 @@ movhps %xmm1, 1 * SIZE(CO2) ALIGN_4 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -1649,10 +1649,10 @@ movq BO, B ALIGN_4 -.L80: +L(80): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1665,10 +1665,10 @@ movq M, I sarq $2, I # i = (m >> 2) NOBRANCH - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1710,10 +1710,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_4 -.L92: +L(92): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm2, %xmm3 @@ -1770,10 +1770,10 @@ subq $ -4 * SIZE, BO subq $1, %rax BRANCH - jg .L92 + jg L(92) ALIGN_4 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -1781,10 +1781,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): pshufd $0x44, %xmm2, %xmm3 pshufd $0x44, %xmm2, %xmm4 movsd -16 * SIZE(BO), %xmm2 @@ -1802,10 +1802,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): movups ALPHA_R, %xmm7 movsd 0 * SIZE(CO1), %xmm0 @@ -1847,13 +1847,13 @@ addq $8 * SIZE, CO1 decq I BRANCH - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M BRANCH - jle .L110 + jle L(110) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1892,10 +1892,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_4 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm2, %xmm3 @@ -1930,10 +1930,10 @@ subq $-4 * SIZE, BO subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_4 -.L105: +L(105): #ifndef TRMMKERNEL movq K, %rax #else @@ -1941,10 +1941,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L108 + je L(108) ALIGN_4 -.L106: +L(106): pshufd $0x44, %xmm2, %xmm3 movsd -16 * SIZE(BO), %xmm2 @@ -1957,10 +1957,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_4 -.L108: +L(108): movups ALPHA_R, %xmm7 addpd %xmm9, %xmm8 @@ -1986,10 +1986,10 @@ addq $4 * SIZE, CO1 ALIGN_4 -.L110: +L(110): testq $1, M BRANCH - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2028,10 +2028,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_4 -.L112: +L(112): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulsd %xmm0, %xmm2 @@ -2058,10 +2058,10 @@ subq $-4 * SIZE, BO subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_4 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2069,10 +2069,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd %xmm0, %xmm2 addsd %xmm2, %xmm8 movsd -15 * SIZE(AO), %xmm0 @@ -2083,10 +2083,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): movups ALPHA_R, %xmm7 addpd %xmm9, %xmm8 @@ -2102,7 +2102,7 @@ movhps %xmm0, 1 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm3m_kernel_4x4_sse2.S b/kernel/x86_64/zgemm3m_kernel_4x4_sse2.S index cb90b4c8fc..cd60717fc2 100644 --- a/kernel/x86_64/zgemm3m_kernel_4x4_sse2.S +++ b/kernel/x86_64/zgemm3m_kernel_4x4_sse2.S @@ -407,10 +407,10 @@ #endif movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_3 -.L01: +L(01): /* Copying to Sub Buffer */ leaq 16 * SIZE + BUFFER, BO movq C, CO1 # coffset1 = c @@ -423,14 +423,14 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_3 #define RPREFETCHSIZE (8 * 7 + 4) #define WPREFETCHSIZE (8 * 8 + 4) -.L02: +L(02): PREFETCH (RPREFETCHSIZE + 0) * SIZE(B) movq 0 * SIZE(B), %mm0 @@ -504,17 +504,17 @@ subq $-16 * SIZE, B subq $1, %rax - jne .L02 + jne L(02) ALIGN_3 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_3 -.L04: +L(04): movq 0 * SIZE(B), %mm0 movq 1 * SIZE(B), %mm1 movq 2 * SIZE(B), %mm2 @@ -532,19 +532,19 @@ addq $4 * SIZE, B addq $8 * SIZE, BO subq $1, %rax - jne .L04 + jne L(04) ALIGN_3 -.L10: +L(10): movq A, AO # aoffset = a leaq (RPREFETCHSIZE + 0) * SIZE(B), BB movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_3 -.L11: +L(11): PREFETCH 0 * SIZE(BB) PREFETCH 8 * SIZE(BB) subq $-16 * SIZE, BB @@ -607,10 +607,10 @@ leaq (BO, %rax, 8), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -631,7 +631,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -653,7 +653,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -675,7 +675,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -697,7 +697,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -719,7 +719,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -741,7 +741,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -763,7 +763,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) @@ -785,17 +785,17 @@ addq $8 * SIZE, %rax BRANCH - jl .L12 + jl L(12) ALIGN_3 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_3 @@ -814,10 +814,10 @@ #else sarq $2, %rax NOBRANCH - jle .L16 + jle L(16) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -831,10 +831,10 @@ subq $-16 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) #endif -.L16: +L(16): movapd ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -843,7 +843,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L19 + je L(19) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -851,7 +851,7 @@ negq %rax ALIGN_3 -.L17: +L(17): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movapd -14 * SIZE(BO, %rax, 8), %xmm1 @@ -878,10 +878,10 @@ movapd -10 * SIZE(AO, %rax, 4), %xmm2 addq $SIZE, %rax - jl .L17 + jl L(17) ALIGN_3 -.L19: +L(19): movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 movsd 2 * SIZE(CO1), %xmm1 @@ -1026,18 +1026,18 @@ addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_3 -.L20: +L(20): testq $3, M - je .L39 + je L(39) testq $2, M - je .L30 + je L(30) ALIGN_3 -.L21: +L(21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1079,10 +1079,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_3 -.L22: +L(22): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1192,10 +1192,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_3 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1204,10 +1204,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_3 -.L26: +L(26): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movapd 2 * SIZE(BO), %xmm1 @@ -1224,10 +1224,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_3 -.L29: +L(29): movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 movsd 2 * SIZE(CO1), %xmm1 @@ -1304,12 +1304,12 @@ addq $4 * SIZE, CO2 ALIGN_3 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_3 -.L31: +L(31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1351,10 +1351,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_3 -.L32: +L(32): mulsd %xmm0, %xmm1 addsd %xmm1, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1463,10 +1463,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_3 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1475,10 +1475,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): mulsd %xmm0, %xmm1 addsd %xmm1, %xmm8 movsd 2 * SIZE(BO), %xmm1 @@ -1495,10 +1495,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 @@ -1540,25 +1540,25 @@ movhpd %xmm0, 1 * SIZE(CO2, LDC, 2) ALIGN_3 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C # c += 4 * ldc decq J # j -- - jg .L01 + jg L(01) ALIGN_3 -.L40: +L(40): testq $3, N - je .L999 + je L(999) testq $2, N - je .L80 + je L(80) ALIGN_4 -.L41: +L(41): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -1569,10 +1569,10 @@ movq K, %rax sarq $2, %rax - jle .L43 + jle L(43) ALIGN_3 -.L42: +L(42): PREFETCH 56 * SIZE(B) movq 0 * SIZE(B), %mm0 @@ -1605,17 +1605,17 @@ movq %mm7, -1 * SIZE(BO) decq %rax - jne .L42 + jne L(42) ALIGN_3 -.L43: +L(43): movq K, %rax andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_3 -.L44: +L(44): movq 0 * SIZE(B), %mm0 movq 1 * SIZE(B), %mm1 @@ -1627,20 +1627,20 @@ addq $2 * SIZE, B addq $4 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_3 -.L50: +L(50): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_3 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1687,10 +1687,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_3 -.L52: +L(52): mulpd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(BO), %xmm0 @@ -1802,10 +1802,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_3 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1814,10 +1814,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_3 -.L56: +L(56): movapd 0 * SIZE(BO), %xmm1 mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 @@ -1834,10 +1834,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L59: +L(59): movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 movsd 2 * SIZE(CO1), %xmm1 @@ -1911,15 +1911,15 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L51 + jg L(51) ALIGN_3 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_3 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1961,10 +1961,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_3 -.L62: +L(62): mulpd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(BO), %xmm0 @@ -2026,10 +2026,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_3 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -2038,10 +2038,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_3 -.L66: +L(66): mulpd %xmm0, %xmm1 mulpd 2 * SIZE(BO), %xmm0 addpd %xmm1, %xmm8 @@ -2052,10 +2052,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_3 -.L69: +L(69): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -2099,12 +2099,12 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_3 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_3 -.L71: +L(71): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2146,10 +2146,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_3 -.L72: +L(72): mulsd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulsd 2 * SIZE(BO), %xmm0 @@ -2210,10 +2210,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_3 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2222,10 +2222,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): mulsd %xmm0, %xmm1 mulsd 2 * SIZE(BO), %xmm0 addsd %xmm1, %xmm8 @@ -2236,10 +2236,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_3 -.L78: +L(78): addsd %xmm10, %xmm8 addsd %xmm11, %xmm9 @@ -2264,19 +2264,19 @@ movhpd %xmm0, 1 * SIZE(CO2) ALIGN_3 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif leaq (C, LDC, 2), C ALIGN_3 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L81: +L(81): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -2287,10 +2287,10 @@ movq K, %rax sarq $3, %rax - jle .L83 + jle L(83) ALIGN_3 -.L82: +L(82): PREFETCH 56 * SIZE(B) movq 0 * SIZE(B), %mm0 @@ -2323,17 +2323,17 @@ movq %mm7, -1 * SIZE(BO) decq %rax - jne .L82 + jne L(82) ALIGN_3 -.L83: +L(83): movq K, %rax andq $7, %rax BRANCH - jle .L90 + jle L(90) ALIGN_3 -.L84: +L(84): movq 0 * SIZE(B), %mm0 movq %mm0, 0 * SIZE(BO) @@ -2342,19 +2342,19 @@ addq $1 * SIZE, B addq $2 * SIZE, BO decq %rax - jne .L84 + jne L(84) ALIGN_3 -.L90: +L(90): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_3 -.L91: +L(91): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2398,10 +2398,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_3 -.L92: +L(92): mulpd %xmm1, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd -14 * SIZE(AO), %xmm1 @@ -2458,10 +2458,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_3 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2470,10 +2470,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_3 -.L96: +L(96): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO), %xmm1 addpd %xmm0, %xmm8 @@ -2484,10 +2484,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_3 -.L99: +L(99): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -2528,15 +2528,15 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L91 + jg L(91) ALIGN_3 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_3 -.L101: +L(101): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2575,10 +2575,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L105 + je L(105) ALIGN_3 -.L102: +L(102): mulpd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -14 * SIZE(AO), %xmm0 @@ -2611,10 +2611,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_3 -.L105: +L(105): #ifndef TRMMKERNEL movq K, %rax #else @@ -2623,10 +2623,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_3 -.L106: +L(106): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movapd -14 * SIZE(AO), %xmm0 @@ -2635,10 +2635,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L106 + jg L(106) ALIGN_3 -.L109: +L(109): addpd %xmm9, %xmm8 addpd %xmm11, %xmm10 addpd %xmm10, %xmm8 @@ -2664,12 +2664,12 @@ addq $4 * SIZE, CO1 ALIGN_3 -.L110: +L(110): testq $1, M - je .L999 + je L(999) ALIGN_3 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2708,10 +2708,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_3 -.L112: +L(112): mulsd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -15 * SIZE(AO), %xmm0 @@ -2743,10 +2743,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_3 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2755,10 +2755,10 @@ movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_3 -.L116: +L(116): mulsd %xmm0, %xmm1 movsd -15 * SIZE(AO), %xmm0 addsd %xmm1, %xmm8 @@ -2767,10 +2767,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_3 -.L118: +L(118): addsd %xmm10, %xmm8 addsd %xmm11, %xmm9 addsd %xmm9, %xmm8 @@ -2787,7 +2787,7 @@ movhpd %xmm0, 1 * SIZE(CO1) ALIGN_3 -.L999: +L(999): movq %rbx, %rsp EMMS diff --git a/kernel/x86_64/zgemm3m_kernel_4x4_sse3.S b/kernel/x86_64/zgemm3m_kernel_4x4_sse3.S index ce46dbdf05..efe0c1294a 100644 --- a/kernel/x86_64/zgemm3m_kernel_4x4_sse3.S +++ b/kernel/x86_64/zgemm3m_kernel_4x4_sse3.S @@ -382,10 +382,10 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -400,10 +400,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): prefetcht0 0 * SIZE(BB) prefetcht0 8 * SIZE(BB) subq $-8 * SIZE, BB @@ -463,9 +463,9 @@ andq $-8, %rax salq $4, %rax NOBRANCH - je .L15 + je L(15) -.L1X: +L(1X): KERNEL1 (16 * 0) KERNEL2 (16 * 0) KERNEL3 (16 * 0) @@ -484,7 +484,7 @@ KERNEL16(16 * 0) cmpq $128 * 1, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 1) KERNEL2 (16 * 1) KERNEL3 (16 * 1) @@ -503,7 +503,7 @@ KERNEL16(16 * 1) cmpq $128 * 2, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 2) KERNEL2 (16 * 2) KERNEL3 (16 * 2) @@ -522,7 +522,7 @@ KERNEL16(16 * 2) cmpq $128 * 3, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 3) KERNEL2 (16 * 3) KERNEL3 (16 * 3) @@ -541,7 +541,7 @@ KERNEL16(16 * 3) cmpq $128 * 4, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 4) KERNEL2 (16 * 4) KERNEL3 (16 * 4) @@ -560,7 +560,7 @@ KERNEL16(16 * 4) cmpq $128 * 5, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 5) KERNEL2 (16 * 5) KERNEL3 (16 * 5) @@ -579,7 +579,7 @@ KERNEL16(16 * 5) cmpq $128 * 6, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 6) KERNEL2 (16 * 6) KERNEL3 (16 * 6) @@ -598,7 +598,7 @@ KERNEL16(16 * 6) cmpq $128 * 7, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (16 * 7) KERNEL2 (16 * 7) KERNEL3 (16 * 7) @@ -620,18 +620,18 @@ addq $32 * 8 * SIZE, BO subq $128 * 8, %rax BRANCH - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 2), BO # * 64 #else sarq $3, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -858,11 +858,11 @@ addq $32 * SIZE, AO decq %rax BRANCH - jne .L12 + jne L(12) #endif ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else @@ -873,10 +873,10 @@ andq $7, %rax # if (k & 1) BRANCH BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 addpd %xmm9, %xmm0 @@ -908,10 +908,10 @@ addq $4 * SIZE, BO # boffset1 += 8 decq %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): movsd 0 * SIZE(CO1), %xmm8 movhpd 1 * SIZE(CO1), %xmm8 movsd 2 * SIZE(CO1), %xmm9 @@ -1056,17 +1056,17 @@ addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 - jmp .L20 + jg L(11) + jmp L(20) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - je .L30 + je L(30) ALIGN_4 -.L21: +L(21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1104,10 +1104,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1217,10 +1217,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1230,10 +1230,10 @@ movhpd ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -1251,10 +1251,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): movsd 0 * SIZE(CO1), %xmm8 movhpd 1 * SIZE(CO1), %xmm8 movsd 2 * SIZE(CO1), %xmm9 @@ -1331,12 +1331,12 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) ALIGN_4 -.L31: +L(31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1374,10 +1374,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1439,10 +1439,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1452,10 +1452,10 @@ movhpd ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -1467,10 +1467,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): movsd 0 * SIZE(CO1), %xmm8 movhpd 1 * SIZE(CO1), %xmm8 @@ -1516,7 +1516,7 @@ movhpd %xmm8, 1 * SIZE(CO2, LDC, 2) ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif @@ -1524,12 +1524,12 @@ leaq (C, LDC, 4), C # c += 4 * ldc movq BO, B decq J # j -- - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $2, N - je .L80 + je L(80) ALIGN_4 #if defined(TRMMKERNEL) && defined(LEFT) @@ -1543,10 +1543,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1592,10 +1592,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1714,10 +1714,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1727,10 +1727,10 @@ movhpd ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L59 + je L(59) ALIGN_4 -.L56: +L(56): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 addpd %xmm9, %xmm0 @@ -1749,10 +1749,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L59: +L(59): movsd 0 * SIZE(CO1), %xmm8 movhpd 1 * SIZE(CO1), %xmm8 movsd 2 * SIZE(CO1), %xmm9 @@ -1827,15 +1827,15 @@ addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) ALIGN_4 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1873,10 +1873,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addpd %xmm9, %xmm0 @@ -1938,10 +1938,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1951,10 +1951,10 @@ movhpd ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L69 + je L(69) ALIGN_4 -.L66: +L(66): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -1966,10 +1966,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L69: +L(69): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -2013,12 +2013,12 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) ALIGN_4 -.L71: +L(71): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2056,10 +2056,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movddup 1 * SIZE(AO), %xmm8 @@ -2091,10 +2091,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2104,10 +2104,10 @@ movhpd ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulpd %xmm8, %xmm9 movddup 1 * SIZE(AO), %xmm8 addpd %xmm9, %xmm0 @@ -2116,10 +2116,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -2146,7 +2146,7 @@ movhpd %xmm8, 1 * SIZE(CO2) ALIGN_4 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -2154,9 +2154,9 @@ movq BO, B ALIGN_4 -.L80: +L(80): testq $1, N - je .L999 + je L(999) ALIGN_4 #if defined(TRMMKERNEL) && defined(LEFT) @@ -2169,10 +2169,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2216,10 +2216,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(AO), %xmm9 @@ -2274,10 +2274,10 @@ addq $32 * SIZE, AO addq $8 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2287,10 +2287,10 @@ movhpd ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L99 + je L(99) ALIGN_4 -.L96: +L(96): mulpd %xmm9, %xmm8 mulpd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -2301,10 +2301,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L99: +L(99): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -2345,15 +2345,15 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $2, M - je .L110 + je L(110) ALIGN_4 -.L101: +L(101): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2391,10 +2391,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L105 + je L(105) ALIGN_4 -.L102: +L(102): mulpd %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movddup 1 * SIZE(BO), %xmm9 @@ -2426,10 +2426,10 @@ addq $16 * SIZE, AO addq $ 8 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L105: +L(105): #ifndef TRMMKERNEL movq K, %rax #else @@ -2439,10 +2439,10 @@ movhpd ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L109 + je L(109) ALIGN_4 -.L106: +L(106): mulpd %xmm9, %xmm8 movddup 1 * SIZE(BO), %xmm9 addpd %xmm8, %xmm0 @@ -2451,10 +2451,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L106 + jg L(106) ALIGN_4 -.L109: +L(109): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -2480,12 +2480,12 @@ addq $4 * SIZE, CO1 ALIGN_4 -.L110: +L(110): testq $1, M - je .L999 + je L(999) ALIGN_4 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2528,10 +2528,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulpd %xmm9, %xmm8 movapd 2 * SIZE(AO), %xmm9 addpd %xmm8, %xmm0 @@ -2550,10 +2550,10 @@ addq $8 * SIZE, AO addq $8 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2563,10 +2563,10 @@ movhpd ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulsd 0 * SIZE(BO), %xmm9 addsd %xmm9, %xmm0 movsd 1 * SIZE(AO), %xmm9 @@ -2574,10 +2574,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $1 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addpd %xmm1, %xmm0 haddpd %xmm0, %xmm0 @@ -2593,7 +2593,7 @@ movhpd %xmm8, 1 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm3m_kernel_4x8_nehalem.S b/kernel/x86_64/zgemm3m_kernel_4x8_nehalem.S index 8da31d22d8..e7ae5a7065 100644 --- a/kernel/x86_64/zgemm3m_kernel_4x8_nehalem.S +++ b/kernel/x86_64/zgemm3m_kernel_4x8_nehalem.S @@ -164,10 +164,10 @@ movq N, J sarq $3, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L10: +L(10): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -184,10 +184,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -248,10 +248,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm12 @@ -374,10 +374,10 @@ subq $-16 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -387,10 +387,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addps %xmm1, %xmm12 movaps -32 * SIZE(BO), %xmm1 addps %xmm2, %xmm13 @@ -426,10 +426,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): addps %xmm1, %xmm12 addps %xmm2, %xmm13 addps %xmm3, %xmm14 @@ -607,13 +607,13 @@ addq $8 * SIZE, CO2 decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -658,10 +658,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -739,10 +739,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -752,10 +752,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -779,10 +779,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): addps %xmm1, %xmm8 addps %xmm2, %xmm9 addps %xmm3, %xmm10 @@ -864,10 +864,10 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L30: +L(30): testq $1, M BRANCH - jle .L39 + jle L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -906,10 +906,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movaps -32 * SIZE(BO), %xmm2 @@ -949,10 +949,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -962,10 +962,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -980,10 +980,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): addps %xmm2, %xmm8 addps %xmm3, %xmm12 @@ -1025,7 +1025,7 @@ movhps %xmm3, (CO2, %rax) ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addq $8, KK #endif @@ -1036,12 +1036,12 @@ subq $1, J BRANCH - jg .L10 + jg L(10) ALIGN_4 -.L40: +L(40): testq $4, N - jle .L70 + jle L(70) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1055,10 +1055,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L50 + jle L(50) ALIGN_4 -.L41: +L(41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1105,10 +1105,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -1175,10 +1175,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -1188,10 +1188,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): addps %xmm1, %xmm8 movaps -32 * SIZE(BO), %xmm1 addps %xmm2, %xmm9 @@ -1212,10 +1212,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): addps %xmm1, %xmm8 addps %xmm2, %xmm9 addps %xmm3, %xmm10 @@ -1308,13 +1308,13 @@ addq $8 * SIZE, CO2 decq I BRANCH - jg .L41 + jg L(41) ALIGN_4 -.L50: +L(50): testq $2, M BRANCH - jle .L60 + jle L(60) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1354,10 +1354,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1399,10 +1399,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -1412,10 +1412,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addps %xmm1, %xmm8 pshufd $0x50, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1430,10 +1430,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): addps %xmm1, %xmm8 addps %xmm2, %xmm9 @@ -1476,10 +1476,10 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L69 + jle L(69) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1517,10 +1517,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movaps -32 * SIZE(BO), %xmm2 @@ -1548,11 +1548,11 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) addps %xmm9, %xmm8 ALIGN_3 -.L65: +L(65): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -1562,10 +1562,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -1577,10 +1577,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): addps %xmm2, %xmm8 movsd (CO1), %xmm0 @@ -1603,7 +1603,7 @@ movhps %xmm1, (CO2, LDC) ALIGN_4 -.L69: +L(69): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif @@ -1613,9 +1613,9 @@ leaq (C, LDC, 4), C ALIGN_4 -.L70: +L(70): testq $2, N - jle .L100 + jle L(100) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1629,10 +1629,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L80 + jle L(80) ALIGN_4 -.L71: +L(71): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1674,10 +1674,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -1720,10 +1720,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_3 -.L75: +L(75): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -1733,10 +1733,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addps %xmm1, %xmm8 pshufd $0x00, %xmm3, %xmm1 mulps %xmm0, %xmm1 @@ -1751,10 +1751,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_3 -.L78: +L(78): addps %xmm1, %xmm8 addps %xmm2, %xmm9 @@ -1797,13 +1797,13 @@ addq $8 * SIZE, CO2 decq I BRANCH - jg .L71 + jg L(71) ALIGN_4 -.L80: +L(80): testq $2, M BRANCH - jle .L90 + jle L(90) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1843,10 +1843,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): addps %xmm1, %xmm8 movsd -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -1876,10 +1876,10 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) ALIGN_3 -.L85: +L(85): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -1889,10 +1889,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): addps %xmm1, %xmm8 movsd -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -1904,10 +1904,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_3 -.L88: +L(88): addps %xmm1, %xmm8 movsd 0 * SIZE(CO1), %xmm0 @@ -1933,10 +1933,10 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L90: +L(90): testq $1, M BRANCH - jle .L99 + jle L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1974,10 +1974,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_3 -.L92: +L(92): pshufd $0x00, %xmm0, %xmm1 addps %xmm2, %xmm8 movsd -32 * SIZE(BO), %xmm2 @@ -2005,11 +2005,11 @@ subq $1, %rax BRANCH - jg .L92 + jg L(92) addps %xmm9, %xmm8 ALIGN_3 -.L95: +L(95): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -2019,10 +2019,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_3 -.L96: +L(96): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 addps %xmm2, %xmm8 @@ -2034,10 +2034,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_3 -.L98: +L(98): addps %xmm2, %xmm8 movsd (CO1), %xmm0 @@ -2052,7 +2052,7 @@ movhps %xmm0, (CO2) ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -2062,9 +2062,9 @@ leaq (C, LDC, 2), C ALIGN_4 -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -2077,10 +2077,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L110 + jle L(110) ALIGN_4 -.L101: +L(101): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2118,10 +2118,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_3 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -2152,10 +2152,10 @@ subq $ -4 * SIZE, BO subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_3 -.L105: +L(105): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -2165,10 +2165,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L108 + je L(108) ALIGN_3 -.L106: +L(106): addps %xmm1, %xmm8 pshufd $0x00, %xmm3, %xmm1 movss -31 * SIZE(BO), %xmm3 @@ -2180,10 +2180,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_3 -.L108: +L(108): addps %xmm1, %xmm8 movsd 0 * SIZE(CO1), %xmm0 @@ -2207,13 +2207,13 @@ addq $8 * SIZE, CO1 decq I BRANCH - jg .L101 + jg L(101) ALIGN_4 -.L110: +L(110): testq $2, M BRANCH - jle .L120 + jle L(120) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2250,10 +2250,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_3 -.L112: +L(112): addps %xmm1, %xmm8 movss -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -2283,10 +2283,10 @@ subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_3 -.L115: +L(115): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -2296,10 +2296,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_3 -.L116: +L(116): addps %xmm1, %xmm8 movss -32 * SIZE(BO), %xmm1 unpcklps %xmm1, %xmm1 @@ -2311,10 +2311,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_3 -.L118: +L(118): addps %xmm1, %xmm8 movsd 0 * SIZE(CO1), %xmm0 @@ -2331,10 +2331,10 @@ addq $4 * SIZE, CO1 ALIGN_4 -.L120: +L(120): testq $1, M BRANCH - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2371,10 +2371,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L125 + jle L(125) ALIGN_3 -.L122: +L(122): addss %xmm2, %xmm8 movss -32 * SIZE(BO), %xmm2 mulss %xmm0, %xmm2 @@ -2400,10 +2400,10 @@ subq $1, %rax BRANCH - jg .L122 + jg L(122) ALIGN_3 -.L125: +L(125): movups ALPHA_R, %xmm7 #ifndef TRMMKERNEL @@ -2413,10 +2413,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_3 -.L126: +L(126): addss %xmm2, %xmm8 movss -32 * SIZE(BO), %xmm2 mulss %xmm0, %xmm2 @@ -2427,10 +2427,10 @@ subq $1, %rax BRANCH - jg .L126 + jg L(126) ALIGN_3 -.L128: +L(128): addps %xmm2, %xmm8 movsd (CO1), %xmm0 @@ -2443,7 +2443,7 @@ movlps %xmm0, (CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm3m_kernel_8x4_barcelona.S b/kernel/x86_64/zgemm3m_kernel_8x4_barcelona.S index 3dbc0dd8c8..b185c0ad92 100644 --- a/kernel/x86_64/zgemm3m_kernel_8x4_barcelona.S +++ b/kernel/x86_64/zgemm3m_kernel_8x4_barcelona.S @@ -479,9 +479,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L50 + jle L(50) -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -492,10 +492,10 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): prefetch (RPREFETCHSIZE + 0) * SIZE(B) movaps 0 * SIZE(B), %xmm3 @@ -553,17 +553,17 @@ addq $64 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movaps 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -579,10 +579,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): movq C, CO1 leaq (C, LDC, 1), CO2 movq A, AO @@ -591,10 +591,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -655,10 +655,10 @@ leaq (BO, %rax, 8), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_3 -.L12: +L(12): KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -668,7 +668,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -678,7 +678,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -688,7 +688,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -698,7 +698,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -708,7 +708,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -718,7 +718,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -728,7 +728,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -738,10 +738,10 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) BRANCH - jl .L12 + jl L(12) ALIGN_4 -.L15: +L(15): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -750,7 +750,7 @@ movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_3 @@ -763,14 +763,14 @@ addq $64 * SIZE, BO ALIGN_3 -.L16: +L(16): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L18 + je L(18) leaq (, %rax, 8), %rax leaq (AO, %rax, 4), AO @@ -778,7 +778,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulps %xmm1, %xmm0 mulps -28 * SIZE(AO, %rax, 4), %xmm1 addps %xmm0, %xmm8 @@ -805,10 +805,10 @@ movaps %xmm0, %xmm2 addq $SIZE * 2, %rax - jl .L17 + jl L(17) ALIGN_4 -.L18: +L(18): movups 0 * SIZE(CO1), %xmm0 movups 4 * SIZE(CO1), %xmm1 movups 8 * SIZE(CO1), %xmm2 @@ -928,12 +928,12 @@ addq $16 * SIZE, CO1 # coffset += 4 addq $16 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -977,10 +977,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1090,10 +1090,10 @@ addq $ 32 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1102,10 +1102,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1122,10 +1122,10 @@ addq $ 4 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): movups 0 * SIZE(CO1), %xmm8 movups 4 * SIZE(CO1), %xmm9 @@ -1198,9 +1198,9 @@ addq $8 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $2, M - je .L40 + je L(40) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1244,10 +1244,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1364,10 +1364,10 @@ addq $ 16 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1376,10 +1376,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movsd 4 * SIZE(BO), %xmm9 @@ -1397,10 +1397,10 @@ addq $ 2 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): movups 0 * SIZE(CO1), %xmm8 pshufd $0x50, %xmm0, %xmm4 @@ -1441,9 +1441,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L40: +L(40): testq $1, M - je .L49 + je L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1488,10 +1488,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): mulss %xmm8, %xmm9 addss %xmm9, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1608,10 +1608,10 @@ addq $ 8 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1620,10 +1620,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -1641,10 +1641,10 @@ addq $ 1 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): movsd 0 * SIZE(CO1), %xmm8 pshufd $0x50, %xmm0, %xmm4 @@ -1678,19 +1678,19 @@ movlps %xmm4, 0 * SIZE(CO2, LDC, 2) ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C # c += 4 * ldc decq J # j -- - jg .L01 + jg L(01) -.L50: +L(50): testq $2, N - je .L100 + je L(100) -.L51: +L(51): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1701,10 +1701,10 @@ movq K, %rax sarq $2, %rax - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): prefetch (RPREFETCHSIZE + 0) * SIZE(B) movaps 0 * SIZE(B), %xmm3 @@ -1737,17 +1737,17 @@ addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): movq K, %rax andq $3, %rax BRANCH - jle .L60 + jle L(60) ALIGN_4 -.L54: +L(54): movsd 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -1764,20 +1764,20 @@ addq $ 2 * SIZE, B addq $ 8 * SIZE, BO decq %rax - jne .L54 + jne L(54) ALIGN_4 -.L60: +L(60): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L70 + jle L(70) ALIGN_4 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1825,10 +1825,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulps 4 * SIZE(BO), %xmm8 @@ -1941,10 +1941,10 @@ addq $64 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1953,10 +1953,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -1973,10 +1973,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): movups 0 * SIZE(CO1), %xmm8 movups 4 * SIZE(CO1), %xmm9 movups 8 * SIZE(CO1), %xmm10 @@ -2038,12 +2038,12 @@ addq $16 * SIZE, CO1 # coffset += 4 addq $16 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L61 + jg L(61) ALIGN_4 -.L70: +L(70): testq $4, M - je .L80 + je L(80) #if !defined(TRMMKERNEL) || \ @@ -2088,10 +2088,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulps 4 * SIZE(BO), %xmm8 @@ -2152,10 +2152,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2164,10 +2164,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -2178,10 +2178,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -2221,9 +2221,9 @@ addq $8 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L80: +L(80): testq $2, M - je .L90 + je L(90) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2267,10 +2267,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L85 + je L(85) ALIGN_4 -.L82: +L(82): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2339,10 +2339,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L85: +L(85): #ifndef TRMMKERNEL movq K, %rax #else @@ -2351,10 +2351,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movsd 4 * SIZE(BO), %xmm9 @@ -2366,10 +2366,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -2397,9 +2397,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L90: +L(90): testq $1, M - je .L99 + je L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2443,10 +2443,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2515,10 +2515,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2527,10 +2527,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -2542,10 +2542,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): addss %xmm2, %xmm0 addss %xmm3, %xmm1 @@ -2568,7 +2568,7 @@ movlps %xmm2, 0 * SIZE(CO2) ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -2576,11 +2576,11 @@ ALIGN_4 -.L100: +L(100): testq $1, N - je .L999 + je L(999) -.L101: +L(101): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -2591,11 +2591,11 @@ movq K, %rax sarq $3, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): prefetch (RPREFETCHSIZE + 0) * SIZE(B) movups 0 * SIZE(B), %xmm3 @@ -2628,17 +2628,17 @@ addq $32 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): movq K, %rax andq $7, %rax BRANCH - jle .L110 + jle L(110) ALIGN_4 -.L104: +L(104): movss 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -2648,19 +2648,19 @@ addq $ 1 * SIZE, B addq $ 4 * SIZE, BO decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L110: +L(110): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L120 + jle L(120) ALIGN_4 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2707,10 +2707,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulps %xmm9, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulps -28 * SIZE(AO), %xmm9 @@ -2774,10 +2774,10 @@ addq $64 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2786,10 +2786,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulps %xmm9, %xmm8 mulps -28 * SIZE(AO), %xmm9 addps %xmm8, %xmm0 @@ -2800,10 +2800,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): movups 0 * SIZE(CO1), %xmm8 movups 4 * SIZE(CO1), %xmm9 movups 8 * SIZE(CO1), %xmm10 @@ -2835,12 +2835,12 @@ addq $16 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L111 + jg L(111) ALIGN_4 -.L120: +L(120): testq $4, M - je .L130 + je L(130) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2882,10 +2882,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L125 + je L(125) ALIGN_4 -.L122: +L(122): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -28 * SIZE(AO), %xmm8 @@ -2919,10 +2919,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L122 + jne L(122) ALIGN_4 -.L125: +L(125): #ifndef TRMMKERNEL movq K, %rax #else @@ -2931,10 +2931,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): mulps %xmm8, %xmm9 movaps -28 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -2943,10 +2943,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): addps %xmm1, %xmm0 addps %xmm3, %xmm2 addps %xmm2, %xmm0 @@ -2970,9 +2970,9 @@ addq $8 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L130: +L(130): testq $2, M - je .L140 + je L(140) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3014,10 +3014,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L135 + je L(135) ALIGN_4 -.L132: +L(132): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -30 * SIZE(AO), %xmm8 @@ -3061,10 +3061,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L132 + jne L(132) ALIGN_4 -.L135: +L(135): #ifndef TRMMKERNEL movq K, %rax #else @@ -3073,10 +3073,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): mulps %xmm8, %xmm9 movsd -30 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -3085,10 +3085,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): addps %xmm1, %xmm0 movups 0 * SIZE(CO1), %xmm8 @@ -3103,9 +3103,9 @@ addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L140: +L(140): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3147,10 +3147,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L145 + je L(145) ALIGN_4 -.L142: +L(142): mulss %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movss -31 * SIZE(AO), %xmm8 @@ -3182,10 +3182,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L142 + jne L(142) ALIGN_4 -.L145: +L(145): #ifndef TRMMKERNEL movq K, %rax #else @@ -3194,10 +3194,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): mulss %xmm8, %xmm9 movss -31 * SIZE(AO), %xmm8 addss %xmm9, %xmm0 @@ -3206,10 +3206,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): addss %xmm1, %xmm0 addss %xmm3, %xmm2 addss %xmm2, %xmm0 @@ -3223,7 +3223,7 @@ movlps %xmm2, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/zgemm3m_kernel_8x4_core2.S b/kernel/x86_64/zgemm3m_kernel_8x4_core2.S index 0b97d85a9a..65618b4c96 100644 --- a/kernel/x86_64/zgemm3m_kernel_8x4_core2.S +++ b/kernel/x86_64/zgemm3m_kernel_8x4_core2.S @@ -166,10 +166,10 @@ movq N, J sarq $2, J - jle .L50 + jle L(50) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -180,10 +180,10 @@ movq K, %rax sarq $2, %rax - jle .L05 + jle L(05) ALIGN_4 -.L02: +L(02): prefetcht0 (PREFETCH_R + 0) * SIZE(B) movaps -32 * SIZE(B), %xmm3 movaps -28 * SIZE(B), %xmm7 @@ -233,17 +233,17 @@ subq $-16 * SIZE, B subq $-64 * SIZE, BO subq $1, %rax - jne .L02 + jne L(02) ALIGN_4 -.L05: +L(05): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L06: +L(06): movaps -32 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -259,10 +259,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO subq $1, %rax - jne .L06 + jne L(06) ALIGN_4 -.L10: +L(10): movq B, BB movq C, CO1 # coffset1 = c @@ -271,10 +271,10 @@ movq M, I sarq $3, I - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -331,10 +331,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): addps %xmm2, %xmm10 movaps -32 * SIZE(BO), %xmm2 addps %xmm3, %xmm14 @@ -463,10 +463,10 @@ subq $-64 * SIZE, BO subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else @@ -474,10 +474,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): addps %xmm2, %xmm10 movaps -32 * SIZE(BO), %xmm2 addps %xmm3, %xmm14 @@ -512,10 +512,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): movaps ALPHA, %xmm7 addps %xmm2, %xmm10 @@ -658,12 +658,12 @@ addq $16 * SIZE, CO1 addq $16 * SIZE, CO2 subq $1, I - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M - jle .L30 + jle L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -699,10 +699,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L25 + jle L(25) ALIGN_4 -.L21: +L(21): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -772,10 +772,10 @@ subq $-16 * SIZE, AO subq $-64 * SIZE, BO subq $1, %rax - jg .L21 + jg L(21) ALIGN_4 -.L25: +L(25): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -784,10 +784,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): movaps -32 * SIZE(AO), %xmm0 movaps -32 * SIZE(BO), %xmm2 movaps -28 * SIZE(BO), %xmm3 @@ -807,10 +807,10 @@ addq $ 4 * SIZE, AO addq $16 * SIZE, BO subq $1, %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 movsd 4 * SIZE(CO1), %xmm1 @@ -887,9 +887,9 @@ addq $8 * SIZE, CO2 ALIGN_4 -.L30: +L(30): testq $2, M - jle .L40 + jle L(40) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -930,10 +930,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L35 + jle L(35) ALIGN_4 -.L31: +L(31): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -32 * SIZE(AO), %xmm0 @@ -1003,10 +1003,10 @@ subq $ -8 * SIZE, AO subq $-64 * SIZE, BO subq $1, %rax - jg .L31 + jg L(31) ALIGN_4 -.L35: +L(35): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1015,10 +1015,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): movsd -32 * SIZE(AO), %xmm0 movsd -32 * SIZE(BO), %xmm2 movsd -28 * SIZE(BO), %xmm3 @@ -1038,10 +1038,10 @@ addq $ 2 * SIZE, AO addq $16 * SIZE, BO subq $1, %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -1086,9 +1086,9 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L40: +L(40): testq $1, M - jle .L49 + jle L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1130,10 +1130,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L45 + jle L(45) ALIGN_4 -.L41: +L(41): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movss -32 * SIZE(AO), %xmm0 @@ -1203,10 +1203,10 @@ subq $ -4 * SIZE, AO subq $-64 * SIZE, BO subq $1, %rax - jg .L41 + jg L(41) ALIGN_4 -.L45: +L(45): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1215,10 +1215,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): movss -32 * SIZE(AO), %xmm0 movss -32 * SIZE(BO), %xmm2 movss -28 * SIZE(BO), %xmm3 @@ -1238,10 +1238,10 @@ addq $ 1 * SIZE, AO addq $16 * SIZE, BO subq $1, %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): movsd 0 * SIZE(CO1), %xmm0 pshufd $0x50, %xmm8, %xmm4 @@ -1275,22 +1275,22 @@ movlps %xmm4, 0 * SIZE(CO2, LDC, 2) ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C subq $1, J - jg .L01 + jg L(01) ALIGN_4 -.L50: +L(50): testq $2, N - jle .L100 + jle L(100) ALIGN_4 -.L51: +L(51): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -1301,12 +1301,12 @@ movq K, %rax sarq $3, %rax - jle .L53 + jle L(53) addq %rax, %rax ALIGN_4 -.L52: +L(52): movaps -32 * SIZE(B), %xmm3 movaps -28 * SIZE(B), %xmm7 @@ -1336,17 +1336,17 @@ addq $32 * SIZE, BO subq $1, %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): movq K, %rax andq $7, %rax BRANCH - jle .L55 + jle L(55) ALIGN_4 -.L54: +L(54): movss -32 * SIZE(B), %xmm8 movss -31 * SIZE(B), %xmm9 @@ -1359,20 +1359,20 @@ addq $2 * SIZE, B addq $8 * SIZE, BO subq $1, %rax - jne .L54 + jne L(54) ALIGN_4 -.L55: +L(55): movq C, CO1 leaq (C, LDC, 1), CO2 movq A, AO # aoffset = a movq M, I sarq $3, I - jle .L70 + jle L(70) ALIGN_4 -.L60: +L(60): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1410,10 +1410,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L65 + jle L(65) ALIGN_4 -.L61: +L(61): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -1491,10 +1491,10 @@ subq $-32 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jg .L61 + jg L(61) ALIGN_4 -.L65: +L(65): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1503,10 +1503,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): movaps -32 * SIZE(AO), %xmm0 movaps -28 * SIZE(AO), %xmm1 @@ -1528,10 +1528,10 @@ addq $8 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 movsd 4 * SIZE(CO1), %xmm1 @@ -1601,12 +1601,12 @@ addq $16 * SIZE, CO1 addq $16 * SIZE, CO2 subq $1, I - jg .L60 + jg L(60) ALIGN_4 -.L70: +L(70): testq $4, M - jle .L80 + jle L(80) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1642,10 +1642,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L75 + jle L(75) ALIGN_4 -.L71: +L(71): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -1685,10 +1685,10 @@ subq $-16 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jg .L71 + jg L(71) ALIGN_4 -.L75: +L(75): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1697,10 +1697,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): movaps -32 * SIZE(AO), %xmm0 movaps -32 * SIZE(BO), %xmm2 movaps -28 * SIZE(BO), %xmm3 @@ -1714,10 +1714,10 @@ addq $4 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm10, %xmm8 addps %xmm11, %xmm9 @@ -1761,9 +1761,9 @@ addq $8 * SIZE, CO2 ALIGN_4 -.L80: +L(80): testq $2, M - jle .L90 + jle L(90) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1799,10 +1799,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L85 + jle L(85) ALIGN_4 -.L81: +L(81): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -32 * SIZE(AO), %xmm0 @@ -1842,10 +1842,10 @@ subq $ -8 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jg .L81 + jg L(81) ALIGN_4 -.L85: +L(85): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1854,10 +1854,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): movsd -32 * SIZE(AO), %xmm0 movsd -32 * SIZE(BO), %xmm2 movsd -28 * SIZE(BO), %xmm3 @@ -1871,10 +1871,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): addps %xmm10, %xmm8 addps %xmm11, %xmm9 @@ -1904,9 +1904,9 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L90: +L(90): testq $1, M - jle .L99 + jle L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1942,10 +1942,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L95 + jle L(95) ALIGN_4 -.L91: +L(91): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1986,10 +1986,10 @@ subq $ -4 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jg .L91 + jg L(91) ALIGN_4 -.L95: +L(95): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -1998,10 +1998,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): movss -32 * SIZE(AO), %xmm0 movss -32 * SIZE(BO), %xmm2 movss -28 * SIZE(BO), %xmm3 @@ -2015,10 +2015,10 @@ addq $1 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): addss %xmm10, %xmm8 addss %xmm11, %xmm9 @@ -2041,7 +2041,7 @@ movlps %xmm4, 0 * SIZE(CO2) ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -2051,12 +2051,12 @@ -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) ALIGN_4 -.L101: +L(101): /* Copying to Sub Buffer */ leaq BUFFER, BO @@ -2067,12 +2067,12 @@ movq K, %rax sarq $4, %rax - jle .L103 + jle L(103) addq %rax, %rax ALIGN_4 -.L102: +L(102): movss -32 * SIZE(B), %xmm0 movss -31 * SIZE(B), %xmm1 movss -30 * SIZE(B), %xmm2 @@ -2103,17 +2103,17 @@ addq $ 8 * SIZE, B subq $-32 * SIZE, BO subq $1, %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): movq K, %rax andq $15, %rax BRANCH - jle .L105 + jle L(105) ALIGN_4 -.L104: +L(104): movss -32 * SIZE(B), %xmm8 shufps $0, %xmm8, %xmm8 @@ -2123,19 +2123,19 @@ addq $1 * SIZE, B addq $4 * SIZE, BO subq $1, %rax - jne .L104 + jne L(104) ALIGN_4 -.L105: +L(105): movq C, CO1 movq A, AO movq M, I sarq $3, I - jle .L120 + jle L(120) ALIGN_4 -.L110: +L(110): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2172,10 +2172,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L115 + jle L(115) ALIGN_4 -.L111: +L(111): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -2221,10 +2221,10 @@ subq $-32 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L111 + jg L(111) ALIGN_4 -.L115: +L(115): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2233,10 +2233,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): movaps -32 * SIZE(AO), %xmm0 movaps -28 * SIZE(AO), %xmm1 @@ -2251,10 +2251,10 @@ addq $8 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addps %xmm9, %xmm8 addps %xmm13, %xmm12 @@ -2293,12 +2293,12 @@ addq $16 * SIZE, CO1 subq $1, I - jg .L110 + jg L(110) ALIGN_4 -.L120: +L(120): testq $4, M - jle .L130 + jle L(130) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2334,10 +2334,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L125 + jle L(125) ALIGN_4 -.L121: +L(121): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -2365,10 +2365,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L121 + jg L(121) ALIGN_4 -.L125: +L(125): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2377,10 +2377,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): movaps -32 * SIZE(AO), %xmm0 movaps -32 * SIZE(BO), %xmm2 @@ -2390,10 +2390,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): addps %xmm10, %xmm8 addps %xmm11, %xmm9 @@ -2420,9 +2420,9 @@ addq $8 * SIZE, CO1 ALIGN_4 -.L130: +L(130): testq $2, M - jle .L140 + jle L(140) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2458,10 +2458,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L135 + jle L(135) ALIGN_4 -.L131: +L(131): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -32 * SIZE(AO), %xmm0 @@ -2487,10 +2487,10 @@ subq $ -8 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L131 + jg L(131) ALIGN_4 -.L135: +L(135): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2499,10 +2499,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): movsd -32 * SIZE(AO), %xmm0 movsd -32 * SIZE(BO), %xmm2 @@ -2512,10 +2512,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): addps %xmm10, %xmm8 addps %xmm11, %xmm9 addps %xmm9, %xmm8 @@ -2534,9 +2534,9 @@ addq $4 * SIZE, CO1 ALIGN_4 -.L140: +L(140): testq $1, M - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2572,10 +2572,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L145 + jle L(145) ALIGN_4 -.L141: +L(141): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movss -32 * SIZE(AO), %xmm0 @@ -2601,10 +2601,10 @@ subq $ -4 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jg .L141 + jg L(141) ALIGN_4 -.L145: +L(145): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -2613,10 +2613,10 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): movss -32 * SIZE(AO), %xmm0 movss -32 * SIZE(BO), %xmm2 @@ -2626,10 +2626,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): addss %xmm10, %xmm8 addss %xmm11, %xmm9 addss %xmm9, %xmm8 @@ -2644,7 +2644,7 @@ movlps %xmm4, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/zgemm3m_kernel_8x4_penryn.S b/kernel/x86_64/zgemm3m_kernel_8x4_penryn.S index 1255c2cd5c..13e07bbfdc 100644 --- a/kernel/x86_64/zgemm3m_kernel_8x4_penryn.S +++ b/kernel/x86_64/zgemm3m_kernel_8x4_penryn.S @@ -163,10 +163,10 @@ movq N, J sarq $2, J NOBRANCH - jle .L50 + jle L(50) ALIGN_4 -.L10: +L(10): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -183,10 +183,10 @@ movq M, I sarq $3, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -246,10 +246,10 @@ #endif sarq $3, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH -32 * SIZE(PREA) addps %xmm6, %xmm10 addps %xmm3, %xmm14 @@ -500,10 +500,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): prefetcht0 -16 * SIZE(BB) #ifndef TRMMKERNEL @@ -513,10 +513,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): addps %xmm6, %xmm10 addps %xmm3, %xmm14 movaps %xmm2, %xmm3 @@ -552,10 +552,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): movups ALPHA_R, %xmm7 addps %xmm6, %xmm10 @@ -725,13 +725,13 @@ addq $16 * SIZE, CO2 decq I BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M BRANCH - jle .L30 + jle L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -777,10 +777,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): addps %xmm6, %xmm10 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x39, %xmm2, %xmm7 @@ -847,10 +847,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -858,10 +858,10 @@ #endif andq $3, %rax BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): addps %xmm6, %xmm10 pshufd $0x39, %xmm2, %xmm7 mulps %xmm0, %xmm2 @@ -882,10 +882,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): movups ALPHA_R, %xmm7 addps %xmm6, %xmm10 @@ -980,10 +980,10 @@ addq $8 * SIZE, CO2 ALIGN_4 -.L30: +L(30): testq $2, M BRANCH - jle .L40 + jle L(40) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1028,10 +1028,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x44, %xmm0, %xmm1 @@ -1077,10 +1077,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1088,10 +1088,10 @@ #endif andq $3, %rax BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): pshufd $0x44, %xmm0, %xmm1 movsd -30 * SIZE(AO), %xmm0 addps %xmm3, %xmm8 @@ -1107,10 +1107,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): movups ALPHA_R, %xmm7 addps %xmm10, %xmm8 @@ -1159,10 +1159,10 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L40: +L(40): testq $1, M BRANCH - jle .L49 + jle L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1200,10 +1200,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x00, %xmm0, %xmm1 @@ -1235,10 +1235,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1246,10 +1246,10 @@ #endif andq $3, %rax BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 mulps %xmm1, %xmm2 @@ -1261,10 +1261,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): movups ALPHA_R, %xmm7 addps %xmm9, %xmm8 @@ -1288,7 +1288,7 @@ movhps %xmm8, 0 * SIZE(CO2, LDC, 2) ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif @@ -1299,12 +1299,12 @@ subq $1, J BRANCH - jg .L10 + jg L(10) ALIGN_4 -.L50: +L(50): testq $2, N - jle .L90 + jle L(90) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1322,10 +1322,10 @@ movq M, I sarq $3, I NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1377,10 +1377,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm3, %xmm8 @@ -1456,10 +1456,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1467,10 +1467,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addps %xmm3, %xmm8 pshufd $0x00, %xmm2, %xmm3 mulps %xmm0, %xmm3 @@ -1493,10 +1493,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): movups ALPHA_R, %xmm7 addps %xmm3, %xmm8 @@ -1574,13 +1574,13 @@ addq $16 * SIZE, CO2 decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $4, M BRANCH - jle .L70 + jle L(70) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1623,10 +1623,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm3, %xmm8 @@ -1668,10 +1668,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1679,10 +1679,10 @@ #endif andq $3, %rax BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): addps %xmm3, %xmm8 pshufd $0x00, %xmm2, %xmm3 mulps %xmm0, %xmm3 @@ -1697,10 +1697,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): movups ALPHA_R, %xmm7 addps %xmm10, %xmm8 @@ -1750,10 +1750,10 @@ addq $8 * SIZE, CO2 ALIGN_4 -.L70: +L(70): testq $2, M BRANCH - jle .L80 + jle L(80) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1792,10 +1792,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L75 + jle L(75) ALIGN_3 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm3, %xmm8 @@ -1827,10 +1827,10 @@ subq $1, %rax BRANCH - jg .L72 + jg L(72) ALIGN_3 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -1838,10 +1838,10 @@ #endif andq $3, %rax BRANCH - je .L78 + je L(78) ALIGN_3 -.L76: +L(76): addps %xmm3, %xmm8 pshufd $0x44, %xmm0, %xmm1 movsd -30 * SIZE(AO), %xmm0 @@ -1854,10 +1854,10 @@ subq $1, %rax BRANCH - jg .L76 + jg L(76) ALIGN_3 -.L78: +L(78): movups ALPHA_R, %xmm7 addps %xmm9, %xmm8 @@ -1885,10 +1885,10 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L80: +L(80): testq $1, M BRANCH - jle .L89 + jle L(89) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1926,10 +1926,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L85 + jle L(85) ALIGN_3 -.L82: +L(82): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x00, %xmm0, %xmm1 @@ -1961,10 +1961,10 @@ subq $1, %rax BRANCH - jg .L82 + jg L(82) ALIGN_3 -.L85: +L(85): #ifndef TRMMKERNEL movq K, %rax #else @@ -1972,10 +1972,10 @@ #endif andq $3, %rax BRANCH - je .L88 + je L(88) ALIGN_3 -.L86: +L(86): pshufd $0x00, %xmm0, %xmm1 movss -31 * SIZE(AO), %xmm0 mulps %xmm1, %xmm2 @@ -1987,10 +1987,10 @@ subq $1, %rax BRANCH - jg .L86 + jg L(86) ALIGN_3 -.L88: +L(88): movups ALPHA_R, %xmm7 addps %xmm9, %xmm8 @@ -2007,7 +2007,7 @@ movhps %xmm4, 0 * SIZE(CO2) ALIGN_4 -.L89: +L(89): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -2017,9 +2017,9 @@ leaq (C, LDC, 2), C ALIGN_4 -.L90: +L(90): testq $1, N - jle .L999 + jle L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -2032,10 +2032,10 @@ movq M, I sarq $3, I NOBRANCH - jle .L100 + jle L(100) ALIGN_4 -.L91: +L(91): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2076,10 +2076,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L95 + jle L(95) ALIGN_3 -.L92: +L(92): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x00, %xmm2, %xmm3 @@ -2123,10 +2123,10 @@ subq $1, %rax BRANCH - jg .L92 + jg L(92) ALIGN_3 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2134,10 +2134,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_3 -.L96: +L(96): pshufd $0x00, %xmm2, %xmm3 movss -31 * SIZE(BO), %xmm2 mulps %xmm3, %xmm0 @@ -2152,10 +2152,10 @@ subq $1, %rax BRANCH - jg .L96 + jg L(96) ALIGN_3 -.L98: +L(98): movups ALPHA_R, %xmm7 addps %xmm10, %xmm8 @@ -2197,13 +2197,13 @@ addq $16 * SIZE, CO1 decq I BRANCH - jg .L91 + jg L(91) ALIGN_4 -.L100: +L(100): testq $4, M BRANCH - jle .L110 + jle L(110) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2241,10 +2241,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L105 + jle L(105) ALIGN_3 -.L102: +L(102): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x00, %xmm2, %xmm3 @@ -2274,10 +2274,10 @@ subq $1, %rax BRANCH - jg .L102 + jg L(102) ALIGN_3 -.L105: +L(105): #ifndef TRMMKERNEL movq K, %rax #else @@ -2285,10 +2285,10 @@ #endif andq $3, %rax BRANCH - je .L108 + je L(108) ALIGN_3 -.L106: +L(106): pshufd $0x00, %xmm2, %xmm3 movss -31 * SIZE(BO), %xmm2 mulps %xmm0, %xmm3 @@ -2300,10 +2300,10 @@ subq $1, %rax BRANCH - jg .L106 + jg L(106) ALIGN_3 -.L108: +L(108): movups ALPHA_R, %xmm7 addps %xmm9, %xmm8 @@ -2330,10 +2330,10 @@ addq $8 * SIZE, CO1 ALIGN_4 -.L110: +L(110): testq $2, M BRANCH - jle .L120 + jle L(120) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2372,10 +2372,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L115 + jle L(115) ALIGN_3 -.L112: +L(112): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x00, %xmm2, %xmm3 @@ -2405,10 +2405,10 @@ subq $1, %rax BRANCH - jg .L112 + jg L(112) ALIGN_3 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2416,10 +2416,10 @@ #endif andq $3, %rax BRANCH - je .L118 + je L(118) ALIGN_3 -.L116: +L(116): pshufd $0x00, %xmm2, %xmm3 movss -31 * SIZE(BO), %xmm2 mulps %xmm0, %xmm3 @@ -2431,10 +2431,10 @@ subq $1, %rax BRANCH - jg .L116 + jg L(116) ALIGN_3 -.L118: +L(118): movups ALPHA_R, %xmm7 movsd 0 * SIZE(CO1), %xmm0 @@ -2451,10 +2451,10 @@ addq $4 * SIZE, CO1 ALIGN_4 -.L120: +L(120): testq $1, M BRANCH - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2492,10 +2492,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L125 + jle L(125) ALIGN_3 -.L122: +L(122): mulss %xmm0, %xmm2 movss -31 * SIZE(AO), %xmm0 addss %xmm2, %xmm8 @@ -2521,10 +2521,10 @@ subq $1, %rax BRANCH - jg .L122 + jg L(122) ALIGN_3 -.L125: +L(125): #ifndef TRMMKERNEL movq K, %rax #else @@ -2532,10 +2532,10 @@ #endif andq $3, %rax BRANCH - je .L128 + je L(128) ALIGN_3 -.L126: +L(126): mulss %xmm0, %xmm2 movss -31 * SIZE(AO), %xmm0 addss %xmm2, %xmm8 @@ -2546,10 +2546,10 @@ subq $1, %rax BRANCH - jg .L126 + jg L(126) ALIGN_3 -.L128: +L(128): movups ALPHA_R, %xmm7 addss %xmm9, %xmm8 @@ -2564,7 +2564,7 @@ movlps %xmm4, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm3m_kernel_8x4_sse.S b/kernel/x86_64/zgemm3m_kernel_8x4_sse.S index a3c69163e4..d1b9e81818 100644 --- a/kernel/x86_64/zgemm3m_kernel_8x4_sse.S +++ b/kernel/x86_64/zgemm3m_kernel_8x4_sse.S @@ -417,9 +417,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L50 + jle L(50) -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -430,12 +430,12 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): PREFETCH (RPREFETCHSIZE + 0) * SIZE(B) movd 0 * SIZE(B), %mm0 @@ -482,17 +482,17 @@ addq $32 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movd 0 * SIZE(B), %mm0 movd 1 * SIZE(B), %mm1 movd 2 * SIZE(B), %mm2 @@ -515,10 +515,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a @@ -527,10 +527,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): PREFETCH 0 * SIZE(BB) subq $-16 * SIZE, BB @@ -592,10 +592,10 @@ leaq (BO, %rax, 8), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -616,7 +616,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -637,7 +637,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -658,7 +658,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -679,7 +679,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -700,7 +700,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -721,7 +721,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -742,7 +742,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -763,17 +763,17 @@ addq $16 * SIZE, %rax BRANCH - jl .L12 + jl L(12) ALIGN_3 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_3 @@ -792,10 +792,10 @@ #else sarq $2, %rax NOBRANCH - jle .L16 + jle L(16) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -809,10 +809,10 @@ subq $-32 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) #endif -.L16: +L(16): movaps ALPHA, %xmm7 #ifndef TRMMKERNEL @@ -821,7 +821,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L18 + je L(18) leaq (, %rax, 8), %rax leaq (AO, %rax, 4), AO @@ -829,7 +829,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulps %xmm0, %xmm1 addps %xmm1, %xmm8 movaps -28 * SIZE(BO, %rax, 8), %xmm1 @@ -856,10 +856,10 @@ movaps -20 * SIZE(AO, %rax, 4), %xmm2 addq $SIZE * 2, %rax - jl .L17 + jl L(17) ALIGN_4 -.L18: +L(18): movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 movsd 4 * SIZE(CO1), %xmm1 @@ -995,12 +995,12 @@ addq $16 * SIZE, CO1 # coffset += 4 addq $16 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1044,10 +1044,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) @@ -1161,10 +1161,10 @@ addq $ 32 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1173,10 +1173,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1193,10 +1193,10 @@ addq $ 4 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 movsd 4 * SIZE(CO1), %xmm9 @@ -1277,9 +1277,9 @@ addq $8 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $2, M - je .L40 + je L(40) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1323,10 +1323,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) @@ -1445,10 +1445,10 @@ addq $ 16 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1457,10 +1457,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1478,10 +1478,10 @@ addq $ 2 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 @@ -1526,9 +1526,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L40: +L(40): testq $1, M - je .L49 + je L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1573,10 +1573,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): mulss %xmm8, %xmm9 addss %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) @@ -1695,10 +1695,10 @@ addq $ 8 * SIZE, AO addq $128 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1707,10 +1707,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -1728,10 +1728,10 @@ addq $ 1 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): movsd 0 * SIZE(CO1), %xmm8 pshufd $0x50, %xmm0, %xmm4 @@ -1765,19 +1765,19 @@ movlps %xmm4, 0 * SIZE(CO2, LDC, 2) ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C # c += 4 * ldc decq J # j -- - jg .L01 + jg L(01) -.L50: +L(50): testq $2, N - je .L100 + je L(100) -.L51: +L(51): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1788,10 +1788,10 @@ movq K, %rax sarq $2, %rax - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): #if defined(PENTIUM4) || defined(GENERIC) movss 0 * SIZE(B), %xmm0 movss 1 * SIZE(B), %xmm1 @@ -1869,17 +1869,17 @@ #endif decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): movq K, %rax andq $3, %rax BRANCH - jle .L60 + jle L(60) ALIGN_4 -.L54: +L(54): #if defined(PENTIUM4) || defined(GENERIC) movss 0 * SIZE(B), %xmm0 movss 1 * SIZE(B), %xmm1 @@ -1907,20 +1907,20 @@ addq $ 2 * SIZE, B addq $ 8 * SIZE, BO decq %rax - jne .L54 + jne L(54) ALIGN_4 -.L60: +L(60): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L70 + jle L(70) ALIGN_4 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1968,10 +1968,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2091,10 +2091,10 @@ addq $64 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -2103,10 +2103,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -2123,10 +2123,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 movsd 4 * SIZE(CO1), %xmm9 @@ -2196,12 +2196,12 @@ addq $16 * SIZE, CO1 # coffset += 4 addq $16 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L61 + jg L(61) ALIGN_4 -.L70: +L(70): testq $4, M - je .L80 + je L(80) #if !defined(TRMMKERNEL) || \ @@ -2246,10 +2246,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2313,10 +2313,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2325,10 +2325,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -2339,10 +2339,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -2386,9 +2386,9 @@ addq $8 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L80: +L(80): testq $2, M - je .L90 + je L(90) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2432,10 +2432,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L85 + je L(85) ALIGN_4 -.L82: +L(82): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) @@ -2506,10 +2506,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L85: +L(85): #ifndef TRMMKERNEL movq K, %rax #else @@ -2518,10 +2518,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -2533,10 +2533,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -2564,9 +2564,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L90: +L(90): testq $1, M - je .L99 + je L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2610,10 +2610,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) @@ -2684,10 +2684,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2696,10 +2696,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movss 4 * SIZE(BO), %xmm9 @@ -2711,10 +2711,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): addss %xmm2, %xmm0 addss %xmm3, %xmm1 @@ -2735,7 +2735,7 @@ movlps %xmm2, 0 * SIZE(CO2) ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -2743,11 +2743,11 @@ ALIGN_4 -.L100: +L(100): testq $1, N - je .L999 + je L(999) -.L101: +L(101): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -2758,11 +2758,11 @@ movq K, %rax sarq $3, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): #if defined(PENTIUM4) || defined(GENERIC) movss 0 * SIZE(B), %xmm0 movss 1 * SIZE(B), %xmm1 @@ -2840,17 +2840,17 @@ #endif decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): movq K, %rax andq $7, %rax BRANCH - jle .L110 + jle L(110) ALIGN_4 -.L104: +L(104): #if defined(PENTIUM4) || defined(GENERIC) movss 0 * SIZE(B), %xmm0 shufps $0, %xmm0, %xmm0 @@ -2867,19 +2867,19 @@ addq $ 1 * SIZE, B addq $ 4 * SIZE, BO decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L110: +L(110): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L120 + jle L(120) ALIGN_4 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2926,10 +2926,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulps %xmm9, %xmm8 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3002,10 +3002,10 @@ addq $64 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -3014,10 +3014,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulps %xmm9, %xmm8 mulps -28 * SIZE(AO), %xmm9 addps %xmm8, %xmm0 @@ -3028,10 +3028,10 @@ addq $8 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 movsd 4 * SIZE(CO1), %xmm9 @@ -3067,12 +3067,12 @@ addq $16 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L111 + jg L(111) ALIGN_4 -.L120: +L(120): testq $4, M - je .L130 + je L(130) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3114,10 +3114,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L125 + je L(125) ALIGN_4 -.L122: +L(122): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3155,10 +3155,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L122 + jne L(122) ALIGN_4 -.L125: +L(125): #ifndef TRMMKERNEL movq K, %rax #else @@ -3167,10 +3167,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): mulps %xmm8, %xmm9 movaps -28 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -3179,10 +3179,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): addps %xmm1, %xmm0 addps %xmm3, %xmm2 addps %xmm2, %xmm0 @@ -3209,9 +3209,9 @@ addq $8 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L130: +L(130): testq $2, M - je .L140 + je L(140) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3253,10 +3253,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L135 + je L(135) ALIGN_4 -.L132: +L(132): mulps %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3302,10 +3302,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L132 + jne L(132) ALIGN_4 -.L135: +L(135): #ifndef TRMMKERNEL movq K, %rax #else @@ -3314,10 +3314,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): mulps %xmm8, %xmm9 movsd -30 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -3326,10 +3326,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): addps %xmm1, %xmm0 movsd 0 * SIZE(CO1), %xmm8 @@ -3345,9 +3345,9 @@ addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L140: +L(140): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3389,10 +3389,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L145 + je L(145) ALIGN_4 -.L142: +L(142): mulss %xmm8, %xmm9 #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3426,10 +3426,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L142 + jne L(142) ALIGN_4 -.L145: +L(145): #ifndef TRMMKERNEL movq K, %rax #else @@ -3438,10 +3438,10 @@ movaps ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): mulss %xmm8, %xmm9 movss -31 * SIZE(AO), %xmm8 addss %xmm9, %xmm0 @@ -3450,10 +3450,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): addss %xmm1, %xmm0 addss %xmm3, %xmm2 addss %xmm2, %xmm0 @@ -3467,7 +3467,7 @@ movlps %xmm2, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp EMMS movq 0(%rsp), %rbx diff --git a/kernel/x86_64/zgemm3m_kernel_8x4_sse3.S b/kernel/x86_64/zgemm3m_kernel_8x4_sse3.S index adf0a53316..a2fc2a3e10 100644 --- a/kernel/x86_64/zgemm3m_kernel_8x4_sse3.S +++ b/kernel/x86_64/zgemm3m_kernel_8x4_sse3.S @@ -400,9 +400,9 @@ movq N, J sarq $2, J # j = (n >> 2) - jle .L50 + jle L(50) -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -413,10 +413,10 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): movddup 0 * SIZE(B), %xmm0 movddup 2 * SIZE(B), %xmm1 movddup 4 * SIZE(B), %xmm2 @@ -442,17 +442,17 @@ addq $32 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movddup 0 * SIZE(B), %xmm0 movddup 2 * SIZE(B), %xmm1 @@ -462,10 +462,10 @@ addq $4 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a @@ -474,10 +474,10 @@ movq M, I sarq $3, I # i = (m >> 3) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): prefetcht0 0 * SIZE(BB) prefetcht0 8 * SIZE(BB) subq $-16 * SIZE, BB @@ -538,9 +538,9 @@ #if 1 andq $-8, %rax salq $4, %rax - je .L15 + je L(15) -.L1X: +L(1X): KERNEL1 (64 * 0) KERNEL2 (64 * 0) KERNEL3 (64 * 0) @@ -559,7 +559,7 @@ KERNEL16(64 * 0) cmpq $128 * 1, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 1) KERNEL2 (64 * 1) KERNEL3 (64 * 1) @@ -578,7 +578,7 @@ KERNEL16(64 * 1) cmpq $128 * 2, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 2) KERNEL2 (64 * 2) KERNEL3 (64 * 2) @@ -597,7 +597,7 @@ KERNEL16(64 * 2) cmpq $128 * 3, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 3) KERNEL2 (64 * 3) KERNEL3 (64 * 3) @@ -616,7 +616,7 @@ KERNEL16(64 * 3) cmpq $128 * 4, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 4) KERNEL2 (64 * 4) KERNEL3 (64 * 4) @@ -635,7 +635,7 @@ KERNEL16(64 * 4) cmpq $128 * 5, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 5) KERNEL2 (64 * 5) KERNEL3 (64 * 5) @@ -654,7 +654,7 @@ KERNEL16(64 * 5) cmpq $128 * 6, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 6) KERNEL2 (64 * 6) KERNEL3 (64 * 6) @@ -673,7 +673,7 @@ KERNEL16(64 * 6) cmpq $128 * 7, %rax NOBRANCH - jle .L12 + jle L(12) KERNEL1 (64 * 7) KERNEL2 (64 * 7) KERNEL3 (64 * 7) @@ -694,17 +694,17 @@ addq $64 * 8 * SIZE, AO addq $64 * 8 * SIZE, BO subq $128 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO leaq (BO, %rax, 2), BO #else sarq $3, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): KERNEL1 (64 * 0) KERNEL2 (64 * 0) KERNEL3 (64 * 0) @@ -725,11 +725,11 @@ addq $64 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L12 + jne L(12) #endif ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else @@ -738,10 +738,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -772,10 +772,10 @@ addq $8 * SIZE, AO addq $8 * SIZE, BO decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 movsd 4 * SIZE(CO1), %xmm9 @@ -926,12 +926,12 @@ addq $16 * SIZE, CO1 # coffset += 4 addq $16 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $4, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -975,10 +975,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm9, %xmm0 @@ -1088,10 +1088,10 @@ addq $32 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1100,10 +1100,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -1121,10 +1121,10 @@ addq $4 * SIZE, AO addq $8 * SIZE, BO decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 movsd 4 * SIZE(CO1), %xmm9 @@ -1205,9 +1205,9 @@ addq $8 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $2, M - je .L40 + je L(40) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1248,10 +1248,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): shufps $0x50, %xmm9, %xmm9 mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1329,10 +1329,10 @@ addq $16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1341,10 +1341,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): shufps $0x50, %xmm9, %xmm9 mulps %xmm8, %xmm9 addps %xmm9, %xmm0 @@ -1358,10 +1358,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -1407,9 +1407,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L40: +L(40): testq $1, M - je .L49 + je L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1448,10 +1448,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L45 + je L(45) ALIGN_4 -.L42: +L(42): shufps $0, %xmm8, %xmm8 movhps 4 * SIZE(BO), %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1505,10 +1505,10 @@ addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1517,10 +1517,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_4 -.L46: +L(46): shufps $0, %xmm8, %xmm8 movhps 4 * SIZE(BO), %xmm9 mulps %xmm8, %xmm9 @@ -1531,10 +1531,10 @@ addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L46 + jg L(46) ALIGN_4 -.L48: +L(48): addps %xmm1, %xmm0 movsd 0 * SIZE(CO1), %xmm8 @@ -1557,19 +1557,19 @@ movhps %xmm0, 0 * SIZE(CO2, LDC, 2) ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK #endif leaq (C, LDC, 4), C # c += 4 * ldc decq J # j -- - jg .L01 + jg L(01) -.L50: +L(50): testq $2, N - je .L100 + je L(100) -.L51: +L(51): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1580,10 +1580,10 @@ movq K, %rax sarq $3, %rax - jle .L53 + jle L(53) ALIGN_4 -.L52: +L(52): movddup 0 * SIZE(B), %xmm0 movddup 2 * SIZE(B), %xmm1 movddup 4 * SIZE(B), %xmm2 @@ -1609,37 +1609,37 @@ addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L53: +L(53): movq K, %rax andq $7, %rax BRANCH - jle .L60 + jle L(60) ALIGN_4 -.L54: +L(54): movddup 0 * SIZE(B), %xmm0 movaps %xmm0, 0 * SIZE(BO) addq $ 2 * SIZE, B addq $ 4 * SIZE, BO decq %rax - jne .L54 + jne L(54) ALIGN_4 -.L60: +L(60): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L70 + jle L(70) ALIGN_4 -.L61: +L(61): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1684,10 +1684,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm9, %xmm0 @@ -1806,10 +1806,10 @@ addq $64 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1818,10 +1818,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -1840,10 +1840,10 @@ addq $8 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): movsd 0 * SIZE(CO1), %xmm8 movhps 2 * SIZE(CO1), %xmm8 movsd 4 * SIZE(CO1), %xmm9 @@ -1913,12 +1913,12 @@ addq $16 * SIZE, CO1 # coffset += 4 addq $16 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L61 + jg L(61) ALIGN_4 -.L70: +L(70): testq $4, M - je .L80 + je L(80) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1959,10 +1959,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm9, %xmm0 @@ -2024,10 +2024,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2036,10 +2036,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -2051,10 +2051,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -2100,9 +2100,9 @@ addq $8 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L80: +L(80): testq $2, M - je .L90 + je L(90) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2141,10 +2141,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L85 + je L(85) ALIGN_4 -.L82: +L(82): shufps $0x50, %xmm9, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulps %xmm8, %xmm9 @@ -2190,10 +2190,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L82 + jne L(82) ALIGN_4 -.L85: +L(85): #ifndef TRMMKERNEL movq K, %rax #else @@ -2202,10 +2202,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L88 + je L(88) ALIGN_4 -.L86: +L(86): shufps $0x50, %xmm9, %xmm9 mulps %xmm8, %xmm9 movddup 2 * SIZE(AO), %xmm8 @@ -2215,10 +2215,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L86 + jg L(86) ALIGN_4 -.L88: +L(88): addps %xmm1, %xmm0 movsd 0 * SIZE(CO1), %xmm8 @@ -2244,9 +2244,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L90: +L(90): testq $1, M - je .L99 + je L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2285,10 +2285,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L95 + je L(95) ALIGN_4 -.L92: +L(92): shufps $0, %xmm8, %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulps %xmm8, %xmm9 @@ -2334,10 +2334,10 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L92 + jne L(92) ALIGN_4 -.L95: +L(95): #ifndef TRMMKERNEL movq K, %rax #else @@ -2346,10 +2346,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L98 + je L(98) ALIGN_4 -.L96: +L(96): shufps $0, %xmm8, %xmm8 mulps %xmm8, %xmm9 movss 1 * SIZE(AO), %xmm8 @@ -2359,10 +2359,10 @@ addq $1 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L96 + jg L(96) ALIGN_4 -.L98: +L(98): addps %xmm1, %xmm0 movsd 0 * SIZE(CO1), %xmm8 @@ -2378,18 +2378,18 @@ movhps %xmm12, 0 * SIZE(CO2) ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif leaq (C, LDC, 2), C # c += 4 * ldc ALIGN_4 -.L100: +L(100): testq $1, N - je .L999 + je L(999) -.L101: +L(101): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -2400,11 +2400,11 @@ movq K, %rax sarq $3, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movss 0 * SIZE(B), %xmm0 movss 1 * SIZE(B), %xmm1 movss 2 * SIZE(B), %xmm2 @@ -2435,17 +2435,17 @@ addq $16 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): movq K, %rax andq $7, %rax BRANCH - jle .L110 + jle L(110) ALIGN_4 -.L104: +L(104): movss 0 * SIZE(B), %xmm0 movss %xmm0, 0 * SIZE(BO) movss %xmm0, 1 * SIZE(BO) @@ -2453,19 +2453,19 @@ addq $ 1 * SIZE, B addq $ 2 * SIZE, BO decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L110: +L(110): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $3, I # i = (m >> 3) - jle .L120 + jle L(120) ALIGN_4 -.L111: +L(111): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -2509,10 +2509,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L115 + je L(115) ALIGN_4 -.L112: +L(112): mulps %xmm8, %xmm9 movaps 4 * SIZE(AO), %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2583,10 +2583,10 @@ addq $64 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L112 + jne L(112) ALIGN_4 -.L115: +L(115): #ifndef TRMMKERNEL movq K, %rax #else @@ -2595,10 +2595,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L118 + je L(118) ALIGN_4 -.L116: +L(116): mulps %xmm8, %xmm9 movaps 4 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -2611,10 +2611,10 @@ addq $8 * SIZE, AO addq $2 * SIZE, BO decq %rax - jg .L116 + jg L(116) ALIGN_4 -.L118: +L(118): addps %xmm1, %xmm0 addps %xmm5, %xmm4 @@ -2653,12 +2653,12 @@ addq $16 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L111 + jg L(111) ALIGN_4 -.L120: +L(120): testq $4, M - je .L130 + je L(130) #if !defined(TRMMKERNEL) || \ @@ -2704,10 +2704,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L125 + je L(125) ALIGN_4 -.L122: +L(122): mulps %xmm8, %xmm9 movaps 4 * SIZE(AO), %xmm8 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2745,10 +2745,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L122 + jne L(122) ALIGN_4 -.L125: +L(125): #ifndef TRMMKERNEL movq K, %rax #else @@ -2757,10 +2757,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L128 + je L(128) ALIGN_4 -.L126: +L(126): mulps %xmm8, %xmm9 movaps 4 * SIZE(AO), %xmm8 addps %xmm9, %xmm0 @@ -2769,10 +2769,10 @@ addq $4 * SIZE, AO addq $2 * SIZE, BO decq %rax - jg .L126 + jg L(126) ALIGN_4 -.L128: +L(128): addps %xmm1, %xmm0 movsd 0 * SIZE(CO1), %xmm8 @@ -2797,9 +2797,9 @@ addq $8 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L130: +L(130): testq $2, M - je .L140 + je L(140) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2840,10 +2840,10 @@ movq %rax, KKK #endif sarq $4, %rax - je .L135 + je L(135) ALIGN_4 -.L132: +L(132): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps 4 * SIZE(AO), %xmm8 @@ -2875,10 +2875,10 @@ addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L132 + jne L(132) ALIGN_4 -.L135: +L(135): #ifndef TRMMKERNEL movq K, %rax #else @@ -2887,10 +2887,10 @@ movaps ALPHA, %xmm15 andq $15, %rax # if (k & 1) BRANCH - je .L138 + je L(138) ALIGN_4 -.L136: +L(136): movsd 0 * SIZE(AO), %xmm8 movsd 0 * SIZE(BO), %xmm9 @@ -2900,10 +2900,10 @@ addq $2 * SIZE, AO addq $2 * SIZE, BO decq %rax - jg .L136 + jg L(136) ALIGN_4 -.L138: +L(138): addps %xmm1, %xmm0 addps %xmm3, %xmm2 addps %xmm2, %xmm0 @@ -2925,9 +2925,9 @@ addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L140: +L(140): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2968,10 +2968,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L145 + je L(145) ALIGN_4 -.L142: +L(142): mulss %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movss 1 * SIZE(AO), %xmm8 @@ -3003,10 +3003,10 @@ addq $ 8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L142 + jne L(142) ALIGN_4 -.L145: +L(145): #ifndef TRMMKERNEL movq K, %rax #else @@ -3015,10 +3015,10 @@ movaps ALPHA, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L148 + je L(148) ALIGN_4 -.L146: +L(146): movss 0 * SIZE(AO), %xmm8 movss 0 * SIZE(BO), %xmm9 mulps %xmm8, %xmm9 @@ -3027,10 +3027,10 @@ addq $1 * SIZE, AO addq $2 * SIZE, BO decq %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): addss %xmm1, %xmm0 addss %xmm3, %xmm2 addss %xmm2, %xmm0 @@ -3045,7 +3045,7 @@ movlps %xmm12, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/zgemm_beta.S b/kernel/x86_64/zgemm_beta.S index 1612d92f16..3f5a49d054 100644 --- a/kernel/x86_64/zgemm_beta.S +++ b/kernel/x86_64/zgemm_beta.S @@ -96,35 +96,35 @@ movq STACK_LDC, LDC testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) salq $ZBASE_SHIFT, LDC #ifdef DOUBLE ucomisd %xmm15, %xmm0 - jne .L71 + jne L(71) ucomisd %xmm15, %xmm1 - jne .L71 + jne L(71) #else ucomiss %xmm15, %xmm0 - jne .L71 + jne L(71) ucomiss %xmm15, %xmm1 - jne .L71 + jne L(71) #endif ALIGN_2 -.L53: +L(53): movq C, C1 # c_offset1 = c_offset addq LDC, C # c_offset += ldc movq M, I sarq $2, I - jle .L56 + jle L(56) ALIGN_2 -.L57: +L(57): #ifdef OPTERON prefetchw 64 * SIZE(C1) #endif @@ -139,39 +139,39 @@ MOVSD %xmm0, 7 * SIZE(C1) addq $8 * SIZE, C1 # c_offset1 += 8 decq I # i-- - jg .L57 + jg L(57) ALIGN_2 -.L56: +L(56): movq M, I andq $3, I - jle .L62 + jle L(62) ALIGN_2 -.L63: +L(63): MOVSD %xmm0, 0 * SIZE(C1) MOVSD %xmm0, 1 * SIZE(C1) addq $2 * SIZE,C1 decq I - jg .L63 + jg L(63) ALIGN_2 -.L62: +L(62): decq N # j -- - jg .L53 - jmp .L999 + jg L(53) + jmp L(999) ALIGN_3 -.L71: +L(71): movq C, C1 addq LDC, C # c_offset += ldc movq M, I sarq $1, I - jle .L84 + jle L(84) ALIGN_3 -.L85: +L(85): #ifdef OPTERON prefetchw 16 * SIZE(C1) #endif @@ -207,15 +207,15 @@ MOVSD %xmm8, 3 * SIZE(C1) addq $4 * SIZE, C1 decq I - jg .L85 + jg L(85) ALIGN_3 -.L84: +L(84): testq $1, M - jle .L74 + jle L(74) ALIGN_3 -.L75: +L(75): prefetchnta 80 * SIZE(C1) MOVSD 0 * SIZE(C1), %xmm2 @@ -234,12 +234,12 @@ MOVSD %xmm4, 1 * SIZE(C1) ALIGN_2 -.L74: +L(74): decq N - jg .L71 + jg L(71) ALIGN_2 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 movups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/zgemm_kernel_1x4_nehalem.S b/kernel/x86_64/zgemm_kernel_1x4_nehalem.S index 0d6acf3309..b8a618ea3d 100644 --- a/kernel/x86_64/zgemm_kernel_1x4_nehalem.S +++ b/kernel/x86_64/zgemm_kernel_1x4_nehalem.S @@ -178,15 +178,15 @@ movq %r11, KK #endif testq M, M - jle .L999 + jle L(999) movq N, J sarq $2, J NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -203,7 +203,7 @@ movq M, I ALIGN_4 -.L11: +L(11): prefetcht2 -16 * SIZE(BB) subq $-8 * SIZE, BB @@ -260,10 +260,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm12 @@ -389,10 +389,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): movddup ALPHA_R, %xmm6 movddup ALPHA_I, %xmm7 @@ -403,10 +403,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm1, %xmm12 movaps -16 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm13 @@ -442,10 +442,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): ADD1 %xmm1, %xmm12 ADD2 %xmm2, %xmm13 ADD1 %xmm3, %xmm14 @@ -505,7 +505,7 @@ testq $15, CO1 NOBRANCH - jne .L18x + jne L(18x) #ifndef TRMMKERNEL movaps (CO1), %xmm0 @@ -541,7 +541,7 @@ addq $2 * SIZE, CO2 decq I BRANCH - jg .L11 + jg L(11) #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK @@ -552,11 +552,11 @@ subq $1, J BRANCH - jg .L01 - jmp .L20 + jg L(01) + jmp L(20) ALIGN_4 -.L18x: +L(18x): #ifndef TRMMKERNEL movups (CO1), %xmm0 movups (CO1, LDC), %xmm1 @@ -591,7 +591,7 @@ addq $2 * SIZE, CO2 decq I BRANCH - jg .L11 + jg L(11) #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK @@ -602,13 +602,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L20: +L(20): testq $2, N BRANCH - jle .L30 + jle L(30) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -622,7 +622,7 @@ movq M, I ALIGN_4 -.L21: +L(21): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -667,10 +667,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -742,10 +742,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -753,10 +753,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): ADD1 %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -778,10 +778,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): ADD1 %xmm1, %xmm8 ADD2 %xmm2, %xmm9 ADD1 %xmm3, %xmm10 @@ -857,7 +857,7 @@ addq $2 * SIZE, CO2 decq I BRANCH - jg .L21 + jg L(21) #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK @@ -867,10 +867,10 @@ movq BO, B ALIGN_4 -.L30: +L(30): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -883,7 +883,7 @@ movq M, I ALIGN_4 -.L31: +L(31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -925,10 +925,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -968,13 +968,13 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 ALIGN_3 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -982,10 +982,10 @@ #endif andq $3, %rax BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): ADD1 %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -999,10 +999,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): ADD1 %xmm1, %xmm8 ADD2 %xmm2, %xmm9 @@ -1061,10 +1061,10 @@ addq $2 * SIZE, CO1 decq I BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm_kernel_2x1_atom.S b/kernel/x86_64/zgemm_kernel_2x1_atom.S index d9f320a8a1..a570279569 100644 --- a/kernel/x86_64/zgemm_kernel_2x1_atom.S +++ b/kernel/x86_64/zgemm_kernel_2x1_atom.S @@ -172,10 +172,10 @@ movq N, J testq N, N - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -192,10 +192,10 @@ movq M, I sarq $1, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -249,10 +249,10 @@ #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): ADDSD2 %xmm2, %xmm13 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps %xmm0, %xmm2 @@ -395,10 +395,10 @@ mulsd %xmm3, %xmm6 movsd 1 * SIZE(BO), %xmm3 - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else @@ -407,10 +407,10 @@ andq $3, %rax BRANCH BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): ADDSD2 %xmm2, %xmm13 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -449,10 +449,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): movsd ALPHA_R, %xmm0 movsd ALPHA_I, %xmm1 @@ -513,12 +513,12 @@ addq $4 * SIZE, CO1 decq I - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): testq $1, M - jle .L99 + jle L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -565,10 +565,10 @@ #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADDSD2 %xmm2, %xmm9 movaps %xmm0, %xmm2 @@ -646,10 +646,10 @@ addq $8 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -658,10 +658,10 @@ andq $3, %rax BRANCH BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): ADDSD2 %xmm2, %xmm9 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -684,10 +684,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): movsd ALPHA_R, %xmm0 movsd ALPHA_I, %xmm1 @@ -730,17 +730,17 @@ #endif ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addq $1, KK #endif movq BO, B decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm_kernel_2x2_barcelona.S b/kernel/x86_64/zgemm_kernel_2x2_barcelona.S index 70e8f6039c..7b47a84a24 100644 --- a/kernel/x86_64/zgemm_kernel_2x2_barcelona.S +++ b/kernel/x86_64/zgemm_kernel_2x2_barcelona.S @@ -485,10 +485,10 @@ movq N, J sarq $1, J - jle .L100 + jle L(100) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -505,10 +505,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L30 + jle L(30) ALIGN_4 -.L10: +L(10): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -562,10 +562,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -575,7 +575,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -585,7 +585,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -595,7 +595,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -605,7 +605,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -615,7 +615,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -625,7 +625,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -635,7 +635,7 @@ KERNEL7(16 * 0) KERNEL8(16 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -644,17 +644,17 @@ KERNEL6(16 * 0) KERNEL7(16 * 0) KERNEL8(16 * 0) - jl .L12 + jl L(12) ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_4 @@ -667,7 +667,7 @@ subq $-16 * SIZE, AO ALIGN_4 -.L16: +L(16): movddup ALPHA_R, %xmm6 movddup ALPHA_I, %xmm7 @@ -677,7 +677,7 @@ movq KKK, %rax #endif andq $3, %rax - je .L19 + je L(19) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -685,7 +685,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 ADD1 %xmm0, %xmm8 @@ -712,10 +712,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L17 + jl L(17) ALIGN_4 -.L19: +L(19): prefetch -8 * SIZE(BB) subq $-16 * SIZE, BB @@ -807,12 +807,12 @@ addq $4 * SIZE, CO1 # coffset += 4 addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L30: +L(30): testq $1, M - jle .L99 + jle L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -857,10 +857,10 @@ leaq (BO, %rax, 4), BO negq %rax NOBRANCH - je .L46 + je L(46) ALIGN_4 -.L42: +L(42): mulpd %xmm0, %xmm1 ADD1 %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -916,10 +916,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L42 + jl L(42) ALIGN_4 -.L46: +L(46): movddup ALPHA_R, %xmm6 movddup ALPHA_I, %xmm7 @@ -929,7 +929,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L49 + je L(49) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -937,7 +937,7 @@ negq %rax ALIGN_4 -.L47: +L(47): mulpd %xmm0, %xmm1 ADD1 %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 4), %xmm1 @@ -953,10 +953,10 @@ movddup -11 * SIZE(BO, %rax, 4), %xmm5 addq $SIZE, %rax - jl .L47 + jl L(47) ALIGN_4 -.L49: +L(49): #ifndef TRMMKERNEL movupd (CO1), %xmm0 movupd (CO2), %xmm1 @@ -1014,7 +1014,7 @@ #endif ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif @@ -1023,13 +1023,13 @@ leaq (C, LDC, 2), C # c += 2 * ldc decq J # j -- - jg .L01 + jg L(01) -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) -.L101: +L(101): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1040,10 +1040,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L130 + jle L(130) ALIGN_4 -.L110: +L(110): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1089,10 +1089,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L116 + je L(116) ALIGN_4 -.L112: +L(112): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 ADD1 %xmm0, %xmm8 @@ -1144,10 +1144,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L112 + jl L(112) ALIGN_4 -.L116: +L(116): movddup ALPHA_R, %xmm6 movddup ALPHA_I, %xmm7 @@ -1157,7 +1157,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L119 + je L(119) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -1165,7 +1165,7 @@ negq %rax ALIGN_4 -.L117: +L(117): mulpd %xmm1, %xmm0 mulpd -14 * SIZE(AO, %rax, 4), %xmm1 ADD1 %xmm0, %xmm8 @@ -1180,10 +1180,10 @@ movapd %xmm0, %xmm2 addq $SIZE, %rax - jl .L117 + jl L(117) ALIGN_4 -.L119: +L(119): #ifndef TRMMKERNEL movupd (CO1), %xmm0 movupd 2 * SIZE(CO1), %xmm2 @@ -1243,12 +1243,12 @@ addq $4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L130: +L(130): testq $1, M - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1292,10 +1292,10 @@ leaq (BO, %rax, 2), BO negq %rax NOBRANCH - je .L146 + je L(146) ALIGN_4 -.L142: +L(142): mulpd %xmm0, %xmm1 ADD1 %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -1327,10 +1327,10 @@ addq $4 * SIZE, %rax BRANCH - jl .L142 + jl L(142) ALIGN_4 -.L146: +L(146): movddup ALPHA_R, %xmm6 movddup ALPHA_I, %xmm7 @@ -1340,7 +1340,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L148 + je L(148) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO @@ -1348,7 +1348,7 @@ negq %rax ALIGN_4 -.L147: +L(147): mulpd %xmm0, %xmm1 ADD1 %xmm1, %xmm8 movddup -14 * SIZE(BO, %rax, 2), %xmm1 @@ -1358,10 +1358,10 @@ movddup -13 * SIZE(BO, %rax, 2), %xmm3 addq $SIZE, %rax - jl .L147 + jl L(147) ALIGN_4 -.L148: +L(148): #ifndef TRMMKERNEL movupd (CO1), %xmm0 #endif @@ -1394,7 +1394,7 @@ movhpd %xmm8, 1 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq (%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm_kernel_2x2_bulldozer.S b/kernel/x86_64/zgemm_kernel_2x2_bulldozer.S index 6c8b4c8722..aaaf4d8d44 100644 --- a/kernel/x86_64/zgemm_kernel_2x2_bulldozer.S +++ b/kernel/x86_64/zgemm_kernel_2x2_bulldozer.S @@ -451,13 +451,13 @@ STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -485,23 +485,23 @@ #endif #endif -.L2_0: +L(2_0): movq Ndiv6, J cmpq $0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovups (BO1), %xmm0 vmovups 2 * SIZE(BO1), %xmm1 @@ -510,13 +510,13 @@ addq $4*SIZE,BO1 addq $4*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -530,11 +530,11 @@ movq M, I sarq $1, I // i = (m >> 1) - je .L2_40 + je L(2_40) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -572,7 +572,7 @@ andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -583,7 +583,7 @@ negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -599,7 +599,7 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -615,12 +615,12 @@ KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -628,7 +628,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -640,14 +640,14 @@ negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL2x2_SUB(xxx) - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vmovddup ALPHA_R, %xmm0 vmovddup ALPHA_I, %xmm1 @@ -743,20 +743,20 @@ addq $4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -794,7 +794,7 @@ andq $-8, %rax // K = K - ( K % 8 ) - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -805,7 +805,7 @@ negq %rax ALIGN_4 -.L2_42: +L(2_42): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x2_1(xxx) @@ -821,7 +821,7 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x2_1(xxx) @@ -837,12 +837,12 @@ KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -850,7 +850,7 @@ #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -862,14 +862,14 @@ negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vmovddup ALPHA_R, %xmm0 vmovddup ALPHA_I, %xmm1 @@ -945,17 +945,17 @@ -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -963,30 +963,30 @@ movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -1000,11 +1000,11 @@ movq M, I sarq $1, I // i = (m >> 1) - je .L1_40 + je L(1_40) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1042,7 +1042,7 @@ andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1053,7 +1053,7 @@ negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -1067,7 +1067,7 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_16 + je L(1_16) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -1081,12 +1081,12 @@ KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -1094,7 +1094,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1106,14 +1106,14 @@ negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL2x1_SUB(xxx) - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vmovddup ALPHA_R, %xmm0 vmovddup ALPHA_I, %xmm1 @@ -1186,20 +1186,20 @@ addq $4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1237,7 +1237,7 @@ andq $-8, %rax // K = K - ( K % 8 ) - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1248,7 +1248,7 @@ negq %rax ALIGN_4 -.L1_42: +L(1_42): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x1_1(xxx) @@ -1262,7 +1262,7 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x1_1(xxx) @@ -1276,12 +1276,12 @@ KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -1289,7 +1289,7 @@ #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1301,14 +1301,14 @@ negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vmovddup ALPHA_R, %xmm0 vmovddup ALPHA_I, %xmm1 @@ -1372,7 +1372,7 @@ -.L999: +L(999): vzeroupper movq SP, %rsp diff --git a/kernel/x86_64/zgemm_kernel_2x2_core2.S b/kernel/x86_64/zgemm_kernel_2x2_core2.S index b74e2fe91b..03110c4de1 100644 --- a/kernel/x86_64/zgemm_kernel_2x2_core2.S +++ b/kernel/x86_64/zgemm_kernel_2x2_core2.S @@ -183,10 +183,10 @@ movq N, J sarq $1, J # j = (n >> 2) NOBRANCH - jle .L100 + jle L(100) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -200,10 +200,10 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_3 -.L02: +L(02): prefetcht0 (PREFETCH_R + 0) * SIZE(B) prefetcht0 (PREFETCH_R + 8) * SIZE(B) @@ -261,17 +261,17 @@ subq $-32 * SIZE, BO subq $-16 * SIZE, B decq %rax - jne .L02 + jne L(02) ALIGN_3 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L05 + jle L(05) ALIGN_3 -.L04: +L(04): movapd -14 * SIZE(B), %xmm1 movddup %xmm0, %xmm8 @@ -290,10 +290,10 @@ addq $ 8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_3 -.L05: +L(05): leaq (PREFETCH_R + 0) * SIZE(B), BB movq C, CO1 # coffset1 = c @@ -302,10 +302,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L30 + jle L(30) ALIGN_4 -.L10: +L(10): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -363,10 +363,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): PADDING; ADD1 %xmm2, %xmm10 movaps -15 * SIZE(BO), %xmm2 @@ -496,10 +496,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): prefetcht2 -8 * SIZE(BB) #ifndef TRMMKERNEL @@ -510,10 +510,10 @@ andq $3, %rax BRANCH BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): ADD1 %xmm2, %xmm10 movaps -15 * SIZE(BO), %xmm2 ADD1 %xmm3, %xmm14 @@ -548,10 +548,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L19: +L(19): movapd ALPHA_R, %xmm6 ADD1 %xmm2, %xmm10 ADD1 %xmm3, %xmm14 @@ -653,14 +653,14 @@ addq $4 * SIZE, CO1 # coffset += 4 addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L30: +L(30): testq $1, M - jle .L99 + jle L(99) -.L40: +L(40): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -695,9 +695,9 @@ movq %rax, KKK #endif sarq $2, %rax - je .L42 + je L(42) -.L41: +L(41): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -767,9 +767,9 @@ subq $ -8 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L41 + jne L(41) -.L42: +L(42): #ifndef TRMMKERNEL movq K, %rax #else @@ -777,9 +777,9 @@ #endif andq $3, %rax # if (k & 1) BRANCH - jle .L44 + jle L(44) -.L43: +L(43): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -799,10 +799,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): movapd ALPHA_R, %xmm6 movapd ALPHA_I, %xmm7 @@ -869,20 +869,20 @@ #endif ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif leaq (C, LDC, 2), C # c += 2 * ldc decq J # j -- - jg .L01 + jg L(01) -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) -.L101: +L(101): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -893,10 +893,10 @@ movq K, %rax sarq $2, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 movddup -14 * SIZE(B), %xmm10 @@ -918,17 +918,17 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): movq K, %rax andq $3, %rax BRANCH - jle .L105 + jle L(105) ALIGN_4 -.L104: +L(104): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 @@ -938,19 +938,19 @@ addq $4 * SIZE, BO addq $2 * SIZE, B decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L105: +L(105): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $1, I # i = (m >> 2) - jle .L130 + jle L(130) ALIGN_4 -.L110: +L(110): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -986,9 +986,9 @@ movq %rax, KKK #endif sarq $2, %rax - je .L112 + je L(112) -.L111: +L(111): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1066,10 +1066,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #ifndef TRMMKERNEL movq K, %rax #else @@ -1077,9 +1077,9 @@ #endif andq $3, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): movapd -16 * SIZE(AO), %xmm0 movapd -14 * SIZE(AO), %xmm1 @@ -1101,10 +1101,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): movapd ALPHA_R, %xmm6 movapd ALPHA_I, %xmm7 @@ -1171,15 +1171,15 @@ addq $4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L130: +L(130): testq $1, M - jle .L999 + jle L(999) ALIGN_4 -.L140: +L(140): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1214,9 +1214,9 @@ movq %rax, KKK #endif sarq $2, %rax - je .L142 + je L(142) -.L141: +L(141): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1256,9 +1256,9 @@ subq $ -8 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L141 + jne L(141) -.L142: +L(142): #ifndef TRMMKERNEL movq K, %rax #else @@ -1266,9 +1266,9 @@ #endif andq $3, %rax # if (k & 1) BRANCH - jle .L144 + jle L(144) -.L143: +L(143): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -1282,10 +1282,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L143 + jg L(143) ALIGN_4 -.L144: +L(144): movapd ALPHA_R, %xmm6 movapd ALPHA_I, %xmm7 @@ -1322,7 +1322,7 @@ movhpd %xmm8, 1 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/zgemm_kernel_2x2_penryn.S b/kernel/x86_64/zgemm_kernel_2x2_penryn.S index 24e1afdeaf..74bc014e60 100644 --- a/kernel/x86_64/zgemm_kernel_2x2_penryn.S +++ b/kernel/x86_64/zgemm_kernel_2x2_penryn.S @@ -204,10 +204,10 @@ movq N, J sarq $1, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -224,10 +224,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -282,10 +282,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm3, %xmm12 movaps -15 * SIZE(BO), %xmm3 @@ -413,10 +413,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): PREFETCHB -8 * SIZE(BB) #ifdef DUNNINGTON PREFETCHB 0 * SIZE(BB) @@ -430,10 +430,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm3, %xmm12 movaps -15 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 @@ -469,10 +469,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #ifndef DUNNINGTON subq $-16 * SIZE, BB #else @@ -579,13 +579,13 @@ addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M BRANCH - jle .L39 + jle L(39) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -631,10 +631,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_4 -.L22: +L(22): ADD1 %xmm3, %xmm12 movaps -15 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -700,10 +700,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -711,10 +711,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): ADD1 %xmm3, %xmm12 movaps -15 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -735,10 +735,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): ADD1 %xmm3, %xmm12 pcmpeqb %xmm0, %xmm0 ADD2 %xmm5, %xmm13 @@ -811,7 +811,7 @@ addq $2 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -821,13 +821,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $1, N BRANCH - jle .L999 + jle L(999) movq C, CO1 leaq (C, LDC, 1), CO2 @@ -841,10 +841,10 @@ movq M, I sarq $1, I # i = (m >> 2) NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -886,10 +886,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_4 -.L52: +L(52): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -964,10 +964,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_4 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -975,10 +975,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -1001,10 +1001,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): pcmpeqb %xmm0, %xmm0 movddup ALPHA_R, %xmm2 psllq $63, %xmm0 @@ -1074,13 +1074,13 @@ addq $4 * SIZE, CO2 decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L79 + jle L(79) ALIGN_4 #if !defined(TRMMKERNEL) || \ @@ -1121,10 +1121,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x4e, %xmm2, %xmm7 @@ -1167,10 +1167,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1178,10 +1178,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm7 @@ -1196,10 +1196,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1259,7 +1259,7 @@ addq $2 * SIZE, CO2 ALIGN_4 -.L79: +L(79): #if defined(TRMMKERNEL) && !defined(LEFT) addq $1, KK #endif @@ -1268,7 +1268,7 @@ movq BO, B ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm_kernel_2x2_piledriver.S b/kernel/x86_64/zgemm_kernel_2x2_piledriver.S index bffe5439db..16b7af9be6 100644 --- a/kernel/x86_64/zgemm_kernel_2x2_piledriver.S +++ b/kernel/x86_64/zgemm_kernel_2x2_piledriver.S @@ -476,13 +476,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $0, OLD_M - je .L999 + je L(999) cmpq $0, OLD_N - je .L999 + je L(999) cmpq $0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -510,23 +510,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif #endif -.L2_0: +L(2_0): movq Ndiv6, J cmpq $0, J - je .L1_0 + je L(1_0) ALIGN_4 -.L2_01: +L(2_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_02b: +L(2_02b): vmovups (BO1), %xmm0 vmovups 2 * SIZE(BO1), %xmm1 @@ -535,13 +535,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $4*SIZE,BO1 addq $4*SIZE,BO decq %rax - jnz .L2_02b + jnz L(2_02b) -.L2_02c: +L(2_02c): movq BO1, B // next offset of B -.L2_10: +L(2_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -555,11 +555,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $1, I // i = (m >> 1) - je .L2_40 + je L(2_40) ALIGN_4 -.L2_11: +L(2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -597,7 +597,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax // K = K - ( K % 8 ) - je .L2_16 + je L(2_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -608,7 +608,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_12: +L(2_12): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -624,7 +624,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_16 + je L(2_16) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x2_1(xxx) @@ -640,12 +640,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_3(xxx) KERNEL2x2_4(xxx) - je .L2_16 + je L(2_16) - jmp .L2_12 + jmp L(2_12) ALIGN_4 -.L2_16: +L(2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -653,7 +653,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_19 + je L(2_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -665,14 +665,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_17: +L(2_17): KERNEL2x2_SUB(xxx) - jl .L2_17 + jl L(2_17) ALIGN_4 -.L2_19: +L(2_19): vmovddup ALPHA_R, %xmm0 vmovddup ALPHA_I, %xmm1 @@ -768,20 +768,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L2_11 + jg L(2_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L2_40: +L(2_40): testq $1, M - jz .L2_60 // to next 2 lines of N + jz L(2_60) // to next 2 lines of N ALIGN_4 -.L2_41: +L(2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -819,7 +819,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax // K = K - ( K % 8 ) - je .L2_46 + je L(2_46) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -830,7 +830,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_42: +L(2_42): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x2_1(xxx) @@ -846,7 +846,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x2_1(xxx) @@ -862,12 +862,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_3(xxx) KERNEL1x2_4(xxx) - je .L2_46 + je L(2_46) - jmp .L2_42 + jmp L(2_42) ALIGN_4 -.L2_46: +L(2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -875,7 +875,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L2_49 + je L(2_49) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -887,14 +887,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_47: +L(2_47): KERNEL1x2_SUB(xxx) - jl .L2_47 + jl L(2_47) ALIGN_4 -.L2_49: +L(2_49): vmovddup ALPHA_R, %xmm0 vmovddup ALPHA_I, %xmm1 @@ -970,17 +970,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L2_60: +L(2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif decq J // j -- - jg .L2_01 // next 2 lines of N + jg L(2_01) // next 2 lines of N -.L1_0: +L(1_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -988,30 +988,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_01: +L(1_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_02b: +L(1_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $2*SIZE,BO1 addq $2*SIZE,BO decq %rax - jnz .L1_02b + jnz L(1_02b) -.L1_02c: +L(1_02c): movq BO1, B // next offset of B -.L1_10: +L(1_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -1025,11 +1025,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $1, I // i = (m >> 1) - je .L1_40 + je L(1_40) ALIGN_4 -.L1_11: +L(1_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1067,7 +1067,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax // K = K - ( K % 8 ) - je .L1_16 + je L(1_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1078,7 +1078,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_12: +L(1_12): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -1092,7 +1092,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_16 + je L(1_16) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_1(xxx) @@ -1106,12 +1106,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_3(xxx) KERNEL2x1_4(xxx) - je .L1_16 + je L(1_16) - jmp .L1_12 + jmp L(1_12) ALIGN_4 -.L1_16: +L(1_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -1119,7 +1119,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_19 + je L(1_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1131,14 +1131,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_17: +L(1_17): KERNEL2x1_SUB(xxx) - jl .L1_17 + jl L(1_17) ALIGN_4 -.L1_19: +L(1_19): vmovddup ALPHA_R, %xmm0 vmovddup ALPHA_I, %xmm1 @@ -1211,20 +1211,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L1_11 + jg L(1_11) ALIGN_4 /************************************************************************** * Rest of M ***************************************************************************/ -.L1_40: +L(1_40): testq $1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_41: +L(1_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1262,7 +1262,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $-8, %rax // K = K - ( K % 8 ) - je .L1_46 + je L(1_46) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1273,7 +1273,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_42: +L(1_42): prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x1_1(xxx) @@ -1287,7 +1287,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL1x1_1(xxx) @@ -1301,12 +1301,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_3(xxx) KERNEL1x1_4(xxx) - je .L1_46 + je L(1_46) - jmp .L1_42 + jmp L(1_42) ALIGN_4 -.L1_46: +L(1_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -1314,7 +1314,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $7, %rax # if (k & 1) - je .L1_49 + je L(1_49) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -1326,14 +1326,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_47: +L(1_47): KERNEL1x1_SUB(xxx) - jl .L1_47 + jl L(1_47) ALIGN_4 -.L1_49: +L(1_49): vmovddup ALPHA_R, %xmm0 vmovddup ALPHA_I, %xmm1 @@ -1397,7 +1397,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L999: +L(999): vzeroupper movq SP, %rsp diff --git a/kernel/x86_64/zgemm_kernel_2x2_sse2.S b/kernel/x86_64/zgemm_kernel_2x2_sse2.S index ae7bb8faff..fcc6903009 100644 --- a/kernel/x86_64/zgemm_kernel_2x2_sse2.S +++ b/kernel/x86_64/zgemm_kernel_2x2_sse2.S @@ -417,10 +417,10 @@ movq N, J sarq $1, J # j = (n >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L01: +L(01): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc @@ -433,10 +433,10 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): PREFETCH (RPREFETCHSIZE + 0) * SIZE(B) movq 0 * SIZE(B), %mm0 @@ -508,17 +508,17 @@ addq $ 32 * SIZE, BO subq $-16 * SIZE, B decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L05 + jle L(05) ALIGN_4 -.L04: +L(04): movq 0 * SIZE(B), %mm0 movq %mm0, -16 * SIZE(BO) movq %mm0, -15 * SIZE(BO) @@ -537,20 +537,20 @@ addq $ 8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L05: +L(05): movq A, AO # aoffset = a leaq (RPREFETCHSIZE + 0) * SIZE(B), BB movq M, I sarq $1, I # i = (m >> 2) - jle .L30 + jle L(30) ALIGN_4 -.L10: +L(10): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -608,10 +608,10 @@ leaq (BO, %rax, 8), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -632,7 +632,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -653,7 +653,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -674,7 +674,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -695,7 +695,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -716,7 +716,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -737,7 +737,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -758,7 +758,7 @@ addq $8 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -779,10 +779,10 @@ addq $8 * SIZE, %rax BRANCH - jl .L12 + jl L(12) ALIGN_3 -.L15: +L(15): PREFETCH 8 * SIZE(BB) subq $-16 * SIZE, BB @@ -792,7 +792,7 @@ movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_3 @@ -811,10 +811,10 @@ #else sarq $2, %rax NOBRANCH - jle .L16 + jle L(16) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -828,10 +828,10 @@ subq $-16 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) #endif -.L16: +L(16): movapd POSINV, %xmm5 movapd ALPHA_R, %xmm6 movapd ALPHA_I, %xmm7 @@ -842,7 +842,7 @@ movq KKK, %rax #endif andq $3, %rax # if (k & 1) - je .L19 + je L(19) leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO @@ -850,7 +850,7 @@ negq %rax ALIGN_3 -.L17: +L(17): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movapd -14 * SIZE(BO, %rax, 8), %xmm1 @@ -877,10 +877,10 @@ movapd -10 * SIZE(AO, %rax, 4), %xmm2 addq $SIZE, %rax - jl .L17 + jl L(17) ALIGN_3 -.L19: +L(19): #ifndef TRMMKERNEL movlpd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 @@ -977,12 +977,12 @@ addq $4 * SIZE, CO1 # coffset += 4 addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L30: +L(30): testq $1, M - jle .L99 + jle L(99) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1022,10 +1022,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L44 + je L(44) ALIGN_4 -.L41: +L(41): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movapd -14 * SIZE(BO), %xmm1 @@ -1128,10 +1128,10 @@ subq $-16 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L41 + jne L(41) ALIGN_4 -.L44: +L(44): #ifndef TRMMKERNEL movq K, %rax #else @@ -1139,7 +1139,7 @@ #endif andq $4, %rax BRANCH - jle .L45 + jle L(45) mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 @@ -1194,7 +1194,7 @@ addq $32 * SIZE, BO ALIGN_4 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1205,10 +1205,10 @@ movapd ALPHA_I, %xmm7 andq $3, %rax # if (k & 1) BRANCH - jle .L47 + jle L(47) ALIGN_4 -.L46: +L(46): mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 movapd -14 * SIZE(BO), %xmm1 @@ -1226,10 +1226,10 @@ addq $8 * SIZE, BO decq %rax - jg .L46 + jg L(46) ALIGN_4 -.L47: +L(47): #ifndef TRMMKERNEL movlpd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 @@ -1293,20 +1293,20 @@ #endif ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif leaq (C, LDC, 2), C # c += 2 * ldc decq J # j -- - jg .L01 + jg L(01) -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) -.L101: +L(101): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1317,10 +1317,10 @@ movq K, %rax sarq $2, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movlpd 0 * SIZE(B), %xmm8 movlpd 1 * SIZE(B), %xmm9 movlpd 2 * SIZE(B), %xmm10 @@ -1350,17 +1350,17 @@ subq $-16 * SIZE, BO addq $ 8 * SIZE, B decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): movq K, %rax andq $3, %rax BRANCH - jle .L105 + jle L(105) ALIGN_4 -.L104: +L(104): movlpd 0 * SIZE(B), %xmm8 movlpd 1 * SIZE(B), %xmm9 @@ -1372,19 +1372,19 @@ addq $4 * SIZE, BO addq $2 * SIZE, B decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L105: +L(105): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $1, I # i = (m >> 2) - jle .L130 + jle L(130) ALIGN_4 -.L110: +L(110): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1424,9 +1424,9 @@ movq %rax, KKK #endif sarq $2, %rax - je .L112 + je L(112) -.L111: +L(111): mulpd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd -14 * SIZE(BO), %xmm0 @@ -1481,10 +1481,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO decq %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #ifndef TRMMKERNEL movq K, %rax #else @@ -1495,9 +1495,9 @@ movapd ALPHA_I, %xmm7 andq $3, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO), %xmm0 addpd %xmm1, %xmm8 @@ -1514,10 +1514,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): #ifndef TRMMKERNEL movlpd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 @@ -1582,12 +1582,12 @@ addq $4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L130: +L(130): testq $1, M - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1628,10 +1628,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L144 + je L(144) ALIGN_4 -.L141: +L(141): mulpd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd -14 * SIZE(BO), %xmm0 @@ -1686,11 +1686,11 @@ subq $-16 * SIZE, AO subq $-32 * SIZE, BO decq %rax - jne .L141 + jne L(141) ALIGN_4 -.L144: +L(144): #ifndef TRMMKERNEL movq K, %rax #else @@ -1698,7 +1698,7 @@ #endif andq $4, %rax # if (k & 1) BRANCH - jle .L145 + jle L(145) mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO), %xmm0 @@ -1728,7 +1728,7 @@ subq $-16 * SIZE, BO ALIGN_4 -.L145: +L(145): movapd POSINV, %xmm5 movapd ALPHA_R, %xmm6 movapd ALPHA_I, %xmm7 @@ -1740,10 +1740,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - jle .L148 + jle L(148) ALIGN_4 -.L146: +L(146): mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BO), %xmm0 addpd %xmm1, %xmm8 @@ -1754,10 +1754,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L146 + jg L(146) ALIGN_4 -.L148: +L(148): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1797,7 +1797,7 @@ movhpd %xmm8, 1 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp EMMS diff --git a/kernel/x86_64/zgemm_kernel_2x2_sse3.S b/kernel/x86_64/zgemm_kernel_2x2_sse3.S index b78f1a902b..e6a8f51ff6 100644 --- a/kernel/x86_64/zgemm_kernel_2x2_sse3.S +++ b/kernel/x86_64/zgemm_kernel_2x2_sse3.S @@ -409,10 +409,10 @@ salq $ZBASE_SHIFT, LDC movq N, J sarq $1, J # j = (n >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -428,10 +428,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L30 + jle L(30) ALIGN_4 -.L10: +L(10): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -486,9 +486,9 @@ andq $-8, %rax salq $4, %rax - je .L12 + je L(12) -.L1X: +L(1X): KERNEL1 (16 * 0) KERNEL2 (16 * 0) KERNEL3 (16 * 0) @@ -507,7 +507,7 @@ KERNEL16(16 * 0) cmpq $128 * 1, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 1) KERNEL2 (16 * 1) KERNEL3 (16 * 1) @@ -526,7 +526,7 @@ KERNEL16(16 * 1) cmpq $128 * 2, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 2) KERNEL2 (16 * 2) KERNEL3 (16 * 2) @@ -545,7 +545,7 @@ KERNEL16(16 * 2) cmpq $128 * 3, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 3) KERNEL2 (16 * 3) KERNEL3 (16 * 3) @@ -564,7 +564,7 @@ KERNEL16(16 * 3) cmpq $128 * 4, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 4) KERNEL2 (16 * 4) KERNEL3 (16 * 4) @@ -583,7 +583,7 @@ KERNEL16(16 * 4) cmpq $128 * 5, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 5) KERNEL2 (16 * 5) KERNEL3 (16 * 5) @@ -602,7 +602,7 @@ KERNEL16(16 * 5) cmpq $128 * 6, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 6) KERNEL2 (16 * 6) KERNEL3 (16 * 6) @@ -621,7 +621,7 @@ KERNEL16(16 * 6) cmpq $128 * 7, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 7) KERNEL2 (16 * 7) KERNEL3 (16 * 7) @@ -642,14 +642,14 @@ addq $32 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $128 * 8, %rax - jg .L1X + jg L(1X) -.L11: +L(11): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 2), BO # * 64 ALIGN_4 -.L12: +L(12): #ifndef TRMMKERNEL movq K, %rax #else @@ -659,10 +659,10 @@ movddup ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L14 + je L(14) ALIGN_4 -.L13: +L(13): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 ADD1 %xmm9, %xmm0 @@ -693,10 +693,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm3, %xmm3 SHUFPD_1 %xmm5, %xmm5 @@ -791,14 +791,14 @@ addq $4 * SIZE, CO1 # coffset += 4 addq $4 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L30: +L(30): testq $1, M - jle .L99 + jle L(99) -.L40: +L(40): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -836,9 +836,9 @@ movq %rax, KKK #endif sarq $3, %rax - je .L42 + je L(42) -.L41: +L(41): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -948,9 +948,9 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L41 + jne L(41) -.L42: +L(42): #ifndef TRMMKERNEL movq K, %rax #else @@ -960,9 +960,9 @@ movddup ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - jle .L44 + jle L(44) -.L43: +L(43): mulpd %xmm8, %xmm9 ADD1 %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -980,10 +980,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm3, %xmm3 @@ -1047,7 +1047,7 @@ #endif ALIGN_4 -.L99: +L(99): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -1055,13 +1055,13 @@ leaq (C, LDC, 2), C # c += 2 * ldc movq BO, B decq J # j -- - jg .L01 + jg L(01) -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) -.L101: +L(101): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1072,10 +1072,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L130 + jle L(130) ALIGN_4 -.L110: +L(110): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1115,9 +1115,9 @@ movq %rax, KKK #endif sarq $3, %rax - je .L112 + je L(112) -.L111: +L(111): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -1236,10 +1236,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #ifndef TRMMKERNEL movq K, %rax #else @@ -1249,9 +1249,9 @@ movddup ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 ADD1 %xmm9, %xmm0 @@ -1270,10 +1270,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm5, %xmm5 @@ -1337,15 +1337,15 @@ addq $4 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L130: +L(130): testq $1, M - jle .L999 + jle L(999) ALIGN_4 -.L140: +L(140): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1383,9 +1383,9 @@ movq %rax, KKK #endif sarq $3, %rax - je .L142 + je L(142) -.L141: +L(141): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -1447,9 +1447,9 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L141 + jne L(141) -.L142: +L(142): #ifndef TRMMKERNEL movq K, %rax #else @@ -1459,9 +1459,9 @@ movddup ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - jle .L144 + jle L(144) -.L143: +L(143): mulpd %xmm8, %xmm9 ADD1 %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -1473,10 +1473,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L143 + jg L(143) ALIGN_4 -.L144: +L(144): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -1510,7 +1510,7 @@ movhpd %xmm0, 1 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm_kernel_2x4_nehalem.S b/kernel/x86_64/zgemm_kernel_2x4_nehalem.S index ce60123884..b57f256b94 100644 --- a/kernel/x86_64/zgemm_kernel_2x4_nehalem.S +++ b/kernel/x86_64/zgemm_kernel_2x4_nehalem.S @@ -184,10 +184,10 @@ movq N, J sarq $2, J NOBRANCH - jle .L30 + jle L(30) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -204,10 +204,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): prefetcht2 -32 * SIZE(BB) subq $-16 * SIZE, BB @@ -263,10 +263,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm12 @@ -389,10 +389,10 @@ subq $-16 * SIZE, AO subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else @@ -400,10 +400,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm1, %xmm12 movaps -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm13 @@ -439,10 +439,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): ADD1 %xmm1, %xmm12 ADD2 %xmm2, %xmm13 ADD1 %xmm3, %xmm14 @@ -546,13 +546,13 @@ addq $4 * SIZE, CO2 decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M BRANCH - jle .L29 + jle L(29) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -597,10 +597,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -670,10 +670,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -681,10 +681,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -706,10 +706,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): ADD1 %xmm1, %xmm8 ADD2 %xmm2, %xmm9 ADD1 %xmm3, %xmm10 @@ -785,7 +785,7 @@ #endif ALIGN_4 -.L29: +L(29): #if defined(TRMMKERNEL) && !defined(LEFT) addq $4, KK #endif @@ -795,13 +795,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L30: +L(30): testq $2, N BRANCH - jle .L50 + jle L(50) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -815,10 +815,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L31: +L(31): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -864,10 +864,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -931,10 +931,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -942,10 +942,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): ADD1 %xmm1, %xmm8 movaps -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -965,10 +965,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): ADD1 %xmm1, %xmm8 ADD2 %xmm2, %xmm9 ADD1 %xmm3, %xmm10 @@ -1047,13 +1047,13 @@ addq $4 * SIZE, CO2 decq I # i -- BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L40: +L(40): testq $1, M BRANCH - jle .L49 + jle L(49) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1098,10 +1098,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1143,10 +1143,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1154,10 +1154,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1172,10 +1172,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): ADD1 %xmm1, %xmm8 ADD2 %xmm2, %xmm9 @@ -1232,7 +1232,7 @@ #endif ALIGN_4 -.L49: +L(49): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -1241,10 +1241,10 @@ movq BO, B ALIGN_4 -.L50: +L(50): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1257,10 +1257,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1305,10 +1305,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -1348,10 +1348,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1359,10 +1359,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): ADD1 %xmm1, %xmm8 movddup -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -1376,10 +1376,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): ADD1 %xmm1, %xmm8 ADD2 %xmm2, %xmm9 @@ -1435,13 +1435,13 @@ addq $4 * SIZE, CO1 decq I # i -- BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1481,10 +1481,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1526,10 +1526,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1537,10 +1537,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1555,10 +1555,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): ADD1 %xmm1, %xmm8 ADD2 %xmm2, %xmm9 @@ -1599,7 +1599,7 @@ movsd %xmm8, (CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm_kernel_4x2_barcelona.S b/kernel/x86_64/zgemm_kernel_4x2_barcelona.S index 06d0bbe14b..2d66045ebd 100644 --- a/kernel/x86_64/zgemm_kernel_4x2_barcelona.S +++ b/kernel/x86_64/zgemm_kernel_4x2_barcelona.S @@ -502,10 +502,10 @@ salq $ZBASE_SHIFT, LDC movq N, J sarq $1, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -517,12 +517,12 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): prefetch (RPREFETCHSIZE + 0) * SIZE(B) movaps 0 * SIZE(B), %xmm3 @@ -568,17 +568,17 @@ addq $32 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movaps 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -603,10 +603,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a @@ -615,10 +615,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -677,10 +677,10 @@ leaq (BO, %rax, 8), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_3 -.L12: +L(12): KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -690,7 +690,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -700,7 +700,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -710,7 +710,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -720,7 +720,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -730,7 +730,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -740,7 +740,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -750,7 +750,7 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) NOBRANCH - je .L15 + je L(15) KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -760,17 +760,17 @@ KERNEL7(32 * 0) KERNEL8(32 * 0) BRANCH - jl .L12 + jl L(12) ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_3 @@ -783,7 +783,7 @@ addq $32 * SIZE, AO ALIGN_3 -.L16: +L(16): #ifndef TRMMKERNEL movq K, %rax #else @@ -793,7 +793,7 @@ movaps ALPHA_I, %xmm7 andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) leaq (, %rax, 8), %rax leaq (AO, %rax, 4), AO @@ -801,7 +801,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulps %xmm1, %xmm0 mulps -28 * SIZE(AO, %rax, 4), %xmm1 addps %xmm0, %xmm8 @@ -828,10 +828,10 @@ movaps %xmm0, %xmm2 addq $SIZE * 2, %rax - jl .L17 + jl L(17) ALIGN_4 -.L18: +L(18): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -920,12 +920,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -971,10 +971,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -1085,10 +1085,10 @@ subq $-128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1098,10 +1098,10 @@ movaps ALPHA_I, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm0, %xmm1 addps %xmm1, %xmm8 movaps -28 * SIZE(BO), %xmm1 @@ -1118,10 +1118,10 @@ subq $- 4 * SIZE, AO subq $-16 * SIZE, BO decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -1184,9 +1184,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1230,10 +1230,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -1351,10 +1351,10 @@ subq $-128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1364,10 +1364,10 @@ movaps ALPHA_I, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm0, %xmm1 addps %xmm1, %xmm8 movsd -28 * SIZE(BO), %xmm1 @@ -1385,10 +1385,10 @@ subq $ -2 * SIZE, AO subq $-16 * SIZE, BO decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movsd 0 * SIZE(CO2), %xmm1 @@ -1442,22 +1442,22 @@ #endif ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif leaq (C, LDC, 2), C # c += 2 * ldc decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L41: +L(41): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1469,10 +1469,10 @@ movq K, %rax sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): prefetch (RPREFETCHSIZE + 0) * SIZE(B) movaps 0 * SIZE(B), %xmm3 @@ -1518,17 +1518,17 @@ addq $32 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): movq K, %rax andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movsd 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -1547,19 +1547,19 @@ addq $2 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1610,10 +1610,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulps %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulps -28 * SIZE(BO), %xmm0 @@ -1729,10 +1729,10 @@ subq $-64 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1742,10 +1742,10 @@ movaps ALPHA_I, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): mulps %xmm0, %xmm1 mulps -28 * SIZE(BO), %xmm0 addps %xmm1, %xmm8 @@ -1762,10 +1762,10 @@ addq $ 8 * SIZE, AO addq $ 8 * SIZE, BO decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -1824,12 +1824,12 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1873,10 +1873,10 @@ #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulps -28 * SIZE(BO), %xmm0 @@ -1936,10 +1936,10 @@ subq $-64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1949,10 +1949,10 @@ movaps ALPHA_I, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm0, %xmm1 mulps -28 * SIZE(BO), %xmm0 addps %xmm1, %xmm8 @@ -1963,10 +1963,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -2016,9 +2016,9 @@ addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L70: +L(70): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2061,10 +2061,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm1, %xmm8 @@ -2134,10 +2134,10 @@ subq $-64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2147,10 +2147,10 @@ movaps ALPHA_I, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm0, %xmm1 addps %xmm1, %xmm8 movsd -28 * SIZE(BO), %xmm1 @@ -2162,10 +2162,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 #endif @@ -2196,7 +2196,7 @@ movsd %xmm8, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/zgemm_kernel_4x2_core2.S b/kernel/x86_64/zgemm_kernel_4x2_core2.S index bb4584a6b7..ac60c97949 100644 --- a/kernel/x86_64/zgemm_kernel_4x2_core2.S +++ b/kernel/x86_64/zgemm_kernel_4x2_core2.S @@ -180,10 +180,10 @@ salq $ZBASE_SHIFT, LDC movq N, J sarq $1, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -196,10 +196,10 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): prefetcht0 (PREFETCH_R + 0) * SIZE(B) movaps -28 * SIZE(B), %xmm7 movaps -24 * SIZE(B), %xmm11 @@ -252,17 +252,17 @@ subq $-16 * SIZE, B subq $-64 * SIZE, BO subq $1, %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): pshufd $0x00, %xmm3, %xmm0 movaps %xmm0, -32 * SIZE(BO) pshufd $0x55, %xmm3, %xmm1 @@ -277,10 +277,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO subq $1, %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): leaq (PREFETCH_R + 0) * SIZE(B), BB movq C, CO1 @@ -289,10 +289,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -348,10 +348,10 @@ movq %rax, KKK #endif sarq $2, %rax - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): addps %xmm2, %xmm10 movaps -32 * SIZE(BO), %xmm2 addps %xmm3, %xmm14 @@ -480,10 +480,10 @@ subq $-64 * SIZE, BO subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): prefetcht2 -16 * SIZE(BB) #ifndef TRMMKERNEL @@ -493,10 +493,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): addps %xmm2, %xmm10 movaps -32 * SIZE(BO), %xmm2 addps %xmm3, %xmm14 @@ -530,10 +530,10 @@ movaps -28 * SIZE(AO), %xmm1 subq $1, %rax - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): movaps ALPHA_R, %xmm6 movaps ALPHA_I, %xmm7 @@ -646,12 +646,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -687,10 +687,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -760,10 +760,10 @@ subq $-16 * SIZE, AO subq $-64 * SIZE, BO subq $1, %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -771,10 +771,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): movaps -32 * SIZE(AO), %xmm0 movaps -32 * SIZE(BO), %xmm2 movaps -28 * SIZE(BO), %xmm3 @@ -794,10 +794,10 @@ addq $ 4 * SIZE, AO addq $16 * SIZE, BO subq $1, %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): movaps ALPHA_R, %xmm6 movaps ALPHA_I, %xmm7 @@ -869,9 +869,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -907,10 +907,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -32 * SIZE(AO), %xmm0 @@ -980,10 +980,10 @@ subq $ -8 * SIZE, AO subq $-64 * SIZE, BO subq $1, %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -991,10 +991,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): movsd -32 * SIZE(AO), %xmm0 movsd -32 * SIZE(BO), %xmm2 movsd -28 * SIZE(BO), %xmm3 @@ -1014,10 +1014,10 @@ addq $ 2 * SIZE, AO addq $16 * SIZE, BO subq $1, %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): movaps ALPHA_R, %xmm6 movaps ALPHA_I, %xmm7 @@ -1082,22 +1082,22 @@ #endif ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif leaq (C, LDC, 2), C # c += 2 * ldc decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L41: +L(41): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1108,10 +1108,10 @@ movq K, %rax sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): movss -32 * SIZE(B), %xmm8 movss -31 * SIZE(B), %xmm9 movss -30 * SIZE(B), %xmm10 @@ -1143,17 +1143,17 @@ addq $32 * SIZE, BO subq $1, %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): movq K, %rax andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movss -32 * SIZE(B), %xmm8 movss -31 * SIZE(B), %xmm9 @@ -1166,19 +1166,19 @@ addq $2 * SIZE, B addq $8 * SIZE, BO subq $1, %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1214,10 +1214,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -1295,10 +1295,10 @@ subq $-32 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1306,10 +1306,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): movaps -32 * SIZE(AO), %xmm0 movaps -28 * SIZE(AO), %xmm1 @@ -1331,10 +1331,10 @@ addq $8 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): movaps ALPHA_R, %xmm6 movaps ALPHA_I, %xmm7 @@ -1404,12 +1404,12 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1445,10 +1445,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps -32 * SIZE(AO), %xmm0 @@ -1488,10 +1488,10 @@ subq $-16 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1499,10 +1499,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): movaps -32 * SIZE(AO), %xmm0 movaps -32 * SIZE(BO), %xmm2 movaps -28 * SIZE(BO), %xmm3 @@ -1516,10 +1516,10 @@ addq $4 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): movaps ALPHA_R, %xmm6 movaps ALPHA_I, %xmm7 @@ -1568,9 +1568,9 @@ addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L70: +L(70): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1606,10 +1606,10 @@ movq %rax, KKK #endif sarq $2, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movsd -32 * SIZE(AO), %xmm0 @@ -1649,10 +1649,10 @@ subq $ -8 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -1660,10 +1660,10 @@ #endif andq $3, %rax BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): movsd -32 * SIZE(AO), %xmm0 movsd -32 * SIZE(BO), %xmm2 movsd -28 * SIZE(BO), %xmm3 @@ -1677,10 +1677,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): movaps ALPHA_R, %xmm6 movaps ALPHA_I, %xmm7 @@ -1713,7 +1713,7 @@ movsd %xmm8, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/zgemm_kernel_4x2_haswell.S b/kernel/x86_64/zgemm_kernel_4x2_haswell.S index 29729b1017..a8712730cf 100644 --- a/kernel/x86_64/zgemm_kernel_4x2_haswell.S +++ b/kernel/x86_64/zgemm_kernel_4x2_haswell.S @@ -1107,13 +1107,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $ 0, OLD_M - je .L999 + je L(999) cmpq $ 0, OLD_N - je .L999 + je L(999) cmpq $ 0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -1134,16 +1134,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /************************************************************************************************/ -.L6_00_0: +L(6_00_0): movq Ndiv6, J cmpq $ 0, J - je .L2_00_0 + je L(2_00_0) ALIGN_4 -.L6_00_01: +L(6_00_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO @@ -1154,7 +1154,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax ALIGN_4 -.L6_00_02b: +L(6_00_02b): vmovups (BO1), %xmm0 vmovups 2 * SIZE(BO1), %xmm1 @@ -1166,13 +1166,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4*SIZE,BO2 addq $ 6*SIZE,BO decq %rax - jnz .L6_00_02b + jnz L(6_00_02b) -.L6_00_02c: +L(6_00_02c): -.L6_00_10: +L(6_00_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc leaq (C, LDC, 1), C // c += 1 * ldc @@ -1181,13 +1181,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 2, I // i = (m >> 2) - je .L6_2_10 + je L(6_2_10) ALIGN_4 /******************************************************************************************************************/ -.L6_4_11: +L(6_4_11): leaq BUFFER1, BO // first buffer to BO @@ -1196,10 +1196,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L6_4_16 + je L(6_4_16) ALIGN_4 -.L6_4_12: +L(6_4_12): KERNEL4x3_SUB KERNEL4x3_SUB @@ -1211,7 +1211,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_SUB KERNEL4x3_SUB - je .L6_4_16 + je L(6_4_16) KERNEL4x3_SUB KERNEL4x3_SUB @@ -1223,33 +1223,33 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_SUB KERNEL4x3_SUB - je .L6_4_16 + je L(6_4_16) - jmp .L6_4_12 + jmp L(6_4_12) ALIGN_4 -.L6_4_16: +L(6_4_16): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L6_4_19 + je L(6_4_19) ALIGN_4 -.L6_4_17: +L(6_4_17): KERNEL4x3_SUB - jnz .L6_4_17 + jnz L(6_4_17) ALIGN_4 -.L6_4_19: +L(6_4_19): SAVE4x3 addq $ 8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L6_4_11 + jg L(6_4_11) ALIGN_4 @@ -1259,11 +1259,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************************************************/ -.L6_2_10: +L(6_2_10): testq $ 2, M - jz .L6_2_40 // to next 2 lines of N + jz L(6_2_40) // to next 2 lines of N -.L6_2_11: +L(6_2_11): leaq BUFFER1, BO // first buffer to BO @@ -1272,10 +1272,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L6_2_16 + je L(6_2_16) ALIGN_4 -.L6_2_12: +L(6_2_12): KERNEL2x3_SUB KERNEL2x3_SUB @@ -1287,7 +1287,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_SUB KERNEL2x3_SUB - je .L6_2_16 + je L(6_2_16) KERNEL2x3_SUB KERNEL2x3_SUB @@ -1299,27 +1299,27 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_SUB KERNEL2x3_SUB - je .L6_2_16 + je L(6_2_16) - jmp .L6_2_12 + jmp L(6_2_12) ALIGN_4 -.L6_2_16: +L(6_2_16): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L6_2_19 + je L(6_2_19) ALIGN_4 -.L6_2_17: +L(6_2_17): KERNEL2x3_SUB - jnz .L6_2_17 + jnz L(6_2_17) ALIGN_4 -.L6_2_19: +L(6_2_19): SAVE2x3 @@ -1330,13 +1330,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /************************************************************************** * Rest of M ***************************************************************************/ -.L6_2_40: +L(6_2_40): testq $ 1, M - jz .L6_2_60 // to next 2 lines of N + jz L(6_2_60) // to next 2 lines of N ALIGN_4 -.L6_2_41: +L(6_2_41): leaq BUFFER1, BO // first buffer to BO @@ -1345,11 +1345,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L6_2_46 + je L(6_2_46) ALIGN_4 -.L6_2_42: +L(6_2_42): KERNEL1x3_SUB KERNEL1x3_SUB @@ -1361,7 +1361,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_SUB KERNEL1x3_SUB - je .L6_2_46 + je L(6_2_46) KERNEL1x3_SUB KERNEL1x3_SUB @@ -1373,40 +1373,40 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_SUB KERNEL1x3_SUB - je .L6_2_46 + je L(6_2_46) - jmp .L6_2_42 + jmp L(6_2_42) ALIGN_4 -.L6_2_46: +L(6_2_46): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L6_2_49 + je L(6_2_49) ALIGN_4 -.L6_2_47: +L(6_2_47): KERNEL1x3_SUB - jnz .L6_2_47 + jnz L(6_2_47) ALIGN_4 -.L6_2_49: +L(6_2_49): SAVE1x3 addq $ 2 * SIZE, CO1 # coffset += 2 decq I # i -- - jg .L6_2_41 + jg L(6_2_41) ALIGN_4 -.L6_2_60: +L(6_2_60): /************************************************************************************************/ @@ -1414,7 +1414,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /************************************************************************************************/ -.L7_00_01: +L(7_00_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO @@ -1424,7 +1424,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax ALIGN_4 -.L7_00_02b: +L(7_00_02b): vmovups 2 * SIZE(BO1), %xmm0 vmovups (BO2), %xmm1 @@ -1436,14 +1436,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4*SIZE,BO2 addq $ 6*SIZE,BO decq %rax - jnz .L7_00_02b + jnz L(7_00_02b) -.L7_00_02c: +L(7_00_02c): movq BO2, B // next offset of B -.L7_00_10: +L(7_00_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc leaq (C, LDC, 1), C // c += 1 * ldc @@ -1452,13 +1452,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 2, I // i = (m >> 2) - je .L7_2_10 + je L(7_2_10) ALIGN_4 /******************************************************************************************************************/ -.L7_4_11: +L(7_4_11): leaq BUFFER1, BO // first buffer to BO @@ -1467,10 +1467,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L7_4_16 + je L(7_4_16) ALIGN_4 -.L7_4_12: +L(7_4_12): KERNEL4x3_SUB KERNEL4x3_SUB @@ -1482,7 +1482,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_SUB KERNEL4x3_SUB - je .L7_4_16 + je L(7_4_16) KERNEL4x3_SUB KERNEL4x3_SUB @@ -1494,34 +1494,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x3_SUB KERNEL4x3_SUB - je .L7_4_16 + je L(7_4_16) - jmp .L7_4_12 + jmp L(7_4_12) ALIGN_4 -.L7_4_16: +L(7_4_16): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L7_4_19 + je L(7_4_19) ALIGN_4 -.L7_4_17: +L(7_4_17): KERNEL4x3_SUB - jnz .L7_4_17 + jnz L(7_4_17) ALIGN_4 -.L7_4_19: +L(7_4_19): SAVE4x3 addq $ 8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L7_4_11 + jg L(7_4_11) ALIGN_4 @@ -1531,11 +1531,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************************************************/ -.L7_2_10: +L(7_2_10): testq $ 2, M - jz .L7_2_40 // to next 2 lines of N + jz L(7_2_40) // to next 2 lines of N -.L7_2_11: +L(7_2_11): leaq BUFFER1, BO // first buffer to BO @@ -1544,10 +1544,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L7_2_16 + je L(7_2_16) ALIGN_4 -.L7_2_12: +L(7_2_12): KERNEL2x3_SUB KERNEL2x3_SUB @@ -1559,7 +1559,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_SUB KERNEL2x3_SUB - je .L7_2_16 + je L(7_2_16) KERNEL2x3_SUB KERNEL2x3_SUB @@ -1571,28 +1571,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x3_SUB KERNEL2x3_SUB - je .L7_2_16 + je L(7_2_16) - jmp .L7_2_12 + jmp L(7_2_12) ALIGN_4 -.L7_2_16: +L(7_2_16): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L7_2_19 + je L(7_2_19) ALIGN_4 -.L7_2_17: +L(7_2_17): KERNEL2x3_SUB - jnz .L7_2_17 + jnz L(7_2_17) ALIGN_4 -.L7_2_19: +L(7_2_19): SAVE2x3 @@ -1603,13 +1603,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /************************************************************************** * Rest of M ***************************************************************************/ -.L7_2_40: +L(7_2_40): testq $ 1, M - jz .L7_2_60 // to next 2 lines of N + jz L(7_2_60) // to next 2 lines of N ALIGN_4 -.L7_2_41: +L(7_2_41): leaq BUFFER1, BO // first buffer to BO @@ -1618,11 +1618,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq K, %rax andq $ -8, %rax // K = K - ( K % 8 ) - je .L7_2_46 + je L(7_2_46) ALIGN_4 -.L7_2_42: +L(7_2_42): KERNEL1x3_SUB KERNEL1x3_SUB @@ -1634,7 +1634,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_SUB KERNEL1x3_SUB - je .L7_2_46 + je L(7_2_46) KERNEL1x3_SUB KERNEL1x3_SUB @@ -1646,66 +1646,66 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x3_SUB KERNEL1x3_SUB - je .L7_2_46 + je L(7_2_46) - jmp .L7_2_42 + jmp L(7_2_42) ALIGN_4 -.L7_2_46: +L(7_2_46): movq K, %rax andq $ 7, %rax # if (k & 1) - je .L7_2_49 + je L(7_2_49) ALIGN_4 -.L7_2_47: +L(7_2_47): KERNEL1x3_SUB - jnz .L7_2_47 + jnz L(7_2_47) ALIGN_4 -.L7_2_49: +L(7_2_49): SAVE1x3 addq $ 2 * SIZE, CO1 # coffset += 2 decq I # i -- - jg .L7_2_41 + jg L(7_2_41) ALIGN_4 -.L7_2_60: +L(7_2_60): decq J // j -- - jg .L6_00_01 // next 6 lines of N + jg L(6_00_01) // next 6 lines of N /************************************************************************************************/ /************************************************************************************************/ -.L2_00_0: +L(2_00_0): movq Nmod6, J sarq $1, J // j = j / 2 cmpq $ 0, J - je .L1_2_0 + je L(1_2_0) ALIGN_4 -.L2_00_01: +L(2_00_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_00_02b: +L(2_00_02b): vmovups (BO1), %xmm0 vmovups 2 * SIZE(BO1), %xmm1 @@ -1714,14 +1714,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4*SIZE,BO1 addq $ 4*SIZE,BO decq %rax - jnz .L2_00_02b + jnz L(2_00_02b) -.L2_00_02c: +L(2_00_02c): movq BO1, B // next offset of B -.L2_00_10: +L(2_00_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -1735,13 +1735,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 2, I // i = (m >> 2) - je .L2_2_10 + je L(2_2_10) ALIGN_4 /******************************************************************************************************************/ -.L2_4_11: +L(2_4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1779,7 +1779,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_4_16 + je L(2_4_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1790,7 +1790,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_12: +L(2_4_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI ,SIZE) @@ -1814,7 +1814,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL4x2_SUB - je .L2_4_16 + je L(2_4_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI ,SIZE) @@ -1838,12 +1838,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL4x2_SUB - je .L2_4_16 + je L(2_4_16) - jmp .L2_4_12 + jmp L(2_4_12) ALIGN_4 -.L2_4_16: +L(2_4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -1851,7 +1851,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_4_19 + je L(2_4_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1863,15 +1863,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_17: +L(2_4_17): KERNEL4x2_SUB - jl .L2_4_17 + jl L(2_4_17) ALIGN_4 -.L2_4_19: +L(2_4_19): SAVE4x2 @@ -1893,7 +1893,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L2_4_11 + jg L(2_4_11) ALIGN_4 @@ -1903,11 +1903,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************************************************/ -.L2_2_10: +L(2_2_10): testq $ 2, M - jz .L2_2_40 // to next 2 lines of N + jz L(2_2_40) // to next 2 lines of N -.L2_2_11: +L(2_2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1945,7 +1945,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_2_16 + je L(2_2_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -1956,7 +1956,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_2_12: +L(2_2_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -1976,7 +1976,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_2_16 + je L(2_2_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -1996,12 +1996,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_2_16 + je L(2_2_16) - jmp .L2_2_12 + jmp L(2_2_12) ALIGN_4 -.L2_2_16: +L(2_2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -2009,7 +2009,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_2_19 + je L(2_2_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2021,15 +2021,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_2_17: +L(2_2_17): KERNEL2x2_SUB - jl .L2_2_17 + jl L(2_2_17) ALIGN_4 -.L2_2_19: +L(2_2_19): SAVE2x2 @@ -2056,13 +2056,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /************************************************************************** * Rest of M ***************************************************************************/ -.L2_2_40: +L(2_2_40): testq $ 1, M - jz .L2_2_60 // to next 2 lines of N + jz L(2_2_60) // to next 2 lines of N ALIGN_4 -.L2_2_41: +L(2_2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2100,7 +2100,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_2_46 + je L(2_2_46) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2111,7 +2111,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_2_42: +L(2_2_42): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2129,7 +2129,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_2_46 + je L(2_2_46) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2147,12 +2147,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_2_46 + je L(2_2_46) - jmp .L2_2_42 + jmp L(2_2_42) ALIGN_4 -.L2_2_46: +L(2_2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -2160,7 +2160,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_2_49 + je L(2_2_49) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2172,15 +2172,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_2_47: +L(2_2_47): KERNEL1x2_SUB - jl .L2_2_47 + jl L(2_2_47) ALIGN_4 -.L2_2_49: +L(2_2_49): SAVE1x2 @@ -2202,23 +2202,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 2 * SIZE, CO1 # coffset += 2 decq I # i -- - jg .L2_2_41 + jg L(2_2_41) ALIGN_4 -.L2_2_60: +L(2_2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $ 2, KK #endif decq J // j -- - jg .L2_00_01 // next 2 lines of N + jg L(2_00_01) // next 2 lines of N -.L1_2_0: +L(1_2_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -2226,30 +2226,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $ 1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_00_01: +L(1_00_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_00_02b: +L(1_00_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $ 2*SIZE,BO1 addq $ 2*SIZE,BO decq %rax - jnz .L1_00_02b + jnz L(1_00_02b) -.L1_00_02c: +L(1_00_02c): movq BO1, B // next offset of B -.L1_00_10: +L(1_00_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -2263,14 +2263,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 2, I // i = (m >> 2) - je .L1_2_10 + je L(1_2_10) ALIGN_4 /*******************************************************************************************************/ -.L1_4_11: +L(1_4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2308,7 +2308,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_4_16 + je L(1_4_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -2319,7 +2319,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_12: +L(1_4_12): KERNEL4x1_SUB KERNEL4x1_SUB @@ -2331,7 +2331,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_4_16 + je L(1_4_16) KERNEL4x1_SUB KERNEL4x1_SUB @@ -2343,12 +2343,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_4_16 + je L(1_4_16) - jmp .L1_4_12 + jmp L(1_4_12) ALIGN_4 -.L1_4_16: +L(1_4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -2356,7 +2356,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_4_19 + je L(1_4_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -2368,15 +2368,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_17: +L(1_4_17): KERNEL4x1_SUB - jl .L1_4_17 + jl L(1_4_17) ALIGN_4 -.L1_4_19: +L(1_4_19): SAVE4x1 @@ -2399,19 +2399,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L1_4_11 + jg L(1_4_11) ALIGN_4 /*******************************************************************************************************/ -.L1_2_10: +L(1_2_10): testq $ 2, M - jz .L1_2_40 + jz L(1_2_40) -.L1_2_11: +L(1_2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2449,7 +2449,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_2_16 + je L(1_2_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -2460,7 +2460,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_2_12: +L(1_2_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2477,7 +2477,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_2_16 + je L(1_2_16) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_SUB @@ -2493,12 +2493,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_2_16 + je L(1_2_16) - jmp .L1_2_12 + jmp L(1_2_12) ALIGN_4 -.L1_2_16: +L(1_2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -2506,7 +2506,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_2_19 + je L(1_2_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -2518,15 +2518,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_2_17: +L(1_2_17): KERNEL2x1_SUB - jl .L1_2_17 + jl L(1_2_17) ALIGN_4 -.L1_2_19: +L(1_2_19): SAVE2x1 @@ -2554,13 +2554,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /************************************************************************** * Rest of M ***************************************************************************/ -.L1_2_40: +L(1_2_40): testq $ 1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_2_41: +L(1_2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2598,7 +2598,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_2_46 + je L(1_2_46) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -2609,7 +2609,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_2_42: +L(1_2_42): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2625,7 +2625,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_2_46 + je L(1_2_46) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -2641,12 +2641,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_2_46 + je L(1_2_46) - jmp .L1_2_42 + jmp L(1_2_42) ALIGN_4 -.L1_2_46: +L(1_2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -2654,7 +2654,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_2_49 + je L(1_2_49) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -2666,15 +2666,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_2_47: +L(1_2_47): KERNEL1x1_SUB - jl .L1_2_47 + jl L(1_2_47) ALIGN_4 -.L1_2_49: +L(1_2_49): SAVE1x1 @@ -2696,7 +2696,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 2 * SIZE, CO1 # coffset += 2 decq I # i -- - jg .L1_2_41 + jg L(1_2_41) ALIGN_4 @@ -2704,7 +2704,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L999: +L(999): vzeroupper movq SP, %rsp @@ -2796,13 +2796,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. STACK_TOUCH cmpq $ 0, OLD_M - je .L999 + je L(999) cmpq $ 0, OLD_N - je .L999 + je L(999) cmpq $ 0, OLD_K - je .L999 + je L(999) movq OLD_M, M movq OLD_N, N @@ -2830,23 +2830,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif #endif -.L2_00_0: +L(2_00_0): movq Ndiv6, J cmpq $ 0, J - je .L1_2_0 + je L(1_2_0) ALIGN_4 -.L2_00_01: +L(2_00_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L2_00_02b: +L(2_00_02b): vmovups (BO1), %xmm0 vmovups 2 * SIZE(BO1), %xmm1 @@ -2855,14 +2855,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 4*SIZE,BO1 addq $ 4*SIZE,BO decq %rax - jnz .L2_00_02b + jnz L(2_00_02b) -.L2_00_02c: +L(2_00_02c): movq BO1, B // next offset of B -.L2_00_10: +L(2_00_10): movq C, CO1 leaq (C, LDC, 2), C // c += 2 * ldc @@ -2876,13 +2876,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 2, I // i = (m >> 2) - je .L2_2_10 + je L(2_2_10) ALIGN_4 /******************************************************************************************************************/ -.L2_4_11: +L(2_4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2920,7 +2920,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_4_16 + je L(2_4_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -2931,7 +2931,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_12: +L(2_4_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI ,SIZE) @@ -2955,7 +2955,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL4x2_SUB - je .L2_4_16 + je L(2_4_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI ,SIZE) @@ -2979,12 +2979,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prefetcht0 A_PR1(AO,%rax,SIZE) KERNEL4x2_SUB - je .L2_4_16 + je L(2_4_16) - jmp .L2_4_12 + jmp L(2_4_12) ALIGN_4 -.L2_4_16: +L(2_4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -2992,7 +2992,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_4_19 + je L(2_4_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -3004,15 +3004,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_4_17: +L(2_4_17): KERNEL4x2_SUB - jl .L2_4_17 + jl L(2_4_17) ALIGN_4 -.L2_4_19: +L(2_4_19): SAVE4x2 @@ -3034,7 +3034,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L2_4_11 + jg L(2_4_11) ALIGN_4 @@ -3044,11 +3044,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************************************************/ -.L2_2_10: +L(2_2_10): testq $ 2, M - jz .L2_2_40 // to next 2 lines of N + jz L(2_2_40) // to next 2 lines of N -.L2_2_11: +L(2_2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3086,7 +3086,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_2_16 + je L(2_2_16) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -3097,7 +3097,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_2_12: +L(2_2_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3117,7 +3117,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_2_16 + je L(2_2_16) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3137,12 +3137,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB KERNEL2x2_SUB - je .L2_2_16 + je L(2_2_16) - jmp .L2_2_12 + jmp L(2_2_12) ALIGN_4 -.L2_2_16: +L(2_2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3150,7 +3150,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_2_19 + je L(2_2_19) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -3162,15 +3162,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_2_17: +L(2_2_17): KERNEL2x2_SUB - jl .L2_2_17 + jl L(2_2_17) ALIGN_4 -.L2_2_19: +L(2_2_19): SAVE2x2 @@ -3197,13 +3197,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /************************************************************************** * Rest of M ***************************************************************************/ -.L2_2_40: +L(2_2_40): testq $ 1, M - jz .L2_2_60 // to next 2 lines of N + jz L(2_2_60) // to next 2 lines of N ALIGN_4 -.L2_2_41: +L(2_2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3241,7 +3241,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L2_2_46 + je L(2_2_46) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -3252,7 +3252,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_2_42: +L(2_2_42): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3270,7 +3270,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_2_46 + je L(2_2_46) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3288,12 +3288,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB KERNEL1x2_SUB - je .L2_2_46 + je L(2_2_46) - jmp .L2_2_42 + jmp L(2_2_42) ALIGN_4 -.L2_2_46: +L(2_2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -3301,7 +3301,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L2_2_49 + je L(2_2_49) movq %rax, BI // Index for BO leaq ( ,BI,4), BI // BI = BI * 4 ; number of values @@ -3313,15 +3313,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L2_2_47: +L(2_2_47): KERNEL1x2_SUB - jl .L2_2_47 + jl L(2_2_47) ALIGN_4 -.L2_2_49: +L(2_2_49): SAVE1x2 @@ -3343,23 +3343,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 2 * SIZE, CO1 # coffset += 2 decq I # i -- - jg .L2_2_41 + jg L(2_2_41) ALIGN_4 -.L2_2_60: +L(2_2_60): #if defined(TRMMKERNEL) && !defined(LEFT) addq $ 2, KK #endif decq J // j -- - jg .L2_00_01 // next 2 lines of N + jg L(2_00_01) // next 2 lines of N -.L1_2_0: +L(1_2_0): /************************************************************************************************ * Loop for Nmod6 % 2 > 0 @@ -3367,30 +3367,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq Nmod6, J andq $ 1, J // j % 2 - je .L999 + je L(999) ALIGN_4 -.L1_00_01: +L(1_00_01): // copy to sub buffer movq B, BO1 leaq BUFFER1, BO // first buffer to BO movq K, %rax ALIGN_4 -.L1_00_02b: +L(1_00_02b): vmovups (BO1), %xmm0 vmovups %xmm0, (BO) addq $ 2*SIZE,BO1 addq $ 2*SIZE,BO decq %rax - jnz .L1_00_02b + jnz L(1_00_02b) -.L1_00_02c: +L(1_00_02c): movq BO1, B // next offset of B -.L1_00_10: +L(1_00_10): movq C, CO1 leaq (C, LDC, 1), C // c += 1 * ldc @@ -3404,14 +3404,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. movq M, I sarq $ 2, I // i = (m >> 2) - je .L1_2_10 + je L(1_2_10) ALIGN_4 /*******************************************************************************************************/ -.L1_4_11: +L(1_4_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3449,7 +3449,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_4_16 + je L(1_4_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -3460,7 +3460,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_12: +L(1_4_12): KERNEL4x1_SUB KERNEL4x1_SUB @@ -3472,7 +3472,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_4_16 + je L(1_4_16) KERNEL4x1_SUB KERNEL4x1_SUB @@ -3484,12 +3484,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB KERNEL4x1_SUB - je .L1_4_16 + je L(1_4_16) - jmp .L1_4_12 + jmp L(1_4_12) ALIGN_4 -.L1_4_16: +L(1_4_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3497,7 +3497,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_4_19 + je L(1_4_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -3509,15 +3509,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_4_17: +L(1_4_17): KERNEL4x1_SUB - jl .L1_4_17 + jl L(1_4_17) ALIGN_4 -.L1_4_19: +L(1_4_19): SAVE4x1 @@ -3540,19 +3540,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 8 * SIZE, CO1 # coffset += 8 decq I # i -- - jg .L1_4_11 + jg L(1_4_11) ALIGN_4 /*******************************************************************************************************/ -.L1_2_10: +L(1_2_10): testq $ 2, M - jz .L1_2_40 + jz L(1_2_40) -.L1_2_11: +L(1_2_11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3590,7 +3590,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_2_16 + je L(1_2_16) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -3601,7 +3601,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_2_12: +L(1_2_12): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3618,7 +3618,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_2_16 + je L(1_2_16) prefetcht0 B_PR1(BO,BI,SIZE) KERNEL2x1_SUB @@ -3634,12 +3634,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB KERNEL2x1_SUB - je .L1_2_16 + je L(1_2_16) - jmp .L1_2_12 + jmp L(1_2_12) ALIGN_4 -.L1_2_16: +L(1_2_16): #ifndef TRMMKERNEL movq K, %rax #else @@ -3647,7 +3647,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_2_19 + je L(1_2_19) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -3659,15 +3659,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_2_17: +L(1_2_17): KERNEL2x1_SUB - jl .L1_2_17 + jl L(1_2_17) ALIGN_4 -.L1_2_19: +L(1_2_19): SAVE2x1 @@ -3695,13 +3695,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /************************************************************************** * Rest of M ***************************************************************************/ -.L1_2_40: +L(1_2_40): testq $ 1, M - jz .L999 + jz L(999) ALIGN_4 -.L1_2_41: +L(1_2_41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -3739,7 +3739,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. andq $ -8, %rax // K = K - ( K % 8 ) - je .L1_2_46 + je L(1_2_46) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -3750,7 +3750,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_2_42: +L(1_2_42): prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3766,7 +3766,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_2_46 + je L(1_2_46) prefetcht0 A_PR1(AO,%rax,SIZE) prefetcht0 B_PR1(BO,BI,SIZE) @@ -3782,12 +3782,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB KERNEL1x1_SUB - je .L1_2_46 + je L(1_2_46) - jmp .L1_2_42 + jmp L(1_2_42) ALIGN_4 -.L1_2_46: +L(1_2_46): #ifndef TRMMKERNEL movq K, %rax #else @@ -3795,7 +3795,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif andq $ 7, %rax # if (k & 1) - je .L1_2_49 + je L(1_2_49) movq %rax, BI // Index for BO leaq ( ,BI,2), BI // BI = BI * 2 ; number of values @@ -3807,15 +3807,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. negq %rax ALIGN_4 -.L1_2_47: +L(1_2_47): KERNEL1x1_SUB - jl .L1_2_47 + jl L(1_2_47) ALIGN_4 -.L1_2_49: +L(1_2_49): SAVE1x1 @@ -3837,7 +3837,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addq $ 2 * SIZE, CO1 # coffset += 2 decq I # i -- - jg .L1_2_41 + jg L(1_2_41) ALIGN_4 @@ -3845,7 +3845,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.L999: +L(999): vzeroupper movq SP, %rsp diff --git a/kernel/x86_64/zgemm_kernel_4x2_penryn.S b/kernel/x86_64/zgemm_kernel_4x2_penryn.S index 9aa852ae6e..0dfa421287 100644 --- a/kernel/x86_64/zgemm_kernel_4x2_penryn.S +++ b/kernel/x86_64/zgemm_kernel_4x2_penryn.S @@ -185,10 +185,10 @@ movq N, J sarq $1, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -205,10 +205,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -266,10 +266,10 @@ #endif sarq $3, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH -32 * SIZE(PREA) ADD1 %xmm6, %xmm10 ADD1 %xmm3, %xmm14 @@ -520,10 +520,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): prefetcht0 -16 * SIZE(BB) #ifndef TRMMKERNEL @@ -533,10 +533,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm6, %xmm10 ADD1 %xmm3, %xmm14 movaps %xmm2, %xmm3 @@ -572,10 +572,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): ADD1 %xmm6, %xmm10 ADD1 %xmm3, %xmm14 ADD2 %xmm4, %xmm11 @@ -686,13 +686,13 @@ addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M BRANCH - jle .L30 + jle L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -734,10 +734,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): ADD1 %xmm6, %xmm10 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0xb1, %xmm2, %xmm7 @@ -804,10 +804,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -815,10 +815,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): ADD1 %xmm6, %xmm10 pshufd $0xb1, %xmm2, %xmm7 mulps %xmm0, %xmm2 @@ -839,10 +839,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): ADD1 %xmm6, %xmm10 ADD2 %xmm4, %xmm11 @@ -919,10 +919,10 @@ addq $4 * SIZE, CO2 ALIGN_4 -.L30: +L(30): testq $1, M BRANCH - jle .L39 + jle L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -964,10 +964,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): ADD1 %xmm6, %xmm10 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0xb1, %xmm2, %xmm7 @@ -1034,10 +1034,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1045,10 +1045,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): ADD1 %xmm6, %xmm10 pshufd $0xb1, %xmm2, %xmm7 mulps %xmm0, %xmm2 @@ -1069,10 +1069,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): ADD1 %xmm6, %xmm10 ADD2 %xmm4, %xmm11 @@ -1142,7 +1142,7 @@ #endif ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addq $2, KK #endif @@ -1152,13 +1152,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax @@ -1171,10 +1171,10 @@ movq M, I sarq $2, I NOBRANCH - jle .L50 + jle L(50) ALIGN_4 -.L41: +L(41): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1220,10 +1220,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm3, %xmm8 @@ -1299,10 +1299,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #ifndef TRMMKERNEL movq K, %rax #else @@ -1310,10 +1310,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): addps %xmm3, %xmm8 pshufd $0x00, %xmm2, %xmm3 mulps %xmm0, %xmm3 @@ -1336,10 +1336,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): addps %xmm3, %xmm8 addps %xmm4, %xmm12 addps %xmm5, %xmm9 @@ -1421,13 +1421,13 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- BRANCH - jg .L41 + jg L(41) ALIGN_4 -.L50: +L(50): testq $2, M BRANCH - jle .L60 + jle L(60) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1467,10 +1467,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm3, %xmm8 @@ -1512,10 +1512,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1523,10 +1523,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): addps %xmm3, %xmm8 pshufd $0x00, %xmm2, %xmm3 mulps %xmm0, %xmm3 @@ -1541,10 +1541,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): addps %xmm3, %xmm8 movddup ALPHA_R, %xmm2 addps %xmm4, %xmm9 @@ -1605,10 +1605,10 @@ addq $4 * SIZE, CO1 ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L999 + jle L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1648,10 +1648,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm3, %xmm8 @@ -1693,10 +1693,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1704,10 +1704,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): addps %xmm3, %xmm8 pshufd $0x00, %xmm2, %xmm3 mulps %xmm0, %xmm3 @@ -1722,10 +1722,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): addps %xmm3, %xmm8 movddup ALPHA_R, %xmm2 addps %xmm4, %xmm9 @@ -1765,7 +1765,7 @@ movsd %xmm8, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemm_kernel_4x2_sse.S b/kernel/x86_64/zgemm_kernel_4x2_sse.S index 5841f8b9e0..abb2b2f4e5 100644 --- a/kernel/x86_64/zgemm_kernel_4x2_sse.S +++ b/kernel/x86_64/zgemm_kernel_4x2_sse.S @@ -437,10 +437,10 @@ salq $ZBASE_SHIFT, LDC movq N, J sarq $1, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -452,12 +452,12 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): PREFETCH (RPREFETCHSIZE + 0) * SIZE(B) movss 0 * SIZE(B), %xmm8 @@ -507,17 +507,17 @@ addq $32 * SIZE, BO addq $ 8 * SIZE, B decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movss 0 * SIZE(B), %xmm8 movss 1 * SIZE(B), %xmm9 movss 2 * SIZE(B), %xmm10 @@ -545,10 +545,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a @@ -557,10 +557,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -620,10 +620,10 @@ leaq (BO, %rax, 8), BO negq %rax NOBRANCH - je .L15 + je L(15) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -644,7 +644,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -665,7 +665,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -686,7 +686,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -707,7 +707,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -728,7 +728,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -749,7 +749,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -770,7 +770,7 @@ addq $16 * SIZE, %rax NOBRANCH - je .L15 + je L(15) KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -791,17 +791,17 @@ addq $16 * SIZE, %rax BRANCH - jl .L12 + jl L(12) ALIGN_3 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else movq KKK, %rax #endif testq $4, %rax - je .L16 + je L(16) xorq %rax, %rax ALIGN_3 @@ -820,10 +820,10 @@ #else sarq $2, %rax NOBRANCH - jle .L16 + jle L(16) ALIGN_3 -.L12: +L(12): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -837,11 +837,11 @@ subq $-32 * SIZE, AO decq %rax BRANCH - jg .L12 + jg L(12) #endif -.L16: +L(16): #ifndef TRMMKERNEL movq K, %rax #else @@ -851,7 +851,7 @@ movaps ALPHA_I, %xmm7 andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) leaq (, %rax, 8), %rax leaq (AO, %rax, 4), AO @@ -859,7 +859,7 @@ negq %rax ALIGN_4 -.L17: +L(17): mulps %xmm0, %xmm1 addps %xmm1, %xmm8 movaps -28 * SIZE(BO, %rax, 8), %xmm1 @@ -886,10 +886,10 @@ movaps -20 * SIZE(AO, %rax, 4), %xmm2 addq $SIZE * 2, %rax - jl .L17 + jl L(17) ALIGN_4 -.L18: +L(18): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -978,12 +978,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1029,10 +1029,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm0, %xmm1 #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1147,10 +1147,10 @@ subq $-128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1160,10 +1160,10 @@ movaps ALPHA_I, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm0, %xmm1 addps %xmm1, %xmm8 movaps -28 * SIZE(BO), %xmm1 @@ -1180,10 +1180,10 @@ subq $- 4 * SIZE, AO subq $-16 * SIZE, BO decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -1246,9 +1246,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1292,10 +1292,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm0, %xmm1 #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1415,10 +1415,10 @@ subq $-128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1428,10 +1428,10 @@ movaps ALPHA_I, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm0, %xmm1 addps %xmm1, %xmm8 movaps -28 * SIZE(BO), %xmm1 @@ -1449,10 +1449,10 @@ subq $ -2 * SIZE, AO subq $-16 * SIZE, BO decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): #ifndef TRMMKERNEL #ifdef movsd xorps %xmm0, %xmm0 @@ -1512,22 +1512,22 @@ #endif ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif leaq (C, LDC, 2), C # c += 2 * ldc decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L41: +L(41): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1539,10 +1539,10 @@ movq K, %rax sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): movss 0 * SIZE(B), %xmm8 movss 1 * SIZE(B), %xmm9 movss 2 * SIZE(B), %xmm10 @@ -1595,17 +1595,17 @@ addq $32 * SIZE, BO addq $ 8 * SIZE, B decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): movq K, %rax andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movss 0 * SIZE(B), %xmm8 movss 1 * SIZE(B), %xmm9 @@ -1625,19 +1625,19 @@ addq $2 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1688,10 +1688,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulps %xmm0, %xmm1 #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1815,10 +1815,10 @@ subq $-64 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1828,10 +1828,10 @@ movaps ALPHA_I, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): mulps %xmm0, %xmm1 mulps -28 * SIZE(BO), %xmm0 addps %xmm1, %xmm8 @@ -1848,10 +1848,10 @@ addq $ 8 * SIZE, AO addq $ 8 * SIZE, BO decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -1910,12 +1910,12 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1959,10 +1959,10 @@ #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm0, %xmm1 #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2026,10 +2026,10 @@ subq $-64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -2039,10 +2039,10 @@ movaps ALPHA_I, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm0, %xmm1 mulps -28 * SIZE(BO), %xmm0 addps %xmm1, %xmm8 @@ -2053,10 +2053,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #ifndef TRMMKERNEL movsd 0 * SIZE(CO1), %xmm0 movhps 2 * SIZE(CO1), %xmm0 @@ -2106,9 +2106,9 @@ addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L70: +L(70): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -2151,10 +2151,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm0, %xmm1 #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2226,10 +2226,10 @@ subq $-64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2239,10 +2239,10 @@ movaps ALPHA_I, %xmm7 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm0, %xmm1 addps %xmm1, %xmm8 movaps -28 * SIZE(BO), %xmm1 @@ -2254,10 +2254,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): #ifndef TRMMKERNEL #ifdef movsd xorps %xmm0, %xmm0 @@ -2291,7 +2291,7 @@ movlps %xmm8, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/zgemm_kernel_4x2_sse3.S b/kernel/x86_64/zgemm_kernel_4x2_sse3.S index 4e5504c426..6a1d0cbac3 100644 --- a/kernel/x86_64/zgemm_kernel_4x2_sse3.S +++ b/kernel/x86_64/zgemm_kernel_4x2_sse3.S @@ -409,10 +409,10 @@ salq $ZBASE_SHIFT, LDC movq N, J sarq $1, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -423,10 +423,10 @@ movq K, %rax sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): movddup 0 * SIZE(B), %xmm0 movddup 2 * SIZE(B), %xmm1 movddup 4 * SIZE(B), %xmm2 @@ -451,17 +451,17 @@ addq $16 * SIZE, B addq $32 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): movq K, %rax andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movddup 0 * SIZE(B), %xmm0 movddup 2 * SIZE(B), %xmm1 @@ -471,10 +471,10 @@ addq $4 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a @@ -483,10 +483,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): prefetcht0 0 * SIZE(BB) subq $-8 * SIZE, BB @@ -542,9 +542,9 @@ #if 1 andq $-8, %rax salq $4, %rax - je .L15 + je L(15) -.L1X: +L(1X): KERNEL1 (32 * 0) KERNEL2 (32 * 0) KERNEL3 (32 * 0) @@ -562,7 +562,7 @@ KERNEL15(32 * 0) KERNEL16(32 * 0) cmpq $128 * 1, %rax - jle .L12 + jle L(12) KERNEL1 (32 * 1) KERNEL2 (32 * 1) KERNEL3 (32 * 1) @@ -580,7 +580,7 @@ KERNEL15(32 * 1) KERNEL16(32 * 1) cmpq $128 * 2, %rax - jle .L12 + jle L(12) KERNEL1 (32 * 2) KERNEL2 (32 * 2) KERNEL3 (32 * 2) @@ -598,7 +598,7 @@ KERNEL15(32 * 2) KERNEL16(32 * 2) cmpq $128 * 3, %rax - jle .L12 + jle L(12) KERNEL1 (32 * 3) KERNEL2 (32 * 3) KERNEL3 (32 * 3) @@ -616,7 +616,7 @@ KERNEL15(32 * 3) KERNEL16(32 * 3) cmpq $128 * 4, %rax - jle .L12 + jle L(12) KERNEL1 (32 * 4) KERNEL2 (32 * 4) KERNEL3 (32 * 4) @@ -634,7 +634,7 @@ KERNEL15(32 * 4) KERNEL16(32 * 4) cmpq $128 * 5, %rax - jle .L12 + jle L(12) KERNEL1 (32 * 5) KERNEL2 (32 * 5) KERNEL3 (32 * 5) @@ -652,7 +652,7 @@ KERNEL15(32 * 5) KERNEL16(32 * 5) cmpq $128 * 6, %rax - jle .L12 + jle L(12) KERNEL1 (32 * 6) KERNEL2 (32 * 6) KERNEL3 (32 * 6) @@ -670,7 +670,7 @@ KERNEL15(32 * 6) KERNEL16(32 * 6) cmpq $128 * 7, %rax - jle .L12 + jle L(12) KERNEL1 (32 * 7) KERNEL2 (32 * 7) KERNEL3 (32 * 7) @@ -691,17 +691,17 @@ addq $64 * 8 * SIZE, AO addq $64 * 8 * SIZE, BO subq $128 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 2), BO # * 64 #else sarq $3, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): KERNEL1 (32 * 0) KERNEL2 (32 * 0) KERNEL3 (32 * 0) @@ -722,11 +722,11 @@ addq $64 * SIZE, AO addq $64 * SIZE, BO decq %rax - jne .L12 + jne L(12) #endif ALIGN_4 -.L15: +L(15): #ifndef TRMMKERNEL movq K, %rax #else @@ -736,10 +736,10 @@ movaps ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -770,10 +770,10 @@ addq $8 * SIZE, AO addq $8 * SIZE, BO decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \ defined(NR) || defined(NC) || defined(TR) || defined(TC) @@ -881,12 +881,12 @@ addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -929,10 +929,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm9, %xmm0 @@ -1043,10 +1043,10 @@ addq $64 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #ifndef TRMMKERNEL movq K, %rax #else @@ -1056,10 +1056,10 @@ movaps ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -1077,10 +1077,10 @@ addq $ 4 * SIZE, AO addq $ 8 * SIZE, BO decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \ defined(NR) || defined(NC) || defined(TR) || defined(TC) @@ -1152,9 +1152,9 @@ addq $4 * SIZE, CO2 # coffset += 4 ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1196,10 +1196,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): shufps $0x50, %xmm9, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulps %xmm8, %xmm9 @@ -1278,10 +1278,10 @@ addq $64 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #ifndef TRMMKERNEL movq K, %rax #else @@ -1291,10 +1291,10 @@ movaps ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): shufps $0x50, %xmm9, %xmm9 mulps %xmm8, %xmm9 addps %xmm9, %xmm0 @@ -1308,10 +1308,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): movaps %xmm0, %xmm6 movlhps %xmm1, %xmm0 movhlps %xmm6, %xmm1 @@ -1371,22 +1371,22 @@ #endif ALIGN_4 -.L39: +L(39): #if defined(TRMMKERNEL) && !defined(LEFT) addl $2, KK #endif leaq (C, LDC, 2), C # c += 2 * ldc decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $1, N - je .L999 + je L(999) ALIGN_4 -.L41: +L(41): #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK @@ -1397,10 +1397,10 @@ movq K, %rax sarq $3, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): movddup 0 * SIZE(B), %xmm0 movddup 2 * SIZE(B), %xmm1 movddup 4 * SIZE(B), %xmm2 @@ -1425,17 +1425,17 @@ addq $16 * SIZE, B addq $32 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): movq K, %rax andq $7, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movddup 0 * SIZE(B), %xmm0 movaps %xmm0, 0 * SIZE(BO) @@ -1443,19 +1443,19 @@ addq $2 * SIZE, B addq $4 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): movq C, CO1 # coffset1 = c movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) @@ -1499,10 +1499,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm9, %xmm0 @@ -1621,10 +1621,10 @@ addq $64 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #ifndef TRMMKERNEL movq K, %rax #else @@ -1634,10 +1634,10 @@ movaps ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -1656,10 +1656,10 @@ addq $ 8 * SIZE, AO addq $ 4 * SIZE, BO decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \ defined(NR) || defined(NC) || defined(TR) || defined(TC) @@ -1726,12 +1726,12 @@ addq $8 * SIZE, CO1 # coffset += 4 decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1769,10 +1769,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) addps %xmm9, %xmm0 @@ -1835,10 +1835,10 @@ addq $32 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #ifndef TRMMKERNEL movq K, %rax #else @@ -1848,10 +1848,10 @@ movaps ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movshdup 0 * SIZE(BO), %xmm9 @@ -1863,10 +1863,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \ defined(NR) || defined(NC) || defined(TR) || defined(TC) shufps $0xb1, %xmm1, %xmm1 @@ -1909,9 +1909,9 @@ addq $4 * SIZE, CO1 # coffset += 4 ALIGN_4 -.L70: +L(70): testq $1, M - je .L999 + je L(999) #if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ @@ -1949,10 +1949,10 @@ movq %rax, KKK #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): shufps $0x50, %xmm9, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulps %xmm8, %xmm9 @@ -1998,10 +1998,10 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #ifndef TRMMKERNEL movq K, %rax #else @@ -2011,10 +2011,10 @@ movaps ALPHA_I, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): shufps $0x50, %xmm9, %xmm9 mulps %xmm8, %xmm9 movddup 2 * SIZE(AO), %xmm8 @@ -2024,10 +2024,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm1, %xmm0 movhlps %xmm0, %xmm1 @@ -2070,7 +2070,7 @@ movsd %xmm0, 0 * SIZE(CO1) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/zgemm_kernel_4x4_sandy.S b/kernel/x86_64/zgemm_kernel_4x4_sandy.S index e1176d131c..feac59114b 100644 --- a/kernel/x86_64/zgemm_kernel_4x4_sandy.S +++ b/kernel/x86_64/zgemm_kernel_4x4_sandy.S @@ -276,9 +276,9 @@ movq %r11, kk; MOVQ bn,j; SARQ $2,j; # Rn = 4 -JLE .L0_loopE; +JLE L(0_loopE); ALIGN_5; -.L0_bodyB:; +L(0_bodyB):; #if defined(TRMMKERNEL) && defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk; @@ -291,9 +291,9 @@ LEAQ (bb, k, 1), prebb; # Rn=4 SIZE=8 COMPLEX=2 MOVQ ba,ptrba; MOVQ bm,i; SARQ $2,i; # Rm = 4 -JLE .L1_loopE; +JLE L(1_loopE); ALIGN_5; -.L1_bodyB:; +L(1_bodyB):; #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -340,9 +340,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2,k; # Unroll 4 times -JLE .L2_loopE; +JLE L(2_loopE); ALIGN_5; -.L2_bodyB:; +L(2_bodyB):; #### Computing kernel #### #### Unroll time 1 #### @@ -563,17 +563,17 @@ MUL_DY yvec1, yvec5, yvec7; ADD2_DY yvec6, yvec12, yvec12; ADD2_DY yvec7, yvec8, yvec8; DECQ k; -JG .L2_bodyB; +JG L(2_bodyB); ALIGN_5 -.L2_loopE:; +L(2_loopE):; #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L3_loopE; +JLE L(3_loopE); ALIGN_5 -.L3_bodyB: +L(3_bodyB): #### Unroll time 1 #### LD_DY 4*SIZE(ptrba), yvec1; MUL_DY yvec0, yvec2, yvec6; @@ -683,15 +683,15 @@ MUL_DY yvec1, yvec4, yvec6; MUL_DY yvec1, yvec5, yvec7; ADD2_DY yvec6, yvec12, yvec12; ADD2_DY yvec7, yvec8, yvec8; -.L3_loopE:; +L(3_loopE):; #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L4_loopE; +JLE L(4_loopE); ALIGN_5 -.L4_loopB:; +L(4_loopB):; #### Unroll time 1 #### PREFETCH0 PRESIZE*SIZE(ptrba); LD_DY 4*SIZE(ptrba), yvec1; @@ -744,7 +744,7 @@ MUL_DY yvec1, yvec4, yvec6; MUL_DY yvec1, yvec5, yvec7; ADD2_DY yvec6, yvec12, yvec12; ADD2_DY yvec7, yvec8, yvec8; -.L4_loopE:; +L(4_loopE):; #### Handle #### XOR_DY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -831,7 +831,7 @@ ADDSUB_DY yvec4, yvec8, yvec8; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L4_loopEx; +JNE L(4_loopEx); ALIGN_5 #### Store Back #### EXTRA_DY $1,yvec15,xvec7; @@ -888,12 +888,12 @@ ADDQ $4, kk; #endif ADDQ $8*SIZE,C0; ADDQ $8*SIZE,C1; -.L1_bodyE:; +L(1_bodyE):; DECQ i; -JG .L1_bodyB; -JMP .L1_loopE; +JG L(1_bodyB); +JMP L(1_loopE); ALIGN_5 -.L4_loopEx: +L(4_loopEx): EXTRA_DY $1, yvec15, xvec7; EXTRA_DY $1, yvec14, xvec6; #ifndef TRMMKERNEL @@ -1003,13 +1003,13 @@ ADDQ $4, kk; ADDQ $8*SIZE, C0; ADDQ $8*SIZE, C1; DECQ i; -JG .L1_bodyB; +JG L(1_bodyB); ALIGN_5; -.L1_loopE:; +L(1_loopE):; TEST $2, bm; -JLE .L5_loopE; +JLE L(5_loopE); ALIGN_5 -.L5_bodyB: +L(5_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -1039,9 +1039,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L7_loopE; +JLE L(7_loopE); ALIGN_5 -.L7_bodyB: +L(7_bodyB): #### Compute kernel #### #### Unroll times 1 #### LD_DY 0*SIZE(ptrba), yvec0; @@ -1173,17 +1173,17 @@ MUL_DY yvec0, yvec5, yvec7; ADD2_DY yvec7, yvec12, yvec12; ADDQ $32*SIZE, ptrbb; DECQ k; -JG .L7_bodyB; +JG L(7_bodyB); ALIGN_5 -.L7_loopE: +L(7_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L8_loopE; +JLE L(8_loopE); ALIGN_5 -.L8_bodyB: +L(8_bodyB): #### Unroll times 1 #### LD_DY 0*SIZE(ptrba), yvec0; EDUP_DY 0*SIZE(ptrbb), yvec2; @@ -1249,15 +1249,15 @@ ADDQ $8*SIZE, ptrba; MUL_DY yvec0, yvec5, yvec7; ADD2_DY yvec7, yvec12, yvec12; ADDQ $16*SIZE, ptrbb; -.L8_loopE: +L(8_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L9_loopE; +JLE L(9_loopE); ALIGN_5 -.L9_bodyB: +L(9_bodyB): #### Unroll times 1 #### LD_DY 0*SIZE(ptrba), yvec0; EDUP_DY 0*SIZE(ptrbb), yvec2; @@ -1292,7 +1292,7 @@ ADD2_DY yvec7, yvec12, yvec12; ADDQ $4*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L9_loopE: +L(9_loopE): #### Handle #### XOR_DY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -1343,7 +1343,7 @@ ADD2_DY yvec2, yvec12, yvec12; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L9_loopEx; +JNE L(9_loopEx); ALIGN_5 #### Writing back #### EXTRA_DY $1, yvec15, xvec7; @@ -1380,9 +1380,9 @@ ADDQ $2, kk; #endif ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -JMP .L5_loopE; +JMP L(5_loopE); ALIGN_5 -.L9_loopEx: +L(9_loopEx): EXTRA_DY $1, yvec15, xvec7; EXTRA_DY $1, yvec14, xvec6; EXTRA_DY $1, yvec13, xvec5; @@ -1443,11 +1443,11 @@ ADDQ $2, kk; #endif ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -.L5_loopE: +L(5_loopE): TEST $1, bm; -JLE .L6_loopE; +JLE L(6_loopE); ALIGN_5 -.L6_bodyB: +L(6_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -1475,9 +1475,9 @@ ADDQ $4, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L10_loopE; +JLE L(10_loopE); ALIGN_5 -.L10_bodyB: +L(10_bodyB): LD_DY 0*SIZE(ptrba), yvec0; #### A1r A1i A2r A2i EDUP_DY 0*SIZE(ptrbb), yvec2; EDUP_DY 4*SIZE(ptrbb), yvec3; @@ -1549,17 +1549,17 @@ ADD2_DY yvec7, yvec14, yvec14 ADDQ $8*SIZE, ptrba; ADDQ $32*SIZE, ptrbb; DECQ k; -JG .L10_bodyB; +JG L(10_bodyB); ALIGN_5 -.L10_loopE: +L(10_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L11_loopE; +JLE L(11_loopE); ALIGN_5 -.L11_bodyB: +L(11_bodyB): LD_DY 0*SIZE(ptrba), yvec0; #### A1r A1i A2r A2i EDUP_DY 0*SIZE(ptrbb), yvec2; EDUP_DY 4*SIZE(ptrbb), yvec3; @@ -1597,15 +1597,15 @@ ADD2_DY yvec7, yvec14, yvec14; ADDQ $4*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; -.L11_loopE: +L(11_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L12_loopE; +JLE L(12_loopE); ALIGN_5 -.L12_bodyB: +L(12_bodyB): LD_DY 0*SIZE(ptrba), yvec0; #### A1r A1i A2r A2i EDUP_DY 0*SIZE(ptrbb), yvec2; EDUP_DY 4*SIZE(ptrbb), yvec3; @@ -1626,7 +1626,7 @@ ADD2_DY yvec7, yvec14, yvec14; ADDQ $2*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L12_loopE: +L(12_loopE): #### Handle #### XOR_DY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -1691,7 +1691,7 @@ ADDQ $1, kk; #endif ADDQ $2*SIZE, C0; ADDQ $2*SIZE, C1; -.L6_loopE: +L(6_loopE): #if defined(TRMMKERNEL) && !defined(LEFT) ADDQ $4, kk; #endif @@ -1699,15 +1699,15 @@ MOVQ bk,k; SALQ $6,k; ADDQ k,bb; LEAQ (C,ldc,4),C; -.L0_bodyE:; +L(0_bodyE):; DECQ j; -JG .L0_bodyB; +JG L(0_bodyB); ALIGN_5; -.L0_loopE:; +L(0_loopE):; TEST $2, bn; -JLE .L20_loopE; +JLE L(20_loopE); ALIGN_5 -.L20_bodyB: +L(20_bodyB): #if defined(TRMMKERNEL) && defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk; @@ -1717,9 +1717,9 @@ LEAQ (C, ldc, 1), C1; MOVQ ba, ptrba; MOVQ bm, i; SARQ $2, i; -JLE .L21_loopE; +JLE L(21_loopE); ALIGN_5 -.L21_bodyB: +L(21_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -1749,9 +1749,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L211_loopE; +JLE L(211_loopE); ALIGN_5 -.L211_bodyB: +L(211_bodyB): #### Unroll time 1 #### EDUP_DY 0*SIZE(ptrbb), yvec2; LD_DY 0*SIZE(ptrba), yvec0; @@ -1870,17 +1870,17 @@ MUL_DY yvec1, yvec5, yvec7; ADD2_DY yvec7, yvec12, yvec12; ADDQ $32*SIZE, ptrba; DECQ k; -JG .L211_bodyB; +JG L(211_bodyB); ALIGN_5 -.L211_loopE: +L(211_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L212_loopE; +JLE L(212_loopE); ALIGN_5 -.L212_bodyB: +L(212_bodyB): #### Unroll time 1 #### EDUP_DY 0*SIZE(ptrbb), yvec2; LD_DY 0*SIZE(ptrba), yvec0; @@ -1942,15 +1942,15 @@ ADD2_DY yvec7, yvec12, yvec12; ADDQ $8*SIZE, ptrbb; ADDQ $16*SIZE, ptrba; -.L212_loopE: +L(212_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L213_loopE; +JLE L(213_loopE); ALIGN_5 -.L213_bodyB: +L(213_bodyB): #### Unroll time 1 #### EDUP_DY 0*SIZE(ptrbb), yvec2; LD_DY 0*SIZE(ptrba), yvec0; @@ -1982,7 +1982,7 @@ ADD2_DY yvec7, yvec12, yvec12; ADDQ $4*SIZE, ptrbb; ADDQ $8*SIZE, ptrba; -.L213_loopE: +L(213_loopE): #### Handle #### XOR_DY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -2037,7 +2037,7 @@ EXTRA_DY $1, yvec12, xvec4; MOVQ C0, %rax; OR ldc, %rax; TEST $15, %rax; -JNE .L213_loopEx; +JNE L(213_loopEx); ALIGN_5 #### Writing back #### #ifndef TRMMKERNEL @@ -2071,10 +2071,10 @@ ADDQ $4, kk; ADDQ $8*SIZE, C0; ADDQ $8*SIZE, C1; DECQ i; -JG .L21_bodyB; -JMP .L21_loopE; +JG L(21_bodyB); +JMP L(21_loopE); ALIGN_5 -.L213_loopEx: +L(213_loopEx): #ifndef TRMMKERNEL LDL_DX 0*SIZE(C0), xvec0, xvec0; LDH_DX 1*SIZE(C0), xvec0, xvec0; @@ -2132,13 +2132,13 @@ ADDQ $4, kk; ADDQ $8*SIZE, C0; ADDQ $8*SIZE, C1; DECQ i; -JG .L21_bodyB; +JG L(21_bodyB); ALIGN_5 -.L21_loopE: +L(21_loopE): TEST $2, bm; -JLE .L22_loopE; +JLE L(22_loopE); ALIGN_5 -.L22_bodyB: +L(22_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -2166,9 +2166,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L221_loopE; +JLE L(221_loopE); ALIGN_5 -.L221_bodyB: +L(221_bodyB): #### Unroll time 1 #### EDUP_DY 0*SIZE(ptrbb), yvec2; LD_DY 0*SIZE(ptrba), yvec0; @@ -2247,17 +2247,17 @@ MUL_DY yvec0, yvec5, yvec6; ADD2_DY yvec6, yvec13, yvec13; ADDQ $16*SIZE, ptrba; DECQ k; -JG .L221_bodyB; +JG L(221_bodyB); ALIGN_5 -.L221_loopE: +L(221_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L222_loopE; +JLE L(222_loopE); ALIGN_5 -.L222_bodyB: +L(222_bodyB): #### Unroll time 1 #### EDUP_DY 0*SIZE(ptrbb), yvec2; LD_DY 0*SIZE(ptrba), yvec0; @@ -2298,15 +2298,15 @@ ADD2_DY yvec6, yvec13, yvec13; ADDQ $8*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L222_loopE: +L(222_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L223_loopE; +JLE L(223_loopE); ALIGN_5 -.L223_bodyB: +L(223_bodyB): #### Unroll time 1 #### EDUP_DY 0*SIZE(ptrbb), yvec2; LD_DY 0*SIZE(ptrba), yvec0; @@ -2328,7 +2328,7 @@ ADD2_DY yvec6, yvec13, yvec13; ADDQ $4*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L223_loopE: +L(223_loopE): #### Handle #### XOR_DY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -2396,11 +2396,11 @@ ADDQ $2, kk; ADDQ $4*SIZE, C0; ADDQ $4*SIZE, C1; -.L22_loopE: +L(22_loopE): TEST $1, bm; -JLE .L23_loopE; +JLE L(23_loopE); ALIGN_5 -.L23_bodyB: +L(23_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -2427,9 +2427,9 @@ ADDQ $2, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L231_loopE; +JLE L(231_loopE); ALIGN_5 -.L231_bodyB: +L(231_bodyB): LD_DY 0*SIZE(ptrba), yvec0; #### A1r A1i A2r A2i EDUP_DY 0*SIZE(ptrbb), yvec2; @@ -2477,17 +2477,17 @@ ADD2_DY yvec6, yvec15, yvec15; ADDQ $8*SIZE, ptrba; ADDQ $16*SIZE, ptrbb; DECQ k; -JG .L231_bodyB; +JG L(231_bodyB); ALIGN_5 -.L231_loopE: +L(231_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L232_loopE; +JLE L(232_loopE); ALIGN_5 -.L232_bodyB: +L(232_bodyB): LD_DY 0*SIZE(ptrba), yvec0; #### A1r A1i A2r A2i EDUP_DY 0*SIZE(ptrbb), yvec2; @@ -2513,15 +2513,15 @@ ADD2_DY yvec6, yvec15, yvec15; ADDQ $4*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; -.L232_loopE: +L(232_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L233_loopE; +JLE L(233_loopE); ALIGN_5 -.L233_bodyB: +L(233_bodyB): LD_DY 0*SIZE(ptrba), yvec0; #### A1r A1i A2r A2i EDUP_DY 0*SIZE(ptrbb), yvec2; @@ -2536,7 +2536,7 @@ ADD2_DY yvec6, yvec15, yvec15; ADDQ $2*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L233_loopE: +L(233_loopE): #### Handle #### XOR_DY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -2583,7 +2583,7 @@ ADDQ $1, kk; #endif ADDQ $2*SIZE, C0; ADDQ $2*SIZE, C0; -.L23_loopE: +L(23_loopE): #if defined(TRMMKERNEL) && !defined(LEFT) ADDQ $2, kk; #endif @@ -2591,11 +2591,11 @@ MOVQ bk, k; SALQ $5, k; ADDQ k, bb; LEAQ (C, ldc, 2), C; -.L20_loopE: +L(20_loopE): TEST $1, bn; -JLE .L30_loopE; +JLE L(30_loopE); ALIGN_5 -.L30_bodyB: +L(30_bodyB): #if defined(TRMMKERNEL) && defined(LEFT) MOVQ OFFSET, %rax; MOVQ %rax, kk; @@ -2604,9 +2604,9 @@ MOVQ ba, ptrba; MOVQ C, C0; MOVQ bm, i; SARQ $2, i; -JLE .L31_loopE; +JLE L(31_loopE); ALIGN_5 -.L31_bodyB: +L(31_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -2634,9 +2634,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L311_loopE; +JLE L(311_loopE); ALIGN_5 -.L311_bodyB: +L(311_bodyB): LD_DY 0*SIZE(ptrba), yvec0; BROAD_DY 0*SIZE(ptrbb), yvec2; MUL_DY yvec0, yvec2, yvec6; @@ -2711,17 +2711,17 @@ ADD2_DY yvec7, yvec14, yvec14; ADDQ $32*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L311_bodyB; +JG L(311_bodyB); ALIGN_5 -.L311_loopE: +L(311_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L312_loopE; +JLE L(312_loopE); ALIGN_5 -.L312_bodyB: +L(312_bodyB): LD_DY 0*SIZE(ptrba), yvec0; BROAD_DY 0*SIZE(ptrbb), yvec2; MUL_DY yvec0, yvec2, yvec6; @@ -2760,15 +2760,15 @@ ADD2_DY yvec7, yvec14, yvec14; ADDQ $16*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L312_loopE: +L(312_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L313_loopE; +JLE L(313_loopE); ALIGN_5 -.L313_bodyB: +L(313_bodyB): LD_DY 0*SIZE(ptrba), yvec0; BROAD_DY 0*SIZE(ptrbb), yvec2; MUL_DY yvec0, yvec2, yvec6; @@ -2789,7 +2789,7 @@ ADD2_DY yvec7, yvec14, yvec14; ADDQ $8*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L313_loopE: +L(313_loopE): #### Handle #### XOR_DY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -2856,13 +2856,13 @@ ADDQ $4, kk; #endif ADDQ $8*SIZE, C0; DECQ i; -JG .L31_bodyB; +JG L(31_bodyB); ALIGN_5 -.L31_loopE: +L(31_loopE): TEST $2, bm; -JLE .L32_loopE; +JLE L(32_loopE); ALIGN_5 -.L32_bodyB: +L(32_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -2889,9 +2889,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L321_loopE; +JLE L(321_loopE); ALIGN_5 -.L321_bodyB: +L(321_bodyB): LD_DY 0*SIZE(ptrba), yvec0; BROAD_DY 0*SIZE(ptrbb), yvec2; MUL_DY yvec0, yvec2, yvec6; @@ -2930,17 +2930,17 @@ ADD2_DY yvec7, yvec15, yvec15; ADDQ $16*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L321_bodyB; +JG L(321_bodyB); ALIGN_5 -.L321_loopE: +L(321_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L322_loopE; +JLE L(322_loopE); ALIGN_5 -.L322_bodyB: +L(322_bodyB): LD_DY 0*SIZE(ptrba), yvec0; BROAD_DY 0*SIZE(ptrbb), yvec2; MUL_DY yvec0, yvec2, yvec6; @@ -2961,15 +2961,15 @@ ADD2_DY yvec7, yvec15, yvec15; ADDQ $8*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L322_loopE: +L(322_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L323_loopE; +JLE L(323_loopE); ALIGN_5 -.L323_bodyB: +L(323_bodyB): LD_DY 0*SIZE(ptrba), yvec0; BROAD_DY 0*SIZE(ptrbb), yvec2; MUL_DY yvec0, yvec2, yvec6; @@ -2980,7 +2980,7 @@ MUL_DY yvec1, yvec3, yvec7; ADD2_DY yvec7, yvec15, yvec15; ADDQ $4*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L323_loopE: +L(323_loopE): #### Handle #### XOR_DY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -3026,11 +3026,11 @@ ADDQ %rax, ptrbb; ADDQ $2, kk; #endif ADDQ $4*SIZE, C0; -.L32_loopE: +L(32_loopE): TEST $1, bm; -JLE .L33_loopE; +JLE L(33_loopE); ALIGN_5 -.L33_bodyB: +L(33_bodyB): #if !defined(TRMMKERNEL)||(defined(TRMMKERNEL)&&defined(LEFT)&&defined(TRANSA))||(defined(TRMMKERNEL)&&!defined(LEFT)&&!defined(TRANSA)) MOVQ bb,ptrbb; #else @@ -3057,9 +3057,9 @@ ADDQ $1, %rax; MOVQ %rax, kkk; #endif SARQ $2, k; -JLE .L331_loopE; +JLE L(331_loopE); ALIGN_5 -.L331_bodyB: +L(331_bodyB): LD_DX 0*SIZE(ptrba), xvec0; BROAD_DX 0*SIZE(ptrbb), xvec2; MUL_DX xvec0, xvec2, xvec2; @@ -3102,17 +3102,17 @@ ADDSUB_DX xvec3, xvec15, xvec15; ADDQ $8*SIZE, ptrba; ADDQ $8*SIZE, ptrbb; DECQ k; -JG .L331_bodyB; +JG L(331_bodyB); ALIGN_5 -.L331_loopE: +L(331_loopE): #ifndef TRMMKERNEL TEST $2, bk; #else TEST $2, kkk; #endif -JLE .L332_loopE; +JLE L(332_loopE); ALIGN_5 -.L332_bodyB: +L(332_bodyB): LD_DX 0*SIZE(ptrba), xvec0; BROAD_DX 0*SIZE(ptrbb), xvec2; MUL_DX xvec0, xvec2, xvec2; @@ -3135,15 +3135,15 @@ ADDSUB_DX xvec3, xvec15, xvec15; ADDQ $4*SIZE, ptrba; ADDQ $4*SIZE, ptrbb; -.L332_loopE: +L(332_loopE): #ifndef TRMMKERNEL TEST $1, bk; #else TEST $1, kkk; #endif -JLE .L333_loopE; +JLE L(333_loopE); ALIGN_5 -.L333_bodyB: +L(333_bodyB): LD_DX 0*SIZE(ptrba), xvec0; BROAD_DX 0*SIZE(ptrbb), xvec2; MUL_DX xvec0, xvec2, xvec2; @@ -3156,7 +3156,7 @@ ADDSUB_DX xvec3, xvec15, xvec15; ADDQ $2*SIZE, ptrba; ADDQ $2*SIZE, ptrbb; -.L333_loopE: +L(333_loopE): #### Handle #### XOR_DY yvec7, yvec7, yvec7; #if defined(RN) || defined(RT) || defined(CN) || defined(CT) @@ -3199,7 +3199,7 @@ ADDQ %rax, ptrbb; ADDQ $1, kk; #endif ADDQ $2*SIZE, C0; -.L33_loopE: +L(33_loopE): #if defined(TRMMKERNEL) && !defined(LEFT) ADDQ $1, kk; #endif @@ -3207,7 +3207,7 @@ MOVQ bk, k; SALQ $4*SIZE, k; ADDQ k, bb; LEAQ (C, ldc, 1), C; -.L30_loopE: +L(30_loopE): movq 0(%rsp), %rbx; movq 8(%rsp), %rbp; movq 16(%rsp), %r12; diff --git a/kernel/x86_64/zgemm_ncopy_1.S b/kernel/x86_64/zgemm_ncopy_1.S index 60b51f53e4..b2dbf5494a 100644 --- a/kernel/x86_64/zgemm_ncopy_1.S +++ b/kernel/x86_64/zgemm_ncopy_1.S @@ -102,19 +102,19 @@ testq N, N movq N, J - jle .L999 + jle L(999) ALIGN_4 -.L12: +L(12): movq A, AO1 addq LDA, A movq M, I sarq $2, I - jle .L14 + jle L(14) ALIGN_4 -.L13: +L(13): #ifndef DOUBLE movsd 0 * SIZE(AO1), %xmm0 movhps 2 * SIZE(AO1), %xmm0 @@ -146,16 +146,16 @@ addq $8 * SIZE, AO1 addq $8 * SIZE, B decq I - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): movq M, I andq $3, I - jle .L16 + jle L(16) ALIGN_4 -.L15: +L(15): #ifndef DOUBLE movsd 0 * SIZE(AO1), %xmm0 movsd %xmm0, 0 * SIZE(B) @@ -169,15 +169,15 @@ addq $2 * SIZE, AO1 addq $2 * SIZE, B decq I - jg .L15 + jg L(15) ALIGN_4 -.L16: +L(16): decq J - jg .L12 + jg L(12) ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 movups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/zgemm_ncopy_2.S b/kernel/x86_64/zgemm_ncopy_2.S index 8451982260..1c97f1eb60 100644 --- a/kernel/x86_64/zgemm_ncopy_2.S +++ b/kernel/x86_64/zgemm_ncopy_2.S @@ -120,20 +120,20 @@ movq N, J sarq $1, J - jle .L20 + jle L(20) ALIGN_4 -.L12: +L(12): movq A, AO1 leaq (A, LDA), AO2 leaq (A, LDA, 2), A movq M, I sarq $2, I - jle .L14 + jle L(14) ALIGN_4 -.L13: +L(13): #ifdef HAVE_3DNOW prefetchw (WPREFETCHSIZE + 0) * SIZE(B) prefetchw (WPREFETCHSIZE + 8) * SIZE(B) @@ -217,16 +217,16 @@ addq $8 * SIZE, AO2 subq $-16 * SIZE, B decq I - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): movq M, I andq $3, I - jle .L16 + jle L(16) ALIGN_4 -.L15: +L(15): #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 movhps 0 * SIZE(AO2), %xmm0 @@ -246,26 +246,26 @@ addq $2 * SIZE, AO2 addq $4 * SIZE, B decq I - jg .L15 + jg L(15) ALIGN_4 -.L16: +L(16): decq J - jg .L12 + jg L(12) ALIGN_4 -.L20: +L(20): testq $1, N - jle .L999 + jle L(999) movq A, AO1 movq M, I sarq $2, I - jle .L24 + jle L(24) ALIGN_4 -.L23: +L(23): #ifdef HAVE_3DNOW prefetchw (WPREFETCHSIZE + 0) * SIZE(B) prefetchw (WPREFETCHSIZE + 8) * SIZE(B) @@ -305,16 +305,16 @@ addq $8 * SIZE, AO1 addq $8 * SIZE, B decq I - jg .L23 + jg L(23) ALIGN_4 -.L24: +L(24): movq M, I andq $3, I - jle .L999 + jle L(999) ALIGN_4 -.L25: +L(25): #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 @@ -329,11 +329,11 @@ addq $2 * SIZE, AO1 addq $2 * SIZE, B decq I - jg .L25 + jg L(25) ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 movups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/zgemm_tcopy_1.S b/kernel/x86_64/zgemm_tcopy_1.S index 02c0614930..8d8d1afab4 100644 --- a/kernel/x86_64/zgemm_tcopy_1.S +++ b/kernel/x86_64/zgemm_tcopy_1.S @@ -102,19 +102,19 @@ testq N, N movq N, J - jle .L999 + jle L(999) ALIGN_4 -.L12: +L(12): movq A, AO1 addq $2 * SIZE, A movq M, I sarq $1, I - jle .L14 + jle L(14) ALIGN_4 -.L13: +L(13): #ifndef DOUBLE movsd 0 * SIZE(AO1), %xmm0 movhps 0 * SIZE(AO1, LDA, 1), %xmm0 @@ -140,12 +140,12 @@ leaq (AO1, LDA, 2), AO1 addq $4 * SIZE, B decq I - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): testq $1, M - jle .L16 + jle L(16) #ifndef DOUBLE movsd 0 * SIZE(AO1), %xmm0 @@ -159,12 +159,12 @@ addq $2 * SIZE, B ALIGN_4 -.L16: +L(16): decq J - jg .L12 + jg L(12) ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 movups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/zgemm_tcopy_2.S b/kernel/x86_64/zgemm_tcopy_2.S index 121bbc4283..a2f232425f 100644 --- a/kernel/x86_64/zgemm_tcopy_2.S +++ b/kernel/x86_64/zgemm_tcopy_2.S @@ -125,10 +125,10 @@ movq M, J sarq $1, J - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): movq A, AO1 leaq (A, LDA ), AO2 leaq (A, LDA, 2), A @@ -138,10 +138,10 @@ movq N, I sarq $2, I - jle .L13 + jle L(13) ALIGN_4 -.L12: +L(12): #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 movhps 2 * SIZE(AO1), %xmm0 @@ -228,12 +228,12 @@ addq $8 * SIZE, AO2 leaq (BO, M8, 4), BO decq I - jg .L12 + jg L(12) ALIGN_4 -.L13: +L(13): testq $2, N - jle .L14 + jle L(14) #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 @@ -266,9 +266,9 @@ leaq (BO, M8, 4), BO ALIGN_4 -.L14: +L(14): testq $1, N - jle .L19 + jle L(19) #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 @@ -288,27 +288,27 @@ addq $4 * SIZE, BO1 ALIGN_4 -.L19: +L(19): decq J - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M - jle .L999 + jle L(999) ALIGN_4 -.L21: +L(21): movq A, AO1 movq B, BO movq N, I sarq $2, I - jle .L23 + jle L(23) ALIGN_4 -.L22: +L(22): #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 movhps 2 * SIZE(AO1), %xmm0 @@ -360,12 +360,12 @@ addq $8 * SIZE, AO1 leaq (BO, M8, 4), BO decq I - jg .L22 + jg L(22) ALIGN_4 -.L23: +L(23): testq $2, N - jle .L24 + jle L(24) #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 @@ -386,9 +386,9 @@ leaq (BO, M8, 4), BO ALIGN_4 -.L24: +L(24): testq $1, N - jle .L999 + jle L(999) #ifndef DOUBLE movlps 0 * SIZE(AO1), %xmm0 @@ -403,7 +403,7 @@ ALIGN_4 -.L999: +L(999): #ifdef WINDOWS_ABI movups 0(%rsp), %xmm6 movups 16(%rsp), %xmm7 diff --git a/kernel/x86_64/zgemv_n.S b/kernel/x86_64/zgemv_n.S index b903cfb26b..c786cb34a9 100644 --- a/kernel/x86_64/zgemv_n.S +++ b/kernel/x86_64/zgemv_n.S @@ -161,7 +161,7 @@ movsd %xmm0,ALPHAR movsd %xmm1,ALPHAI -.L0t: +L(0t): xorq I,I addq $1,I salq $18,I @@ -169,13 +169,13 @@ movq I,M movsd ALPHAR,%xmm0 movsd ALPHAI,%xmm1 - jge .L00t + jge L(00t) movq MMM,M addq I,M - jle .L999x + jle L(999x) -.L00t: +L(00t): movq AA, A movq NN, N movq LDAX, LDA @@ -196,9 +196,9 @@ subq $-16 * SIZE, A testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) ALIGN_3 movq BUFFER, Y1 @@ -210,7 +210,7 @@ sarq $3, %rax ALIGN_3 -.L01: +L(01): movapd %xmm4, 0 * SIZE(Y1) movapd %xmm4, 2 * SIZE(Y1) movapd %xmm4, 4 * SIZE(Y1) @@ -222,22 +222,22 @@ subq $-16 * SIZE, Y1 decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L10: +L(10): #ifdef ALIGNED_ACCESS testq $SIZE, A - jne .L100 + jne L(100) #endif #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $4, N leaq 16 * SIZE(BUFFER), Y1 @@ -336,16 +336,16 @@ movq M, I sarq $2, I - jle .L15 + jle L(15) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm6) decq I - jle .L14 + jle L(14) ALIGN_3 -.L13: +L(13): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -494,10 +494,10 @@ subq $1, I BRANCH - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): pshufd $0x4e, %xmm4, %xmm5 mulpd %xmm8, %xmm4 addpd %xmm4, %xmm0 @@ -623,9 +623,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L15: +L(15): testq $2, M - je .L17 + je L(17) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm6) @@ -693,9 +693,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L17: +L(17): testq $1, M - je .L19 + je L(19) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm6) @@ -729,23 +729,23 @@ MOVUPS_YS1(-16 * SIZE, Y1, %xmm0) ALIGN_3 -.L19: +L(19): cmpq $4, N - jge .L11 + jge L(11) ALIGN_3 -.L20: +L(20): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 2 ALIGN_3 -.L21: +L(21): #endif subq $2, N @@ -817,7 +817,7 @@ movq M, I sarq $2, I - jle .L25 + jle L(25) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm6) @@ -825,10 +825,10 @@ MOVUPS_A1(-10 * SIZE, A1, %xmm10) decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A2) #endif @@ -913,10 +913,10 @@ subq $1, I BRANCH - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): pshufd $0x4e, %xmm4, %xmm5 mulpd %xmm12, %xmm4 addpd %xmm4, %xmm0 @@ -984,9 +984,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L25: +L(25): testq $2, M - je .L27 + je L(27) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm6) @@ -1026,12 +1026,12 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L27: +L(27): testq $1, M #if GEMV_UNROLL == 2 - je .L29 + je L(29) #else - je .L30 + je L(30) #endif MOVUPS_A1(-16 * SIZE, A1, %xmm4) @@ -1054,20 +1054,20 @@ #if GEMV_UNROLL == 2 ALIGN_3 -.L29: +L(29): cmpq $2, N - jge .L21 + jge L(21) #endif ALIGN_3 -.L30: +L(30): #endif cmpq $1, N - jl .L980 + jl L(980) #if GEMV_UNROLL == 1 -.L31: +L(31): decq N #endif @@ -1124,7 +1124,7 @@ movq M, I sarq $2, I - jle .L35 + jle L(35) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm6) @@ -1132,10 +1132,10 @@ MOVUPS_A1(-10 * SIZE, A1, %xmm10) decq I - jle .L34 + jle L(34) ALIGN_3 -.L33: +L(33): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -1187,10 +1187,10 @@ subq $1, I BRANCH - jg .L33 + jg L(33) ALIGN_3 -.L34: +L(34): pshufd $0x4e, %xmm4, %xmm5 mulpd %xmm12, %xmm4 addpd %xmm4, %xmm0 @@ -1229,9 +1229,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L35: +L(35): testq $2, M - je .L37 + je L(37) MOVUPS_A1(-16 * SIZE, A1, %xmm4) MOVUPS_A1(-14 * SIZE, A1, %xmm6) @@ -1256,12 +1256,12 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L37: +L(37): testq $1, M #if GEMV_UNROLL == 1 - je .L39 + je L(39) #else - je .L980 + je L(980) #endif MOVUPS_A1(-16 * SIZE, A1, %xmm4) @@ -1276,24 +1276,24 @@ #if GEMV_UNROLL == 1 ALIGN_3 -.L39: +L(39): cmpq $1, N - jge .L31 + jge L(31) #endif #ifdef ALIGNED_ACCESS - jmp .L980 + jmp L(980) ALIGN_3 -.L100: +L(100): #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L110 + jl L(110) ALIGN_3 -.L101: +L(101): subq $4, N leaq 16 * SIZE(BUFFER), Y1 @@ -1392,7 +1392,7 @@ movq M, I sarq $2, I - jle .L105 + jle L(105) movsd -16 * SIZE(A1), %xmm4 movhpd -15 * SIZE(A1), %xmm4 @@ -1400,10 +1400,10 @@ movhpd -13 * SIZE(A1), %xmm6 decq I - jle .L104 + jle L(104) ALIGN_3 -.L103: +L(103): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -1568,10 +1568,10 @@ subq $1, I BRANCH - jg .L103 + jg L(103) ALIGN_3 -.L104: +L(104): pshufd $0x4e, %xmm4, %xmm5 mulpd %xmm8, %xmm4 addpd %xmm4, %xmm0 @@ -1711,9 +1711,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L105: +L(105): testq $2, M - je .L107 + je L(107) movsd -16 * SIZE(A1), %xmm4 movhpd -15 * SIZE(A1), %xmm4 @@ -1789,9 +1789,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L107: +L(107): testq $1, M - je .L109 + je L(109) movsd -16 * SIZE(A1), %xmm4 movhpd -15 * SIZE(A1), %xmm4 @@ -1829,23 +1829,23 @@ MOVUPS_YS1(-16 * SIZE, Y1, %xmm0) ALIGN_3 -.L109: +L(109): cmpq $4, N - jge .L101 + jge L(101) ALIGN_3 -.L110: +L(110): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L120 + jl L(120) #if GEMV_UNROLL == 2 ALIGN_3 -.L111: +L(111): #endif subq $2, N @@ -1917,7 +1917,7 @@ movq M, I sarq $2, I - jle .L115 + jle L(115) movsd -16 * SIZE(A1), %xmm4 movhpd -15 * SIZE(A1), %xmm4 @@ -1929,10 +1929,10 @@ movhpd -9 * SIZE(A1), %xmm10 decq I - jle .L114 + jle L(114) ALIGN_3 -.L113: +L(113): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A2) #endif @@ -2025,10 +2025,10 @@ subq $1, I BRANCH - jg .L113 + jg L(113) ALIGN_3 -.L114: +L(114): pshufd $0x4e, %xmm4, %xmm5 mulpd %xmm12, %xmm4 addpd %xmm4, %xmm0 @@ -2100,9 +2100,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L115: +L(115): testq $2, M - je .L117 + je L(117) movsd -16 * SIZE(A1), %xmm4 movhpd -15 * SIZE(A1), %xmm4 @@ -2147,12 +2147,12 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L117: +L(117): testq $1, M #if GEMV_UNROLL == 2 - je .L119 + je L(119) #else - je .L120 + je L(120) #endif movsd -16 * SIZE(A1), %xmm4 @@ -2177,20 +2177,20 @@ #if GEMV_UNROLL == 2 ALIGN_3 -.L119: +L(119): cmpq $2, N - jge .L111 + jge L(111) #endif ALIGN_3 -.L120: +L(120): #endif cmpq $1, N - jl .L980 + jl L(980) #if GEMV_UNROLL == 1 -.L121: +L(121): decq N #endif @@ -2247,7 +2247,7 @@ movq M, I sarq $2, I - jle .L125 + jle L(125) movsd -16 * SIZE(A1), %xmm4 movhpd -15 * SIZE(A1), %xmm4 @@ -2259,10 +2259,10 @@ movhpd -9 * SIZE(A1), %xmm10 decq I - jle .L124 + jle L(124) ALIGN_3 -.L123: +L(123): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -2318,10 +2318,10 @@ subq $1, I BRANCH - jg .L123 + jg L(123) ALIGN_3 -.L124: +L(124): pshufd $0x4e, %xmm4, %xmm5 mulpd %xmm12, %xmm4 addpd %xmm4, %xmm0 @@ -2360,9 +2360,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L125: +L(125): testq $2, M - je .L127 + je L(127) movsd -16 * SIZE(A1), %xmm4 @@ -2390,12 +2390,12 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L127: +L(127): testq $1, M #if GEMV_UNROLL == 1 - je .L129 + je L(129) #else - je .L980 + je L(980) #endif movsd -16 * SIZE(A1), %xmm4 @@ -2411,27 +2411,27 @@ #if GEMV_UNROLL == 1 ALIGN_3 -.L129: +L(129): cmpq $1, N - jge .L121 + jge L(121) #endif #endif ALIGN_3 -.L980: +L(980): testq $SIZE, Y - jne .L990 + jne L(990) movq Y, Y1 movq M, %rax sarq $3, %rax - jle .L184 + jle L(184) ALIGN_3 -.L182: +L(182): movapd (Y), %xmm0 addq INCY, Y movapd (Y), %xmm1 @@ -2477,15 +2477,15 @@ subq $-16 * SIZE, BUFFER decq %rax - jg .L182 + jg L(182) ALIGN_3 -.L184: +L(184): testq $7, M - jle .L999 + jle L(999) testq $4, M - jle .L185 + jle L(185) movapd (Y), %xmm0 addq INCY, Y @@ -2513,9 +2513,9 @@ addq $8 * SIZE, BUFFER ALIGN_3 -.L185: +L(185): testq $2, M - jle .L186 + jle L(186) movapd (Y), %xmm0 addq INCY, Y @@ -2532,27 +2532,27 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L186: +L(186): testq $1, M - jle .L999 + jle L(999) movapd (Y), %xmm0 addpd (BUFFER), %xmm0 movapd %xmm0, (Y1) - jmp .L999 + jmp L(999) ALIGN_3 -.L990: +L(990): movq Y, Y1 movq M, %rax sarq $3, %rax - jle .L994 + jle L(994) ALIGN_3 -.L992: +L(992): movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 addq INCY, Y @@ -2628,15 +2628,15 @@ subq $-16 * SIZE, BUFFER decq %rax - jg .L992 + jg L(992) ALIGN_3 -.L994: +L(994): testq $7, M - jle .L999 + jle L(999) testq $4, M - jle .L995 + jle L(995) movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 @@ -2678,9 +2678,9 @@ addq $8 * SIZE, BUFFER ALIGN_3 -.L995: +L(995): testq $2, M - jle .L996 + jle L(996) movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 @@ -2704,9 +2704,9 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L996: +L(996): testq $1, M - jle .L999 + jle L(999) movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 @@ -2717,12 +2717,12 @@ movhpd %xmm0, 1 * SIZE(Y1) ALIGN_3 -.L999: +L(999): movq M, I salq $ZBASE_SHIFT,I addq I,AA - jmp .L0t -.L999x: + jmp L(0t) +L(999x): movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/zgemv_n_atom.S b/kernel/x86_64/zgemv_n_atom.S index 4fa70925ee..df9c786be8 100644 --- a/kernel/x86_64/zgemv_n_atom.S +++ b/kernel/x86_64/zgemv_n_atom.S @@ -175,9 +175,9 @@ subq $-16 * SIZE, A testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) ALIGN_3 movq BUFFER, Y1 @@ -189,7 +189,7 @@ sarq $3, %rax ALIGN_3 -.L01: +L(01): movapd %xmm4, 0 * SIZE(Y1) movapd %xmm4, 2 * SIZE(Y1) movapd %xmm4, 4 * SIZE(Y1) @@ -201,16 +201,16 @@ subq $-16 * SIZE, Y1 decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L10: +L(10): movq N, J sarq $1, J - jle .L20 + jle L(20) ALIGN_3 -.L11: +L(11): leaq 16 * SIZE(BUFFER), Y1 movq A, A1 leaq (A, LDA, 1), A2 @@ -257,7 +257,7 @@ movq M, I sarq $2, I - jle .L15 + jle L(15) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -276,10 +276,10 @@ ADD2 %xmm12, %xmm1 decq I - jle .L14 + jle L(14) ALIGN_3 -.L13: +L(13): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(A2) #endif @@ -422,10 +422,10 @@ subq $1, I BRANCH - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): movapd %xmm9, %xmm12 mulsd %xmm5, %xmm9 ADD1 %xmm10, %xmm2 @@ -546,9 +546,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L15: +L(15): testq $2, M - je .L17 + je L(17) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -622,9 +622,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L17: +L(17): testq $1, M - je .L19 + je L(19) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -660,14 +660,14 @@ movlpd %xmm1, -15 * SIZE(Y1) ALIGN_3 -.L19: +L(19): decq J - jg .L11 + jg L(11) ALIGN_3 -.L20: +L(20): testq $1, N - jle .L90 + jle L(90) leaq 16 * SIZE(BUFFER), Y1 movq A, A1 @@ -698,7 +698,7 @@ movq M, I sarq $2, I - jle .L25 + jle L(25) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -717,10 +717,10 @@ ADD2 %xmm12, %xmm1 decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(A2) #endif @@ -803,10 +803,10 @@ subq $1, I BRANCH - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): movapd %xmm9, %xmm12 mulsd %xmm5, %xmm9 ADD1 %xmm10, %xmm2 @@ -871,9 +871,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L25: +L(25): testq $2, M - je .L27 + je L(27) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -916,9 +916,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L27: +L(27): testq $1, M - je .L90 + je L(90) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -940,15 +940,15 @@ movlpd %xmm1, -15 * SIZE(Y1) ALIGN_3 -.L90: +L(90): movq Y, Y1 movq M, %rax sarq $3, %rax - jle .L94 + jle L(94) ALIGN_3 -.L92: +L(92): movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 addq INCY, Y @@ -1024,15 +1024,15 @@ subq $-16 * SIZE, BUFFER decq %rax - jg .L92 + jg L(92) ALIGN_3 -.L94: +L(94): testq $7, M - jle .L999 + jle L(999) testq $4, M - jle .L95 + jle L(95) movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 @@ -1074,9 +1074,9 @@ addq $8 * SIZE, BUFFER ALIGN_3 -.L95: +L(95): testq $2, M - jle .L96 + jle L(96) movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 @@ -1100,9 +1100,9 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L96: +L(96): testq $1, M - jle .L999 + jle L(999) movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 @@ -1113,7 +1113,7 @@ movhpd %xmm0, 1 * SIZE(Y1) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemv_n_dup.S b/kernel/x86_64/zgemv_n_dup.S index 42c1963c21..7fb28be4b9 100644 --- a/kernel/x86_64/zgemv_n_dup.S +++ b/kernel/x86_64/zgemv_n_dup.S @@ -151,9 +151,9 @@ subq $-16 * SIZE, A testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) ALIGN_3 movq BUFFER, Y1 @@ -165,7 +165,7 @@ sarq $3, %rax ALIGN_3 -.L01: +L(01): movaps %xmm4, 0 * SIZE(Y1) movaps %xmm4, 2 * SIZE(Y1) movaps %xmm4, 4 * SIZE(Y1) @@ -177,17 +177,17 @@ subq $-16 * SIZE, Y1 decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L10: +L(10): #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $4, N leaq 16 * SIZE(BUFFER), Y1 @@ -270,7 +270,7 @@ movq M, I sarq $2, I - jle .L15 + jle L(15) movddup -16 * SIZE(A1), %xmm4 movddup -14 * SIZE(A1), %xmm5 @@ -278,10 +278,10 @@ movddup -10 * SIZE(A1), %xmm7 decq I - jle .L14 + jle L(14) ALIGN_3 -.L13: +L(13): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -422,10 +422,10 @@ subq $1, I BRANCH - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): mulpd %xmm8, %xmm4 addpd %xmm4, %xmm0 movddup -15 * SIZE(A1), %xmm4 @@ -541,9 +541,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L15: +L(15): testq $2, M - je .L17 + je L(17) movddup -16 * SIZE(A1), %xmm4 movddup -15 * SIZE(A1), %xmm5 @@ -611,9 +611,9 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L17: +L(17): testq $1, M - je .L19 + je L(19) movddup -16 * SIZE(A1), %xmm4 movddup -15 * SIZE(A1), %xmm5 @@ -647,23 +647,23 @@ MOVUPS_YS1(-16 * SIZE, Y1, %xmm0) ALIGN_3 -.L19: +L(19): cmpq $4, N - jge .L11 + jge L(11) ALIGN_3 -.L20: +L(20): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 2 ALIGN_3 -.L21: +L(21): #endif subq $2, N @@ -726,7 +726,7 @@ movq M, I sarq $2, I - jle .L25 + jle L(25) movddup -16 * SIZE(A1), %xmm4 movddup -14 * SIZE(A1), %xmm5 @@ -734,10 +734,10 @@ movddup -10 * SIZE(A1), %xmm7 decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A2) #endif @@ -818,10 +818,10 @@ subq $1, I BRANCH - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): mulpd %xmm8, %xmm4 addpd %xmm4, %xmm0 movddup -15 * SIZE(A1), %xmm4 @@ -885,9 +885,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L25: +L(25): testq $2, M - je .L27 + je L(27) movddup -16 * SIZE(A1), %xmm4 movddup -15 * SIZE(A1), %xmm5 @@ -927,12 +927,12 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L27: +L(27): testq $1, M #if GEMV_UNROLL == 2 - je .L29 + je L(29) #else - je .L30 + je L(30) #endif movddup -16 * SIZE(A1), %xmm4 @@ -955,20 +955,20 @@ #if GEMV_UNROLL == 2 ALIGN_3 -.L29: +L(29): cmpq $2, N - jge .L21 + jge L(21) #endif ALIGN_3 -.L30: +L(30): #endif cmpq $1, N - jl .L980 + jl L(980) #if GEMV_UNROLL == 1 -.L31: +L(31): decq N #endif @@ -1021,7 +1021,7 @@ movq M, I sarq $2, I - jle .L35 + jle L(35) movddup -16 * SIZE(A1), %xmm4 movddup -14 * SIZE(A1), %xmm5 @@ -1029,10 +1029,10 @@ movddup -10 * SIZE(A1), %xmm7 decq I - jle .L34 + jle L(34) ALIGN_3 -.L33: +L(33): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -1082,10 +1082,10 @@ subq $1, I BRANCH - jg .L33 + jg L(33) ALIGN_3 -.L34: +L(34): mulpd %xmm8, %xmm4 addpd %xmm4, %xmm0 movddup -15 * SIZE(A1), %xmm4 @@ -1122,9 +1122,9 @@ subq $-8 * SIZE, Y1 ALIGN_3 -.L35: +L(35): testq $2, M - je .L37 + je L(37) movddup -16 * SIZE(A1), %xmm4 movddup -15 * SIZE(A1), %xmm5 @@ -1149,12 +1149,12 @@ addq $4 * SIZE, Y1 ALIGN_3 -.L37: +L(37): testq $1, M #if GEMV_UNROLL == 1 - je .L39 + je L(39) #else - je .L980 + je L(980) #endif movddup -16 * SIZE(A1), %xmm4 @@ -1169,23 +1169,23 @@ #if GEMV_UNROLL == 1 ALIGN_3 -.L39: +L(39): cmpq $1, N - jge .L31 + jge L(31) #endif -.L980: +L(980): testq $SIZE, Y - jne .L990 + jne L(990) movq Y, Y1 movq M, %rax sarq $3, %rax - jle .L184 + jle L(184) ALIGN_3 -.L182: +L(182): movaps (Y), %xmm0 addq INCY, Y movaps (Y), %xmm1 @@ -1231,15 +1231,15 @@ subq $-16 * SIZE, BUFFER decq %rax - jg .L182 + jg L(182) ALIGN_3 -.L184: +L(184): testq $7, M - jle .L999 + jle L(999) testq $4, M - jle .L185 + jle L(185) movaps (Y), %xmm0 addq INCY, Y @@ -1267,9 +1267,9 @@ addq $8 * SIZE, BUFFER ALIGN_3 -.L185: +L(185): testq $2, M - jle .L186 + jle L(186) movaps (Y), %xmm0 addq INCY, Y @@ -1286,27 +1286,27 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L186: +L(186): testq $1, M - jle .L999 + jle L(999) movaps (Y), %xmm0 addpd (BUFFER), %xmm0 movaps %xmm0, (Y1) - jmp .L999 + jmp L(999) ALIGN_3 -.L990: +L(990): movq Y, Y1 movq M, %rax sarq $3, %rax - jle .L994 + jle L(994) ALIGN_3 -.L992: +L(992): movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 addq INCY, Y @@ -1382,15 +1382,15 @@ subq $-16 * SIZE, BUFFER decq %rax - jg .L992 + jg L(992) ALIGN_3 -.L994: +L(994): testq $7, M - jle .L999 + jle L(999) testq $4, M - jle .L995 + jle L(995) movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 @@ -1432,9 +1432,9 @@ addq $8 * SIZE, BUFFER ALIGN_3 -.L995: +L(995): testq $2, M - jle .L996 + jle L(996) movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 @@ -1458,9 +1458,9 @@ addq $4 * SIZE, BUFFER ALIGN_3 -.L996: +L(996): testq $1, M - jle .L999 + jle L(999) movsd 0 * SIZE(Y), %xmm0 movhpd 1 * SIZE(Y), %xmm0 @@ -1471,7 +1471,7 @@ movhpd %xmm0, 1 * SIZE(Y1) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemv_t.S b/kernel/x86_64/zgemv_t.S index c78927953f..1f9cbada1a 100644 --- a/kernel/x86_64/zgemv_t.S +++ b/kernel/x86_64/zgemv_t.S @@ -156,7 +156,7 @@ movsd %xmm0,ALPHAR movsd %xmm1,ALPHAI -.L0t: +L(0t): xorq I,I addq $1,I salq $19,I @@ -164,13 +164,13 @@ movq I,M movsd ALPHAR,%xmm0 movsd ALPHAI,%xmm1 - jge .L00t + jge L(00t) movq MMM,M addq I,M - jle .L999x + jle L(999x) -.L00t: +L(00t): movq AA, A movq NN, N movq LDAX, LDA @@ -195,9 +195,9 @@ subq $-16 * SIZE, A testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) ALIGN_3 movq BUFFER, X1 @@ -206,10 +206,10 @@ movq M, I sarq $2, I - jle .L05 + jle L(05) ALIGN_4 -.L02: +L(02): movsd 0 * SIZE(X), %xmm0 movhpd 1 * SIZE(X), %xmm0 addq INCX, X @@ -233,38 +233,38 @@ addq $8 * SIZE, X1 decq I - jg .L02 + jg L(02) ALIGN_4 -.L05: +L(05): movq M, I andq $3, I - jle .L10 + jle L(10) ALIGN_2 -.L06: +L(06): movsd 0 * SIZE(X), %xmm0 movhpd 1 * SIZE(X), %xmm0 addq INCX, X movapd %xmm0, 0 * SIZE(X1) addq $2 * SIZE, X1 decq I - jg .L06 + jg L(06) ALIGN_4 -.L10: +L(10): #ifdef ALIGNED_ACCESS testq $SIZE, A - jne .L100 + jne L(100) #endif #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $4, N leaq 16 * SIZE(BUFFER), X1 @@ -290,16 +290,16 @@ movq M, I sarq $2, I - jle .L15 + jle L(15) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm10) decq I - jle .L14 + jle L(14) ALIGN_3 -.L13: +L(13): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -442,10 +442,10 @@ subq $1, I BRANCH - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): pshufd $0x4e, %xmm8, %xmm9 mulpd %xmm12, %xmm8 addpd %xmm8, %xmm0 @@ -565,9 +565,9 @@ subq $-8 * SIZE, X1 ALIGN_3 -.L15: +L(15): testq $2, M - je .L17 + je L(17) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm10) @@ -631,9 +631,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L17: +L(17): testq $1, M - je .L19 + je L(19) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A2(-16 * SIZE, A1, LDA, 1, %xmm10) @@ -665,7 +665,7 @@ SUBPD %xmm11, %xmm7 ALIGN_3 -.L19: +L(19): pcmpeqb %xmm13, %xmm13 psllq $63, %xmm13 shufps $0xc0, %xmm13, %xmm13 @@ -768,21 +768,21 @@ addq INCY, Y1 cmpq $4, N - jge .L11 + jge L(11) ALIGN_3 -.L20: +L(20): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 2 ALIGN_3 -.L21: +L(21): #endif subq $2, N @@ -807,7 +807,7 @@ movq M, I sarq $2, I - jle .L25 + jle L(25) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A1(-16 * SIZE, A2, %xmm10) @@ -815,10 +815,10 @@ MOVUPS_A1(-14 * SIZE, A2, %xmm6) decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -901,10 +901,10 @@ subq $1, I BRANCH - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): pshufd $0x4e, %xmm8, %xmm9 mulpd %xmm4, %xmm8 addpd %xmm8, %xmm0 @@ -970,9 +970,9 @@ subq $-8 * SIZE, X1 ALIGN_3 -.L25: +L(25): testq $2, M - je .L27 + je L(27) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A1(-16 * SIZE, A2, %xmm10) @@ -1010,9 +1010,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L27: +L(27): testq $1, M - je .L29 + je L(29) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A1(-16 * SIZE, A2, %xmm10) @@ -1030,7 +1030,7 @@ SUBPD %xmm11, %xmm3 ALIGN_3 -.L29: +L(29): pcmpeqb %xmm11, %xmm11 psllq $63, %xmm11 shufps $0xc0, %xmm11, %xmm11 @@ -1092,18 +1092,18 @@ #if GEMV_UNROLL == 2 cmpq $2, N - jge .L21 + jge L(21) #endif ALIGN_3 -.L30: +L(30): #endif cmpq $1, N - jl .L999 + jl L(999) #if GEMV_UNROLL == 1 -.L31: +L(31): decq N #endif @@ -1122,16 +1122,16 @@ movq M, I sarq $2, I - jle .L35 + jle L(35) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A1(-14 * SIZE, A1, %xmm12) decq I - jle .L34 + jle L(34) ALIGN_3 -.L33: +L(33): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -1181,10 +1181,10 @@ subq $1, I BRANCH - jg .L33 + jg L(33) ALIGN_3 -.L34: +L(34): pshufd $0x4e, %xmm8, %xmm9 mulpd %xmm4, %xmm8 addpd %xmm8, %xmm0 @@ -1223,9 +1223,9 @@ subq $-8 * SIZE, X1 ALIGN_3 -.L35: +L(35): testq $2, M - je .L37 + je L(37) MOVUPS_A1(-16 * SIZE, A1, %xmm8) MOVUPS_A1(-14 * SIZE, A1, %xmm12) @@ -1247,9 +1247,9 @@ addq $4 * SIZE, A1 ALIGN_3 -.L37: +L(37): testq $1, M - je .L39 + je L(39) MOVUPS_A1(-16 * SIZE, A1, %xmm8) @@ -1260,7 +1260,7 @@ SUBPD %xmm9, %xmm1 ALIGN_3 -.L39: +L(39): pcmpeqb %xmm11, %xmm11 psllq $63, %xmm11 shufps $0xc0, %xmm11, %xmm11 @@ -1303,21 +1303,21 @@ addq INCY, Y1 cmpq $1, N - jge .L31 + jge L(31) #endif #ifdef ALIGNED_ACCESS - jmp .L999 + jmp L(999) ALIGN_3 -.L100: +L(100): #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L110 + jl L(110) ALIGN_3 -.L101: +L(101): subq $4, N leaq 16 * SIZE(BUFFER), X1 @@ -1343,7 +1343,7 @@ movq M, I sarq $2, I - jle .L105 + jle L(105) movsd -16 * SIZE(A1), %xmm8 movhpd -15 * SIZE(A1), %xmm8 @@ -1352,10 +1352,10 @@ movhpd -15 * SIZE(A1, LDA), %xmm10 decq I - jle .L104 + jle L(104) ALIGN_3 -.L103: +L(103): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -1514,10 +1514,10 @@ subq $1, I BRANCH - jg .L103 + jg L(103) ALIGN_3 -.L104: +L(104): pshufd $0x4e, %xmm8, %xmm9 mulpd %xmm12, %xmm8 addpd %xmm8, %xmm0 @@ -1651,9 +1651,9 @@ subq $-8 * SIZE, X1 ALIGN_3 -.L105: +L(105): testq $2, M - je .L107 + je L(107) movsd -16 * SIZE(A1), %xmm8 movhpd -15 * SIZE(A1), %xmm8 @@ -1726,9 +1726,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L107: +L(107): testq $1, M - je .L109 + je L(109) movsd -16 * SIZE(A1), %xmm8 movhpd -15 * SIZE(A1), %xmm8 @@ -1765,7 +1765,7 @@ SUBPD %xmm11, %xmm7 ALIGN_3 -.L109: +L(109): pcmpeqb %xmm13, %xmm13 psllq $63, %xmm13 shufps $0xc0, %xmm13, %xmm13 @@ -1868,21 +1868,21 @@ addq INCY, Y1 cmpq $4, N - jge .L101 + jge L(101) ALIGN_3 -.L110: +L(110): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L120 + jl L(120) #if GEMV_UNROLL == 2 ALIGN_3 -.L111: +L(111): #endif subq $2, N @@ -1907,7 +1907,7 @@ movq M, I sarq $2, I - jle .L115 + jle L(115) movsd -16 * SIZE(A1), %xmm8 movhpd -15 * SIZE(A1), %xmm8 @@ -1920,10 +1920,10 @@ movhpd -13 * SIZE(A2), %xmm6 decq I - jle .L114 + jle L(114) ALIGN_3 -.L113: +L(113): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -2014,10 +2014,10 @@ subq $1, I BRANCH - jg .L113 + jg L(113) ALIGN_3 -.L114: +L(114): pshufd $0x4e, %xmm8, %xmm9 mulpd %xmm4, %xmm8 addpd %xmm8, %xmm0 @@ -2087,9 +2087,9 @@ subq $-8 * SIZE, X1 ALIGN_3 -.L115: +L(115): testq $2, M - je .L117 + je L(117) movsd -16 * SIZE(A1), %xmm8 movhpd -15 * SIZE(A1), %xmm8 @@ -2131,9 +2131,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L117: +L(117): testq $1, M - je .L119 + je L(119) movsd -16 * SIZE(A1), %xmm8 movhpd -15 * SIZE(A1), %xmm8 @@ -2153,7 +2153,7 @@ SUBPD %xmm11, %xmm3 ALIGN_3 -.L119: +L(119): pcmpeqb %xmm11, %xmm11 psllq $63, %xmm11 shufps $0xc0, %xmm11, %xmm11 @@ -2215,18 +2215,18 @@ #if GEMV_UNROLL == 2 cmpq $2, N - jge .L111 + jge L(111) #endif ALIGN_3 -.L120: +L(120): #endif cmpq $1, N - jl .L999 + jl L(999) #if GEMV_UNROLL == 1 -.L121: +L(121): decq N #endif @@ -2245,7 +2245,7 @@ movq M, I sarq $2, I - jle .L125 + jle L(125) movsd -16 * SIZE(A1), %xmm8 movhpd -15 * SIZE(A1), %xmm8 @@ -2253,10 +2253,10 @@ movhpd -13 * SIZE(A1), %xmm12 decq I - jle .L124 + jle L(124) ALIGN_3 -.L123: +L(123): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -2310,10 +2310,10 @@ subq $1, I BRANCH - jg .L123 + jg L(123) ALIGN_3 -.L124: +L(124): pshufd $0x4e, %xmm8, %xmm9 mulpd %xmm4, %xmm8 addpd %xmm8, %xmm0 @@ -2354,9 +2354,9 @@ subq $-8 * SIZE, X1 ALIGN_3 -.L125: +L(125): testq $2, M - je .L127 + je L(127) movsd -16 * SIZE(A1), %xmm8 movhpd -15 * SIZE(A1), %xmm8 @@ -2380,9 +2380,9 @@ addq $4 * SIZE, A1 ALIGN_3 -.L127: +L(127): testq $1, M - je .L129 + je L(129) movsd -16 * SIZE(A1), %xmm8 movhpd -15 * SIZE(A1), %xmm8 @@ -2394,7 +2394,7 @@ SUBPD %xmm9, %xmm1 ALIGN_3 -.L129: +L(129): pcmpeqb %xmm11, %xmm11 psllq $63, %xmm11 shufps $0xc0, %xmm11, %xmm11 @@ -2437,19 +2437,19 @@ addq INCY, Y1 cmpq $1, N - jge .L121 + jge L(121) #endif #endif ALIGN_3 -.L999: +L(999): movq M, I salq $ZBASE_SHIFT,I addq I,AA - jmp .L0t -.L999x: + jmp L(0t) +L(999x): movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/zgemv_t_atom.S b/kernel/x86_64/zgemv_t_atom.S index 73a013a245..501014188d 100644 --- a/kernel/x86_64/zgemv_t_atom.S +++ b/kernel/x86_64/zgemv_t_atom.S @@ -176,9 +176,9 @@ subq $-16 * SIZE, A testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) ALIGN_3 movq BUFFER, X1 @@ -187,10 +187,10 @@ movq M, I sarq $2, I - jle .L05 + jle L(05) ALIGN_4 -.L02: +L(02): movsd 0 * SIZE(X), %xmm0 movhpd 1 * SIZE(X), %xmm0 addq INCX, X @@ -214,32 +214,32 @@ addq $8 * SIZE, X1 decq I - jg .L02 + jg L(02) ALIGN_4 -.L05: +L(05): movq M, I andq $3, I - jle .L10 + jle L(10) ALIGN_2 -.L06: +L(06): movsd 0 * SIZE(X), %xmm0 movhpd 1 * SIZE(X), %xmm0 addq INCX, X movapd %xmm0, 0 * SIZE(X1) addq $2 * SIZE, X1 decq I - jg .L06 + jg L(06) ALIGN_4 -.L10: +L(10): movq N, J sarq $1, J - jle .L20 + jle L(20) ALIGN_3 -.L11: +L(11): leaq 16 * SIZE(BUFFER), X1 movq A, A1 @@ -262,7 +262,7 @@ movq M, I sarq $2, I - jle .L15 + jle L(15) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -274,10 +274,10 @@ mulsd %xmm5, %xmm12 decq I - jle .L14 + jle L(14) ALIGN_3 -.L13: +L(13): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(A1) #endif @@ -413,10 +413,10 @@ subq $1, I BRANCH - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): movapd %xmm9, %xmm13 mulsd %xmm5, %xmm9 ADD1 %xmm8, %xmm0 @@ -537,9 +537,9 @@ subq $-8 * SIZE, X1 ALIGN_3 -.L15: +L(15): testq $2, M - je .L17 + je L(17) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -606,9 +606,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L17: +L(17): testq $1, M - je .L19 + je L(19) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -641,7 +641,7 @@ ADD4 %xmm13, %xmm3 ALIGN_3 -.L19: +L(19): movsd 0 * SIZE(Y), %xmm4 movapd %xmm0, %xmm10 mulsd ALPHA_R, %xmm0 @@ -681,12 +681,12 @@ addq INCY, Y1 decq J - jg .L11 + jg L(11) ALIGN_3 -.L20: +L(20): testq $1, N - jle .L999 + jle L(999) leaq 16 * SIZE(BUFFER), X1 @@ -702,7 +702,7 @@ movq M, I sarq $2, I - jle .L25 + jle L(25) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -712,10 +712,10 @@ mulsd %xmm5, %xmm12 decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(A1) #endif @@ -790,10 +790,10 @@ subq $1, I BRANCH - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): movapd %xmm9, %xmm13 mulsd %xmm5, %xmm9 movsd -11 * SIZE(X1), %xmm5 @@ -858,9 +858,9 @@ subq $-8 * SIZE, X1 ALIGN_3 -.L25: +L(25): testq $2, M - je .L27 + je L(27) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -898,9 +898,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L27: +L(27): testq $1, M - je .L29 + je L(29) movsd -16 * SIZE(A1), %xmm8 movsd -15 * SIZE(A1), %xmm9 @@ -919,7 +919,7 @@ ADD4 %xmm13, %xmm1 ALIGN_3 -.L29: +L(29): movsd 0 * SIZE(Y), %xmm4 movapd %xmm0, %xmm10 mulsd ALPHA_R, %xmm0 @@ -939,7 +939,7 @@ movlpd %xmm1, 1 * SIZE(Y1) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zgemv_t_dup.S b/kernel/x86_64/zgemv_t_dup.S index d509f0eb8d..6218b5a3a0 100644 --- a/kernel/x86_64/zgemv_t_dup.S +++ b/kernel/x86_64/zgemv_t_dup.S @@ -158,9 +158,9 @@ subq $-16 * SIZE, A testq M, M - jle .L999 + jle L(999) testq N, N - jle .L999 + jle L(999) ALIGN_3 movq BUFFER, X1 @@ -169,10 +169,10 @@ movq M, I sarq $2, I - jle .L05 + jle L(05) ALIGN_4 -.L02: +L(02): movsd 0 * SIZE(X), %xmm0 movhpd 1 * SIZE(X), %xmm0 addq INCX, X @@ -196,33 +196,33 @@ addq $8 * SIZE, X1 decq I - jg .L02 + jg L(02) ALIGN_4 -.L05: +L(05): movq M, I andq $3, I - jle .L10 + jle L(10) ALIGN_2 -.L06: +L(06): movsd 0 * SIZE(X), %xmm0 movhpd 1 * SIZE(X), %xmm0 addq INCX, X movapd %xmm0, 0 * SIZE(X1) addq $2 * SIZE, X1 decq I - jg .L06 + jg L(06) ALIGN_4 -.L10: +L(10): #if GEMV_UNROLL >= 4 cmpq $4, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): subq $4, N leaq 16 * SIZE(BUFFER), X1 @@ -248,7 +248,7 @@ movq M, I sarq $2, I - jle .L15 + jle L(15) movddup -16 * SIZE(A1), %xmm8 movddup -15 * SIZE(A1), %xmm9 @@ -256,10 +256,10 @@ movddup -15 * SIZE(A1, LDA), %xmm11 decq I - jle .L14 + jle L(14) ALIGN_3 -.L13: +L(13): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) - 128 + PREOFFSET(A1) #endif @@ -394,10 +394,10 @@ subq $1, I BRANCH - jg .L13 + jg L(13) ALIGN_3 -.L14: +L(14): mulpd %xmm12, %xmm8 addpd %xmm8, %xmm0 movddup -16 * SIZE(A2), %xmm8 @@ -507,9 +507,9 @@ subq $-8 * SIZE, X1 ALIGN_3 -.L15: +L(15): testq $2, M - je .L17 + je L(17) movddup -16 * SIZE(A1), %xmm8 movddup -15 * SIZE(A1), %xmm9 @@ -570,9 +570,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L17: +L(17): testq $1, M - je .L19 + je L(19) movddup -16 * SIZE(A1), %xmm8 movddup -15 * SIZE(A1), %xmm9 @@ -602,7 +602,7 @@ addpd %xmm11, %xmm7 ALIGN_3 -.L19: +L(19): pcmpeqb %xmm13, %xmm13 psllq $63, %xmm13 shufps $0x40, %xmm13, %xmm13 @@ -691,21 +691,21 @@ addq INCY, Y1 cmpq $4, N - jge .L11 + jge L(11) ALIGN_3 -.L20: +L(20): #endif #if GEMV_UNROLL >= 2 cmpq $2, N - jl .L30 + jl L(30) #if GEMV_UNROLL == 2 ALIGN_3 -.L21: +L(21): #endif subq $2, N @@ -729,7 +729,7 @@ movq M, I sarq $2, I - jle .L25 + jle L(25) movddup -16 * SIZE(A1), %xmm8 movddup -15 * SIZE(A1), %xmm9 @@ -737,10 +737,10 @@ movddup -15 * SIZE(A1, LDA), %xmm11 decq I - jle .L24 + jle L(24) ALIGN_3 -.L23: +L(23): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1) #endif @@ -815,10 +815,10 @@ subq $1, I BRANCH - jg .L23 + jg L(23) ALIGN_3 -.L24: +L(24): mulpd %xmm12, %xmm8 addpd %xmm8, %xmm0 movddup -14 * SIZE(A1), %xmm8 @@ -876,9 +876,9 @@ subq $-8 * SIZE, X1 ALIGN_3 -.L25: +L(25): testq $2, M - je .L27 + je L(27) movddup -16 * SIZE(A1), %xmm8 movddup -15 * SIZE(A1), %xmm9 @@ -912,9 +912,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L27: +L(27): testq $1, M - je .L29 + je L(29) movddup -16 * SIZE(A1), %xmm8 movddup -15 * SIZE(A1), %xmm9 @@ -931,7 +931,7 @@ addpd %xmm11, %xmm3 ALIGN_3 -.L29: +L(29): pcmpeqb %xmm13, %xmm13 psllq $63, %xmm13 shufps $0x40, %xmm13, %xmm13 @@ -987,18 +987,18 @@ #if GEMV_UNROLL == 2 cmpq $2, N - jge .L21 + jge L(21) #endif ALIGN_3 -.L30: +L(30): #endif cmpq $1, N - jl .L999 + jl L(999) #if GEMV_UNROLL == 1 -.L31: +L(31): decq N #endif @@ -1018,7 +1018,7 @@ movq M, I sarq $2, I - jle .L35 + jle L(35) movddup -16 * SIZE(A1), %xmm8 movddup -15 * SIZE(A1), %xmm9 @@ -1026,10 +1026,10 @@ movddup -13 * SIZE(A1), %xmm11 decq I - jle .L34 + jle L(34) ALIGN_3 -.L33: +L(33): #ifdef PREFETCH PREFETCH (PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1) #endif @@ -1075,10 +1075,10 @@ subq $1, I BRANCH - jg .L33 + jg L(33) ALIGN_3 -.L34: +L(34): mulpd %xmm12, %xmm8 addpd %xmm8, %xmm0 movddup -12 * SIZE(A1), %xmm8 @@ -1111,9 +1111,9 @@ subq $-8 * SIZE, X1 ALIGN_3 -.L35: +L(35): testq $2, M - je .L37 + je L(37) movddup -16 * SIZE(A1), %xmm8 movddup -15 * SIZE(A1), %xmm9 @@ -1134,9 +1134,9 @@ addq $4 * SIZE, A1 ALIGN_3 -.L37: +L(37): testq $1, M - je .L39 + je L(39) movddup -16 * SIZE(A1), %xmm8 movddup -15 * SIZE(A1), %xmm9 @@ -1147,7 +1147,7 @@ addpd %xmm9, %xmm1 ALIGN_3 -.L39: +L(39): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -1190,11 +1190,11 @@ addq INCY, Y1 cmpq $1, N - jge .L31 + jge L(31) #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/znrm2.S b/kernel/x86_64/znrm2.S index 748fde3102..13cc8ab5c2 100644 --- a/kernel/x86_64/znrm2.S +++ b/kernel/x86_64/znrm2.S @@ -56,9 +56,9 @@ fldz testq M, M - jle .L999 + jle L(999) testq INCX, INCX - je .L999 + je L(999) salq $ZBASE_SHIFT, INCX @@ -66,14 +66,14 @@ fldz fldz cmpq $SIZE * 2, INCX - jne .L40 + jne L(40) movq M, I sarq $2, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #if defined(PREFETCH) PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -109,16 +109,16 @@ faddp %st, %st(1) decq I - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): andq $3, M - jle .L998 + jle L(998) ALIGN_4 -.L21: +L(21): FLD 0 * SIZE(X) fmul %st(0), %st FLD 1 * SIZE(X) @@ -127,17 +127,17 @@ faddp %st,%st(1) addq $2 * SIZE, X decq M - jg .L21 - jmp .L998 + jg L(21) + jmp L(998) ALIGN_4 -.L40: +L(40): movq M, I sarq $2, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): FLD 0 * SIZE(X) fmul %st(0), %st FLD 1 * SIZE(X) @@ -171,16 +171,16 @@ faddp %st, %st(1) decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): andq $3, M - jle .L998 + jle L(998) ALIGN_4 -.L61: +L(61): FLD 0 * SIZE(X) fmul %st(0), %st FLD 1 * SIZE(X) @@ -189,16 +189,16 @@ faddp %st,%st(3) faddp %st,%st(1) decq M - jg .L61 + jg L(61) ALIGN_4 -.L998: +L(998): faddp %st,%st(2) faddp %st,%st(1) faddp %st,%st(1) ALIGN_4 -.L999: +L(999): fsqrt #ifndef XDOUBLE subq $2 * SIZE, %rsp diff --git a/kernel/x86_64/znrm2_sse.S b/kernel/x86_64/znrm2_sse.S index 2274f2e985..43fe15f3eb 100644 --- a/kernel/x86_64/znrm2_sse.S +++ b/kernel/x86_64/znrm2_sse.S @@ -55,10 +55,10 @@ pxor %xmm0, %xmm0 testq M, M - jle .L999 + jle L(999) pxor %xmm1, %xmm1 testq INCX, INCX - je .L999 + je L(999) xorq FLAG, FLAG @@ -66,10 +66,10 @@ leaq (, INCX, 2 * SIZE), INCX pxor %xmm3, %xmm3 cmpq $2 * SIZE, INCX - jne .L40 + jne L(40) testq $SIZE, X - je .L05 + je L(05) movss (X), %xmm4 cvtss2sd %xmm4, %xmm6 @@ -78,13 +78,13 @@ addq $SIZE, X movq $1, FLAG decq M - jle .L19 + jle L(19) ALIGN_3 -.L05: +L(05): movq M, I sarq $3, I - jle .L14 + jle L(14) movsd 0 * SIZE(X), %xmm4 movsd 2 * SIZE(X), %xmm5 @@ -97,10 +97,10 @@ addq $16 * SIZE, X decq I - jle .L12 + jle L(12) ALIGN_3 -.L10: +L(10): #if defined(PREFETCH) PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -147,10 +147,10 @@ subq $-16 * SIZE, X decq I - jg .L10 + jg L(10) ALIGN_3 -.L12: +L(12): cvtps2pd %xmm4, %xmm12 cvtps2pd %xmm5, %xmm13 cvtps2pd %xmm6, %xmm14 @@ -183,9 +183,9 @@ ALIGN_3 -.L14: +L(14): testq $4, M - je .L15 + je L(15) movsd 0 * SIZE(X), %xmm4 movsd 2 * SIZE(X), %xmm5 @@ -210,9 +210,9 @@ addq $8 * SIZE, X ALIGN_3 -.L15: +L(15): testq $2, M - je .L16 + je L(16) movsd 0 * SIZE(X), %xmm4 movsd 2 * SIZE(X), %xmm5 @@ -225,9 +225,9 @@ addq $4 * SIZE, X ALIGN_3 -.L16: +L(16): testq $1, M - je .L19 + je L(19) movsd (X), %xmm4 cvtps2pd %xmm4, %xmm6 @@ -236,24 +236,24 @@ addq $2 * SIZE, X ALIGN_3 -.L19: +L(19): testq FLAG, FLAG - je .L998 + je L(998) movss (X), %xmm4 cvtss2sd %xmm4, %xmm6 mulsd %xmm6, %xmm6 addsd %xmm6, %xmm3 - jmp .L998 + jmp L(998) ALIGN_4 -.L40: +L(40): movq M, I sarq $3, I - jle .L44 + jle L(44) ALIGN_4 -.L41: +L(41): movsd (X), %xmm4 addq INCX, X movsd (X), %xmm5 @@ -302,12 +302,12 @@ addpd %xmm11, %xmm3 decq I - jg .L41 + jg L(41) ALIGN_3 -.L44: +L(44): testq $4, M - je .L45 + je L(45) movsd (X), %xmm4 addq INCX, X @@ -334,9 +334,9 @@ addpd %xmm11, %xmm3 ALIGN_3 -.L45: +L(45): testq $2, M - je .L46 + je L(46) movsd (X), %xmm4 addq INCX, X @@ -351,9 +351,9 @@ addpd %xmm7, %xmm1 ALIGN_3 -.L46: +L(46): testq $1, M - je .L998 + je L(998) movsd (X), %xmm4 cvtps2pd %xmm4, %xmm6 @@ -361,7 +361,7 @@ addpd %xmm6, %xmm3 ALIGN_4 -.L998: +L(998): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -375,7 +375,7 @@ #endif ALIGN_4 -.L999: +L(999): sqrtsd %xmm0, %xmm0 cvtsd2ss %xmm0, %xmm0 diff --git a/kernel/x86_64/zrot.S b/kernel/x86_64/zrot.S index 22d031c076..5f1fb76e25 100644 --- a/kernel/x86_64/zrot.S +++ b/kernel/x86_64/zrot.S @@ -69,19 +69,19 @@ salq $ZBASE_SHIFT, INCY testq N, N - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX - jne .L50 + jne L(50) cmpq $2 * SIZE, INCY - jne .L50 + jne L(50) movq N, I sarq $1, I - jle .L15 + jle L(15) ALIGN_4 -.L10: +L(10): #if defined(PREFETCHW) PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -170,16 +170,16 @@ addq $4 * SIZE, Y decq I - jg .L10 + jg L(10) ALIGN_4 -.L15: +L(15): movq N, I andq $1, I - jle .L999 + jle L(999) ALIGN_4 -.L16: +L(16): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -217,16 +217,16 @@ fsubrp %st, %st(1) FST 1 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_4 -.L50: +L(50): movq N, I sarq $1, I - jle .L55 + jle L(55) ALIGN_4 -.L51: +L(51): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -310,16 +310,16 @@ addq INCY, Y decq I - jg .L51 + jg L(51) ALIGN_4 -.L55: +L(55): movq N, I andq $1, I - jle .L999 + jle L(999) ALIGN_4 -.L56: +L(56): FLD 0 * SIZE(X) FLD 0 * SIZE(Y) @@ -359,7 +359,7 @@ FST 1 * SIZE(Y) ALIGN_4 -.L999: +L(999): ffreep %st ffreep %st ret diff --git a/kernel/x86_64/zrot_sse.S b/kernel/x86_64/zrot_sse.S index 6391311528..0ba3c64963 100644 --- a/kernel/x86_64/zrot_sse.S +++ b/kernel/x86_64/zrot_sse.S @@ -72,15 +72,15 @@ pshufd $0x0, %xmm1, S cmpq $0, N - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX - jne .L50 + jne L(50) cmpq $2 * SIZE, INCY - jne .L50 + jne L(50) testq $2 * SIZE, X - je .L10 + je L(10) movsd 0 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -102,18 +102,18 @@ addq $2 * SIZE, X addq $2 * SIZE, Y decq N - jle .L999 + jle L(999) -.L10: +L(10): testq $1 * SIZE, X - jne .L30 + jne L(30) testq $3 * SIZE, Y - jne .L20 + jne L(20) movq N, %rax sarq $4, %rax - jle .L14 + jle L(14) movaps 0 * SIZE(Y), %xmm1 movaps 4 * SIZE(Y), %xmm3 @@ -126,10 +126,10 @@ movaps 12 * SIZE(X), %xmm10 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #if defined(PREFETCHW) PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -262,10 +262,10 @@ addq $32 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movaps %xmm1, %xmm4 mulps S, %xmm1 movaps %xmm3, %xmm6 @@ -375,12 +375,12 @@ addq $32 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $15, N - jle .L999 + jle L(999) testq $8, N - jle .L15 + jle L(15) movaps 0 * SIZE(Y), %xmm1 movaps 0 * SIZE(X), %xmm0 @@ -446,9 +446,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $4, N - jle .L16 + jle L(16) movaps 0 * SIZE(Y), %xmm1 movaps 0 * SIZE(X), %xmm0 @@ -484,9 +484,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $2, N - jle .L17 + jle L(17) movaps 0 * SIZE(Y), %xmm1 movaps 0 * SIZE(X), %xmm0 @@ -510,9 +510,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $1, N - jle .L999 + jle L(999) movsd 0 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -531,13 +531,13 @@ movlps %xmm0, 0 * SIZE(X) movlps %xmm2, 0 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L20: +L(20): movq N, %rax sarq $4, %rax - jle .L24 + jle L(24) movsd 0 * SIZE(Y), %xmm1 movhps 2 * SIZE(Y), %xmm1 @@ -554,10 +554,10 @@ movaps 12 * SIZE(X), %xmm10 decq %rax - jle .L22 + jle L(22) ALIGN_3 -.L21: +L(21): #if defined(PREFETCHW) PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -706,10 +706,10 @@ addq $32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L22: +L(22): movaps %xmm1, %xmm4 mulps S, %xmm1 movaps %xmm3, %xmm6 @@ -831,12 +831,12 @@ addq $32 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $15, N - jle .L999 + jle L(999) testq $8, N - jle .L25 + jle L(25) movsd 0 * SIZE(Y), %xmm1 movhps 2 * SIZE(Y), %xmm1 @@ -910,9 +910,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $4, N - jle .L26 + jle L(26) movsd 0 * SIZE(Y), %xmm1 movhps 2 * SIZE(Y), %xmm1 @@ -952,9 +952,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $2, N - jle .L27 + jle L(27) movsd 0 * SIZE(Y), %xmm1 movhps 2 * SIZE(Y), %xmm1 @@ -980,9 +980,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $1, N - jle .L999 + jle L(999) movsd 0 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -1001,13 +1001,13 @@ movlps %xmm0, 0 * SIZE(X) movlps %xmm2, 0 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L30: +L(30): movq N, %rax sarq $4, %rax - jle .L34 + jle L(34) movsd 0 * SIZE(Y), %xmm1 movhps 2 * SIZE(Y), %xmm1 @@ -1028,10 +1028,10 @@ movhps 14 * SIZE(X), %xmm10 decq %rax - jle .L32 + jle L(32) ALIGN_3 -.L31: +L(31): #if defined(PREFETCHW) PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -1196,10 +1196,10 @@ addq $32 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L32: +L(32): movaps %xmm1, %xmm4 mulps S, %xmm1 movaps %xmm3, %xmm6 @@ -1333,12 +1333,12 @@ addq $32 * SIZE, Y ALIGN_3 -.L34: +L(34): testq $15, N - jle .L999 + jle L(999) testq $8, N - jle .L35 + jle L(35) movsd 0 * SIZE(Y), %xmm1 movhps 2 * SIZE(Y), %xmm1 @@ -1420,9 +1420,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L35: +L(35): testq $4, N - jle .L36 + jle L(36) movsd 0 * SIZE(Y), %xmm1 movhps 2 * SIZE(Y), %xmm1 @@ -1466,9 +1466,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L36: +L(36): testq $2, N - jle .L37 + jle L(37) movsd 0 * SIZE(Y), %xmm1 movhps 2 * SIZE(Y), %xmm1 @@ -1496,9 +1496,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L37: +L(37): testq $1, N - jle .L999 + jle L(999) movsd 0 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -1517,21 +1517,21 @@ movlps %xmm0, 0 * SIZE(X) movlps %xmm2, 0 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 ALIGN_3 -.L50: +L(50): movq N, %rax cmpq $0, INCX - je .L56 + je L(56) cmpq $0, INCY - je .L56 + je L(56) sarq $2, %rax - jle .L55 + jle L(55) ALIGN_3 -.L53: +L(53): movsd (Y), %xmm1 movhps (Y, INCY), %xmm1 movsd (X), %xmm0 @@ -1583,16 +1583,16 @@ leaq (Y, INCY, 2), Y decq %rax - jg .L53 + jg L(53) ALIGN_3 -.L55: +L(55): movq N, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_3 -.L56: +L(56): movsd (Y), %xmm1 movsd (X), %xmm0 @@ -1615,10 +1615,10 @@ addq INCY, Y decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/zrot_sse2.S b/kernel/x86_64/zrot_sse2.S index e6288c3d5c..f4a0653d6a 100644 --- a/kernel/x86_64/zrot_sse2.S +++ b/kernel/x86_64/zrot_sse2.S @@ -72,23 +72,23 @@ pshufd $0x44, %xmm1, S cmpq $0, N - jle .L999 + jle L(999) cmpq $2 * SIZE, INCX - jne .L50 + jne L(50) cmpq $2 * SIZE, INCY - jne .L50 + jne L(50) -.L10: +L(10): testq $SIZE, X - jne .L30 + jne L(30) testq $SIZE, Y - jne .L20 + jne L(20) movq N, %rax sarq $3, %rax - jle .L14 + jle L(14) movapd 0 * SIZE(Y), %xmm1 movapd 2 * SIZE(Y), %xmm3 @@ -101,10 +101,10 @@ movapd 6 * SIZE(X), %xmm10 decq %rax - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -237,10 +237,10 @@ addq $16 * SIZE, Y decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): movapd %xmm1, %xmm4 mulpd S, %xmm1 movapd %xmm3, %xmm6 @@ -350,12 +350,12 @@ addq $16 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $7, N - jle .L999 + jle L(999) testq $4, N - jle .L15 + jle L(15) movapd 0 * SIZE(Y), %xmm1 movapd 0 * SIZE(X), %xmm0 @@ -421,9 +421,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $2, N - jle .L16 + jle L(16) movapd 0 * SIZE(Y), %xmm1 movapd 0 * SIZE(X), %xmm0 @@ -459,9 +459,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $1, N - jle .L999 + jle L(999) movapd 0 * SIZE(Y), %xmm1 movapd 0 * SIZE(X), %xmm0 @@ -480,18 +480,18 @@ movapd %xmm0, 0 * SIZE(X) movapd %xmm2, 0 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L20: +L(20): movapd -1 * SIZE(Y), %xmm1 movq N, %rax sarq $3, %rax - jle .L24 + jle L(24) ALIGN_3 -.L21: +L(21): #if defined(PREFETCHW) PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -652,15 +652,15 @@ addq $16 * SIZE, X addq $16 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L24: +L(24): testq $7, N - jle .L999 + jle L(999) testq $4, N - jle .L25 + jle L(25) movapd 1 * SIZE(Y), %xmm3 movapd 3 * SIZE(Y), %xmm8 @@ -736,9 +736,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $2, N - jle .L26 + jle L(26) movapd 1 * SIZE(Y), %xmm3 movapd 3 * SIZE(Y), %xmm8 @@ -780,9 +780,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L26: +L(26): testq $1, N - jle .L999 + jle L(999) movapd 1 * SIZE(Y), %xmm4 movapd 0 * SIZE(X), %xmm0 @@ -803,21 +803,21 @@ movapd %xmm0, 0 * SIZE(X) movlps %xmm2, 0 * SIZE(Y) movhps %xmm2, 1 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L30: +L(30): testq $SIZE, Y - jne .L40 + jne L(40) movapd -1 * SIZE(X), %xmm0 movq N, %rax sarq $3, %rax - jle .L34 + jle L(34) ALIGN_3 -.L31: +L(31): #if defined(PREFETCHW) PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -977,15 +977,15 @@ addq $16 * SIZE, Y addq $16 * SIZE, X decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L34: +L(34): testq $7, N - jle .L999 + jle L(999) testq $4, N - jle .L35 + jle L(35) movapd 1 * SIZE(X), %xmm2 movapd 3 * SIZE(X), %xmm8 @@ -1061,9 +1061,9 @@ addq $8 * SIZE, X ALIGN_3 -.L35: +L(35): testq $2, N - jle .L36 + jle L(36) movapd 1 * SIZE(X), %xmm2 movapd 3 * SIZE(X), %xmm8 @@ -1106,9 +1106,9 @@ addq $4 * SIZE, X ALIGN_3 -.L36: +L(36): testq $1, N - jle .L999 + jle L(999) movapd 1 * SIZE(X), %xmm4 movapd 0 * SIZE(Y), %xmm1 @@ -1129,10 +1129,10 @@ movlps %xmm0, 0 * SIZE(X) movhps %xmm0, 1 * SIZE(X) movapd %xmm2, 0 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L40: +L(40): movsd 0 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -1154,11 +1154,11 @@ addq $1 * SIZE, X decq N - jle .L47 + jle L(47) movq N, %rax sarq $3, %rax - jle .L44 + jle L(44) movapd 0 * SIZE(Y), %xmm1 movapd 2 * SIZE(Y), %xmm3 @@ -1171,10 +1171,10 @@ movapd 6 * SIZE(X), %xmm10 decq %rax - jle .L42 + jle L(42) ALIGN_3 -.L41: +L(41): #if defined(PREFETCHW) PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -1307,10 +1307,10 @@ addq $16 * SIZE, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L42: +L(42): movapd %xmm1, %xmm4 mulpd S, %xmm1 movapd %xmm3, %xmm6 @@ -1420,9 +1420,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L44: +L(44): testq $4, N - jle .L45 + jle L(45) movapd 0 * SIZE(Y), %xmm1 movapd 0 * SIZE(X), %xmm0 @@ -1488,9 +1488,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L45: +L(45): testq $2, N - jle .L46 + jle L(46) movapd 0 * SIZE(Y), %xmm1 movapd 0 * SIZE(X), %xmm0 @@ -1526,9 +1526,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L46: +L(46): testq $1, N - jle .L47 + jle L(47) movapd 0 * SIZE(Y), %xmm1 movapd 0 * SIZE(X), %xmm0 @@ -1551,7 +1551,7 @@ addq $2 * SIZE, X ALIGN_3 -.L47: +L(47): movsd 0 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -1569,16 +1569,16 @@ movsd %xmm0, 0 * SIZE(X) movsd %xmm2, 0 * SIZE(Y) - jmp .L999 + jmp L(999) ALIGN_3 -.L50: +L(50): movq N, %rax sarq $2, %rax - jle .L55 + jle L(55) ALIGN_3 -.L53: +L(53): movsd 0 * SIZE(Y), %xmm1 movhps 1 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -1680,16 +1680,16 @@ addq INCY, Y decq %rax - jg .L53 + jg L(53) ALIGN_3 -.L55: +L(55): movq N, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_3 -.L56: +L(56): movsd 0 * SIZE(Y), %xmm1 movhps 1 * SIZE(Y), %xmm1 movsd 0 * SIZE(X), %xmm0 @@ -1716,10 +1716,10 @@ addq INCY, Y decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/zscal.S b/kernel/x86_64/zscal.S index 5ed4c4576b..2b1acefc26 100644 --- a/kernel/x86_64/zscal.S +++ b/kernel/x86_64/zscal.S @@ -60,7 +60,7 @@ FLD 24(%rsp) testq N, N - jle .L999 + jle L(999) fld %st(1) fabs @@ -71,21 +71,21 @@ fldz fcomip %st(1), %st ffreep %st - jne .L30 + jne L(30) EMMS pxor %mm0, %mm0 cmpq $2 * SIZE, INCX - jne .L20 + jne L(20) movq N, I sarq $2, I - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -108,16 +108,16 @@ movq %mm0, 120(X) addq $8 * SIZE, X decq I - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): movq N, I andq $3, I - jle .L18 + jle L(18) ALIGN_2 -.L16: +L(16): movq %mm0, 0(X) movq %mm0, 8(X) movq %mm0, 16(X) @@ -125,21 +125,21 @@ addq $2 * SIZE, X decq I - jg .L16 + jg L(16) -.L18: +L(18): EMMS ret ALIGN_2 -.L20: +L(20): movq N, I sarq $2, I - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): movq %mm0, 0(X) movq %mm0, 8(X) movq %mm0, 16(X) @@ -165,16 +165,16 @@ addq INCX, X decq I - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): movq N, I andq $3, I - jle .L28 + jle L(28) ALIGN_3 -.L26: +L(26): movq %mm0, 0(X) movq %mm0, 8(X) movq %mm0, 16(X) @@ -182,19 +182,19 @@ addq INCX, X decq I - jg .L26 + jg L(26) -.L28: +L(28): EMMS ret ALIGN_3 -.L30: +L(30): movq N, I ALIGN_2 -.L32: +L(32): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -215,10 +215,10 @@ FST 1 * SIZE(X) addq INCX, X decq I - jg .L32 + jg L(32) ALIGN_2 -.L999: +L(999): ffreep %st ffreep %st diff --git a/kernel/x86_64/zscal_atom.S b/kernel/x86_64/zscal_atom.S index 7713626c9a..5f6be9605b 100644 --- a/kernel/x86_64/zscal_atom.S +++ b/kernel/x86_64/zscal_atom.S @@ -69,26 +69,26 @@ salq $ZBASE_SHIFT, INCX testq M, M - jle .L999 + jle L(999) pxor %xmm15, %xmm15 comisd %xmm0, %xmm15 - jne .L30 # Alpha_r != ZERO - jp .L30 + jne L(30) # Alpha_r != ZERO + jp L(30) comisd %xmm1, %xmm15 - jne .L30 # Alpha_i != ZERO + jne L(30) # Alpha_i != ZERO /* Alpha == ZERO */ cmpq $2 * SIZE, INCX - jne .L20 + jne L(20) movq M, I sarq $2, I - jle .L12 + jle L(12) ALIGN_4 -.L11: +L(11): movsd %xmm1, 0 * SIZE(X) movsd %xmm1, 1 * SIZE(X) movsd %xmm1, 2 * SIZE(X) @@ -101,12 +101,12 @@ addq $8 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_4 -.L12: +L(12): testq $2, M - je .L14 + je L(14) movsd %xmm1, 0 * SIZE(X) movsd %xmm1, 1 * SIZE(X) @@ -116,23 +116,23 @@ addq $4 * SIZE, X ALIGN_3 -.L14: +L(14): testq $1, M - je .L999 + je L(999) movsd %xmm1, 0 * SIZE(X) movsd %xmm1, 1 * SIZE(X) addq $2 * SIZE, X - jmp .L999 + jmp L(999) ALIGN_4 -.L20: +L(20): movq M, I # rcx = n sarq $2, I - jle .L22 + jle L(22) ALIGN_4 -.L21: +L(21): movsd %xmm1, 0 * SIZE(X) movsd %xmm1, 1 * SIZE(X) addq INCX, X @@ -146,12 +146,12 @@ movsd %xmm1, 1 * SIZE(X) addq INCX, X decq I - jg .L21 + jg L(21) ALIGN_4 -.L22: +L(22): testq $2, M - je .L23 + je L(23) movsd %xmm1, 0 * SIZE(X) movsd %xmm1, 1 * SIZE(X) @@ -161,22 +161,22 @@ addq INCX, X ALIGN_3 -.L23: +L(23): testq $1, M - je .L999 + je L(999) movsd %xmm1, 0 * SIZE(X) movsd %xmm1, 1 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_4 /* Alpha != ZERO */ -.L30: +L(30): movq X, XX movq M, I sarq $2, I - jle .L35 + jle L(35) movsd 0 * SIZE(X), %xmm2 movsd 1 * SIZE(X), %xmm3 @@ -209,10 +209,10 @@ mulsd %xmm1, %xmm4 decq I - jle .L32 + jle L(32) ALIGN_4 -.L31: +L(31): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -282,10 +282,10 @@ mulsd %xmm1, %xmm4 decq I - jg .L31 + jg L(31) ALIGN_4 -.L32: +L(32): subsd %xmm5, %xmm6 movsd %xmm2, 0 * SIZE(XX) addsd %xmm4, %xmm7 @@ -323,9 +323,9 @@ addq INCX, XX ALIGN_3 -.L35: +L(35): testq $2, M - je .L37 + je L(37) movsd 0 * SIZE(X), %xmm2 movsd 1 * SIZE(X), %xmm3 @@ -362,9 +362,9 @@ addq INCX, XX ALIGN_3 -.L37: +L(37): testq $1, M - je .L999 + je L(999) movsd 0 * SIZE(X), %xmm2 movsd 1 * SIZE(X), %xmm3 @@ -383,7 +383,7 @@ movsd %xmm3, 1 * SIZE(XX) ALIGN_3 -.L999: +L(999): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/zscal_sse.S b/kernel/x86_64/zscal_sse.S index acd6c36549..41d59cf270 100644 --- a/kernel/x86_64/zscal_sse.S +++ b/kernel/x86_64/zscal_sse.S @@ -71,47 +71,47 @@ xor FLAG, FLAG testq M, M - jle .L999 + jle L(999) pxor %xmm15, %xmm15 comiss %xmm0, %xmm15 - jne .L100 # Alpha_r != ZERO - jp .L100 # Alpha_r == NAN + jne L(100) # Alpha_r != ZERO + jp L(100) # Alpha_r == NAN comiss %xmm1, %xmm15 - jne .L100 # Alpha_i != ZERO + jne L(100) # Alpha_i != ZERO /* Alpha == ZERO */ cmpq $2 * SIZE, INCX - jne .L50 + jne L(50) /* INCX == 1 */ cmpq $3, M - jle .L13 + jle L(13) testq $4, X - je .L05 + je L(05) movss %xmm15, 0 * SIZE(X) addq $SIZE, X movq $1, FLAG decq M ALIGN_3 -.L05: +L(05): testq $8, X - je .L06 + je L(06) movlps %xmm15, 0 * SIZE(X) addq $2 * SIZE, X subq $1, M ALIGN_3 -.L06: +L(06): movq M, I # rcx = n sarq $3, I - jle .L12 + jle L(12) ALIGN_4 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -122,53 +122,53 @@ movaps %xmm15, 12 * SIZE(X) addq $16 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_4 -.L12: +L(12): testq $7, M - je .L19 + je L(19) testq $4, M - je .L13 + je L(13) movaps %xmm15, 0 * SIZE(X) movaps %xmm15, 4 * SIZE(X) addq $8 * SIZE, X ALIGN_3 -.L13: +L(13): testq $2, M - je .L14 + je L(14) movlps %xmm15, 0 * SIZE(X) movhps %xmm15, 2 * SIZE(X) addq $4 * SIZE, X ALIGN_3 -.L14: +L(14): testq $1, M - je .L19 + je L(19) movlps %xmm15, 0 * SIZE(X) addq $2 * SIZE, X ALIGN_3 -.L19: +L(19): testq $1, FLAG - je .L999 + je L(999) movss %xmm15, 0 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_4 /* incx != 1 */ -.L50: +L(50): movq M, I # rcx = n sarq $2, I - jle .L52 + jle L(52) ALIGN_4 -.L51: +L(51): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -182,12 +182,12 @@ movsd %xmm15, 0 * SIZE(X) addq INCX, X decq I - jg .L51 + jg L(51) ALIGN_4 -.L52: +L(52): testq $2, M - je .L53 + je L(53) movsd %xmm15, 0 * SIZE(X) addq INCX, X @@ -195,22 +195,22 @@ addq INCX, X ALIGN_3 -.L53: +L(53): testq $1, M - je .L999 + je L(999) movsd %xmm15, 0 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_4 /* Alpha != ZERO */ -.L100: +L(100): testq $SIZE, X - jne .L130 + jne L(130) cmpq $2 * SIZE, INCX - jne .L120 + jne L(120) pshufd $0, %xmm0, %xmm14 pshufd $0, %xmm1, %xmm1 @@ -220,7 +220,7 @@ subq $-32 * SIZE, X testq $2 * SIZE, X - je .L105 + je L(105) movsd -32 * SIZE(X), %xmm0 @@ -232,13 +232,13 @@ movlps %xmm0, -32 * SIZE(X) addq $2 * SIZE, X decq M - jle .L999 + jle L(999) ALIGN_3 -.L105: +L(105): movq M, I sarq $4, I - jle .L115 + jle L(115) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -250,10 +250,10 @@ movaps -4 * SIZE(X), %xmm7 decq I - jle .L112 + jle L(112) ALIGN_4 -.L111: +L(111): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -320,10 +320,10 @@ subq $-32 * SIZE, X decq I - jg .L111 + jg L(111) ALIGN_4 -.L112: +L(112): pshufd $0xb1, %xmm0, %xmm8 mulps %xmm14, %xmm0 mulps %xmm15, %xmm8 @@ -375,9 +375,9 @@ subq $-32 * SIZE, X ALIGN_4 -.L115: +L(115): testq $8, M - je .L116 + je L(116) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -412,9 +412,9 @@ addq $16 * SIZE, X ALIGN_3 -.L116: +L(116): testq $4, M - je .L117 + je L(117) movaps -32 * SIZE(X), %xmm0 movaps -28 * SIZE(X), %xmm1 @@ -434,9 +434,9 @@ addq $8 * SIZE, X ALIGN_3 -.L117: +L(117): testq $2, M - je .L118 + je L(118) movaps -32 * SIZE(X), %xmm0 @@ -449,9 +449,9 @@ addq $4 * SIZE, X ALIGN_3 -.L118: +L(118): testq $1, M - je .L999 + je L(999) movsd -32 * SIZE(X), %xmm0 @@ -461,10 +461,10 @@ addps %xmm8, %xmm0 movlps %xmm0, -32 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_3 -.L120: +L(120): pshufd $0, %xmm0, %xmm14 pshufd $0, %xmm1, %xmm1 subps %xmm1, %xmm15 @@ -474,7 +474,7 @@ movq M, I sarq $3, I - jle .L125 + jle L(125) movsd (X), %xmm0 addq INCX, X @@ -497,10 +497,10 @@ addq INCX, X decq I - jle .L122 + jle L(122) ALIGN_4 -.L121: +L(121): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -566,10 +566,10 @@ addq INCX, X decq I - jg .L121 + jg L(121) ALIGN_4 -.L122: +L(122): pshufd $0xb1, %xmm0, %xmm8 mulps %xmm14, %xmm0 mulps %xmm15, %xmm8 @@ -611,9 +611,9 @@ addq INCX, XX ALIGN_4 -.L125: +L(125): testq $4, M - je .L127 + je L(127) movsd (X), %xmm0 addq INCX, X @@ -646,9 +646,9 @@ addq INCX, XX ALIGN_3 -.L127: +L(127): testq $2, M - je .L128 + je L(128) movsd (X), %xmm0 addq INCX, X @@ -666,9 +666,9 @@ addq INCX, XX ALIGN_3 -.L128: +L(128): testq $1, M - je .L999 + je L(999) movsd (X), %xmm0 @@ -678,12 +678,12 @@ addps %xmm8, %xmm0 movlps %xmm0, (XX) - jmp .L999 + jmp L(999) ALIGN_3 -.L130: +L(130): cmpq $2 * SIZE, INCX - jne .L120 + jne L(120) #if defined(ALIGNED_ACCESS) && !defined(NEHALEM) && !defined(SANDYBRIDGE) @@ -695,7 +695,7 @@ subq $-31 * SIZE, X testq $2 * SIZE, X - je .L130x + je L(130x) movsd -31 * SIZE(X), %xmm0 @@ -707,10 +707,10 @@ movlps %xmm0, -31 * SIZE(X) addq $2 * SIZE, X decq M - jle .L999 + jle L(999) ALIGN_3 -.L130x: +L(130x): shufps $0xb1, %xmm15, %xmm15 movaps -32 * SIZE(X), %xmm0 @@ -718,7 +718,7 @@ movq M, I sarq $4, I - jle .L135 + jle L(135) movaps -28 * SIZE(X), %xmm1 movaps -24 * SIZE(X), %xmm2 @@ -729,10 +729,10 @@ movaps -4 * SIZE(X), %xmm7 decq I - jle .L132 + jle L(132) ALIGN_4 -.L131: +L(131): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -831,10 +831,10 @@ subq $-32 * SIZE, X decq I - jg .L131 + jg L(131) ALIGN_4 -.L132: +L(132): movss %xmm1, %xmm0 pshufd $0x1b, %xmm0, %xmm8 mulps %xmm14, %xmm0 @@ -912,9 +912,9 @@ subq $-32 * SIZE, X ALIGN_4 -.L135: +L(135): testq $8, M - je .L136 + je L(136) movaps -28 * SIZE(X), %xmm1 @@ -967,9 +967,9 @@ addq $16 * SIZE, X ALIGN_3 -.L136: +L(136): testq $4, M - je .L137 + je L(137) movaps -28 * SIZE(X), %xmm1 @@ -1000,9 +1000,9 @@ addq $8 * SIZE, X ALIGN_3 -.L137: +L(137): testq $2, M - je .L138 + je L(138) movaps -28 * SIZE(X), %xmm1 @@ -1021,11 +1021,11 @@ addq $4 * SIZE, X ALIGN_3 -.L138: +L(138): movss %xmm9, -32 * SIZE(X) testq $1, M - je .L999 + je L(999) pshufd $0x1b, %xmm0, %xmm8 mulps %xmm14, %xmm0 @@ -1035,7 +1035,7 @@ pshufd $0x39, %xmm0, %xmm0 movlps %xmm0, -31 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_3 @@ -1049,7 +1049,7 @@ subq $-32 * SIZE, X testq $2 * SIZE, X - je .L130x + je L(130x) movsd -32 * SIZE(X), %xmm0 @@ -1061,13 +1061,13 @@ movlps %xmm0, -32 * SIZE(X) addq $2 * SIZE, X decq M - jle .L999 + jle L(999) ALIGN_3 -.L130x: +L(130x): movq M, I sarq $4, I - jle .L135 + jle L(135) movsd -32 * SIZE(X), %xmm0 movhps -30 * SIZE(X), %xmm0 @@ -1087,10 +1087,10 @@ movhps -2 * SIZE(X), %xmm7 decq I - jle .L132 + jle L(132) ALIGN_4 -.L131: +L(131): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -1181,10 +1181,10 @@ subq $-32 * SIZE, X decq I - jg .L131 + jg L(131) ALIGN_4 -.L132: +L(132): pshufd $0xb1, %xmm0, %xmm8 mulps %xmm14, %xmm0 mulps %xmm15, %xmm8 @@ -1244,9 +1244,9 @@ subq $-32 * SIZE, X ALIGN_4 -.L135: +L(135): testq $8, M - je .L136 + je L(136) movsd -32 * SIZE(X), %xmm0 movhps -30 * SIZE(X), %xmm0 @@ -1291,9 +1291,9 @@ addq $16 * SIZE, X ALIGN_3 -.L136: +L(136): testq $4, M - je .L137 + je L(137) movsd -32 * SIZE(X), %xmm0 movhps -30 * SIZE(X), %xmm0 @@ -1317,9 +1317,9 @@ addq $8 * SIZE, X ALIGN_3 -.L137: +L(137): testq $2, M - je .L138 + je L(138) movsd -32 * SIZE(X), %xmm0 movhps -30 * SIZE(X), %xmm0 @@ -1334,9 +1334,9 @@ addq $4 * SIZE, X ALIGN_3 -.L138: +L(138): testq $1, M - je .L999 + je L(999) movsd -32 * SIZE(X), %xmm0 @@ -1349,7 +1349,7 @@ ALIGN_3 #endif -.L999: +L(999): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/zscal_sse2.S b/kernel/x86_64/zscal_sse2.S index d6a49136d1..2a0bab379a 100644 --- a/kernel/x86_64/zscal_sse2.S +++ b/kernel/x86_64/zscal_sse2.S @@ -77,38 +77,38 @@ xor FLAG, FLAG testq M, M - jle .L999 + jle L(999) pxor %xmm15, %xmm15 comisd %xmm0, %xmm15 - jne .L100 - jp .L100 + jne L(100) + jp L(100) comisd %xmm1, %xmm15 - jne .L100 + jne L(100) /* Alpha == ZERO */ cmpq $2 * SIZE, INCX - jne .L20 + jne L(20) /* INCX == 1 */ testq $SIZE, X - je .L05 + je L(05) movsd %xmm15, 0 * SIZE(X) addq $SIZE, X movq $1, FLAG decq M - jle .L19 + jle L(19) ALIGN_3 -.L05: +L(05): movq M, I # rcx = n sarq $3, I - jle .L12 + jle L(12) ALIGN_4 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -129,12 +129,12 @@ addq $16 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_4 -.L12: +L(12): testq $4, M - je .L13 + je L(13) movaps %xmm15, 0 * SIZE(X) movaps %xmm15, 2 * SIZE(X) @@ -143,42 +143,42 @@ addq $8 * SIZE, X ALIGN_3 -.L13: +L(13): testq $2, M - je .L14 + je L(14) movaps %xmm15, 0 * SIZE(X) movaps %xmm15, 2 * SIZE(X) addq $4 * SIZE, X ALIGN_3 -.L14: +L(14): testq $1, M - je .L19 + je L(19) movaps %xmm15, 0 * SIZE(X) addq $2 * SIZE, X ALIGN_3 -.L19: +L(19): testq $1, FLAG - je .L999 + je L(999) movsd %xmm15, 0 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_4 /* incx != 1 */ -.L20: +L(20): testq $SIZE, X - jne .L30 + jne L(30) /* Aligned Mode */ movq M, I # rcx = n sarq $2, I - jle .L22 + jle L(22) ALIGN_4 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -192,15 +192,15 @@ movaps %xmm15, (X) addq INCX, X decq I - jg .L21 + jg L(21) ALIGN_4 -.L22: +L(22): testq $3, M - je .L999 + je L(999) testq $2, M - je .L23 + je L(23) movaps %xmm15, (X) addq INCX, X @@ -208,23 +208,23 @@ addq INCX, X ALIGN_3 -.L23: +L(23): testq $1, M - je .L999 + je L(999) movaps %xmm15, (X) - jmp .L999 + jmp L(999) ALIGN_4 /* Unaligned Mode */ -.L30: +L(30): movq M, I # rcx = n sarq $2, I - jle .L32 + jle L(32) ALIGN_4 -.L31: +L(31): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -242,15 +242,15 @@ movlps %xmm15, 1 * SIZE(X) addq INCX, X decq I - jg .L31 + jg L(31) ALIGN_4 -.L32: +L(32): testq $3, M - je .L999 + je L(999) testq $2, M - je .L33 + je L(33) movlps %xmm15, 0 * SIZE(X) movlps %xmm15, 1 * SIZE(X) @@ -260,19 +260,19 @@ addq INCX, X ALIGN_3 -.L33: +L(33): testq $1, M - je .L999 + je L(999) movlps %xmm15, 0 * SIZE(X) movlps %xmm15, 1 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_4 /* Alpha != ZERO */ -.L100: +L(100): testq $SIZE, X - jne .L200 + jne L(200) #ifdef HAVE_SSE3 movddup %xmm0, %xmm14 @@ -284,13 +284,13 @@ movlhps %xmm1, %xmm15 cmpq $2 * SIZE, INCX - jne .L120 + jne L(120) subq $-16 * SIZE, X movq M, I sarq $3, I - jle .L115 + jle L(115) movaps -16 * SIZE(X), %xmm0 movaps -14 * SIZE(X), %xmm1 @@ -302,10 +302,10 @@ movaps -2 * SIZE(X), %xmm7 decq I - jle .L112 + jle L(112) ALIGN_4 -.L111: +L(111): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -412,10 +412,10 @@ subq $-16 * SIZE, X decq I - jg .L111 + jg L(111) ALIGN_4 -.L112: +L(112): pshufd $0x4e, %xmm0, %xmm8 mulpd %xmm14, %xmm0 mulpd %xmm15, %xmm8 @@ -467,12 +467,12 @@ subq $-16 * SIZE, X ALIGN_3 -.L115: +L(115): testq $7, M - je .L999 + je L(999) testq $4, M - je .L116 + je L(116) movaps -16 * SIZE(X), %xmm0 movaps -14 * SIZE(X), %xmm1 @@ -507,9 +507,9 @@ addq $8 * SIZE, X ALIGN_3 -.L116: +L(116): testq $2, M - je .L117 + je L(117) movaps -16 * SIZE(X), %xmm0 movaps -14 * SIZE(X), %xmm1 @@ -529,9 +529,9 @@ addq $4 * SIZE, X ALIGN_3 -.L117: +L(117): testq $1, M - je .L999 + je L(999) movaps -16 * SIZE(X), %xmm0 @@ -541,15 +541,15 @@ addpd %xmm8, %xmm0 movaps %xmm0, -16 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_3 -.L120: +L(120): movq X, XX movq M, I sarq $3, I - jle .L125 + jle L(125) movaps (X), %xmm0 addq INCX, X @@ -569,10 +569,10 @@ addq INCX, X decq I - jle .L122 + jle L(122) ALIGN_4 -.L121: +L(121): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -654,10 +654,10 @@ addq INCX, X decq I - jg .L121 + jg L(121) ALIGN_4 -.L122: +L(122): pshufd $0x4e, %xmm0, %xmm8 mulpd %xmm14, %xmm0 mulpd %xmm15, %xmm8 @@ -715,12 +715,12 @@ addq INCX, XX ALIGN_3 -.L125: +L(125): testq $7, M - je .L999 + je L(999) testq $4, M - je .L126 + je L(126) movaps (X), %xmm0 addq INCX, X @@ -761,9 +761,9 @@ addq INCX, XX ALIGN_3 -.L126: +L(126): testq $2, M - je .L127 + je L(127) movaps (X), %xmm0 addq INCX, X @@ -785,9 +785,9 @@ addq INCX, XX ALIGN_3 -.L127: +L(127): testq $1, M - je .L999 + je L(999) movaps (X), %xmm0 @@ -797,12 +797,12 @@ addpd %xmm8, %xmm0 movaps %xmm0, (XX) - jmp .L999 + jmp L(999) ALIGN_3 -.L200: +L(200): cmpq $2 * SIZE, INCX - jne .L220 + jne L(220) #if defined(ALIGNED_ACCESS) && !defined(NEHALEM) && !defined(SANDYBRIDGE) @@ -827,17 +827,17 @@ movq M, I sarq $3, I - jle .L205 + jle L(205) movaps -13 * SIZE(X), %xmm2 movaps -11 * SIZE(X), %xmm3 movaps -9 * SIZE(X), %xmm4 decq I - jle .L202 + jle L(202) ALIGN_4 -.L201: +L(201): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -912,10 +912,10 @@ subq $-16 * SIZE, X decq I - jg .L201 + jg L(201) ALIGN_4 -.L202: +L(202): movaps %xmm1, %xmm8 SHUFPD_1 %xmm2, %xmm0 mulpd %xmm14, %xmm8 @@ -980,9 +980,9 @@ subq $-16 * SIZE, X ALIGN_3 -.L205: +L(205): testq $4, M - je .L206 + je L(206) movaps -13 * SIZE(X), %xmm2 @@ -1023,9 +1023,9 @@ addq $8 * SIZE, X ALIGN_3 -.L206: +L(206): testq $2, M - je .L207 + je L(207) movaps -13 * SIZE(X), %xmm2 @@ -1053,9 +1053,9 @@ addq $4 * SIZE, X ALIGN_3 -.L207: +L(207): testq $1, M - je .L208 + je L(208) movaps -13 * SIZE(X), %xmm2 @@ -1072,13 +1072,13 @@ addq $2 * SIZE, X ALIGN_3 -.L208: +L(208): unpckhpd %xmm0, %xmm0 mulsd %xmm14, %xmm1 mulsd %xmm15, %xmm0 addsd %xmm1, %xmm0 movlps %xmm0, -15 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_3 #else @@ -1092,7 +1092,7 @@ movq M, I sarq $3, I - jle .L205 + jle L(205) movsd -16 * SIZE(X), %xmm0 movhps -15 * SIZE(X), %xmm0 @@ -1113,10 +1113,10 @@ movhps -1 * SIZE(X), %xmm7 decq I - jle .L202 + jle L(202) ALIGN_4 -.L201: +L(201): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -1239,10 +1239,10 @@ subq $-16 * SIZE, X decq I - jg .L201 + jg L(201) ALIGN_4 -.L202: +L(202): pshufd $0x4e, %xmm0, %xmm8 mulpd %xmm14, %xmm0 mulpd %xmm15, %xmm8 @@ -1302,12 +1302,12 @@ subq $-16 * SIZE, X ALIGN_3 -.L205: +L(205): testq $7, M - je .L999 + je L(999) testq $4, M - je .L206 + je L(206) movsd -16 * SIZE(X), %xmm0 movhps -15 * SIZE(X), %xmm0 @@ -1350,9 +1350,9 @@ addq $8 * SIZE, X ALIGN_3 -.L206: +L(206): testq $2, M - je .L207 + je L(207) movsd -16 * SIZE(X), %xmm0 movhps -15 * SIZE(X), %xmm0 @@ -1377,9 +1377,9 @@ addq $4 * SIZE, X ALIGN_3 -.L207: +L(207): testq $1, M - je .L999 + je L(999) movsd -16 * SIZE(X), %xmm0 movhps -15 * SIZE(X), %xmm0 @@ -1391,12 +1391,12 @@ movlps %xmm0, -16 * SIZE(X) movhps %xmm0, -15 * SIZE(X) - jmp .L999 + jmp L(999) ALIGN_3 #endif -.L220: +L(220): movddup %xmm0, %xmm14 pxor %xmm15, %xmm15 subsd %xmm1, %xmm15 @@ -1406,7 +1406,7 @@ movq M, I sarq $3, I - jle .L225 + jle L(225) movsd 0 * SIZE(X), %xmm0 movhps 1 * SIZE(X), %xmm0 @@ -1434,10 +1434,10 @@ addq INCX, X decq I - jle .L222 + jle L(222) ALIGN_4 -.L221: +L(221): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -1544,10 +1544,10 @@ decq I - jg .L221 + jg L(221) ALIGN_4 -.L222: +L(222): pshufd $0x4e, %xmm0, %xmm8 mulpd %xmm14, %xmm0 mulpd %xmm15, %xmm8 @@ -1613,12 +1613,12 @@ addq INCX, XX ALIGN_3 -.L225: +L(225): testq $7, M - je .L999 + je L(999) testq $4, M - je .L226 + je L(226) movsd 0 * SIZE(X), %xmm0 movhps 1 * SIZE(X), %xmm0 @@ -1669,9 +1669,9 @@ addq INCX, XX ALIGN_3 -.L226: +L(226): testq $2, M - je .L227 + je L(227) movsd 0 * SIZE(X), %xmm0 movhps 1 * SIZE(X), %xmm0 @@ -1698,9 +1698,9 @@ addq INCX, XX ALIGN_3 -.L227: +L(227): testq $1, M - je .L999 + je L(999) movsd 0 * SIZE(X), %xmm0 movhps 1 * SIZE(X), %xmm0 @@ -1714,7 +1714,7 @@ movhps %xmm0, 1 * SIZE(XX) ALIGN_3 -.L999: +L(999): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/zsum.S b/kernel/x86_64/zsum.S index aa02637e47..159bfb44b8 100644 --- a/kernel/x86_64/zsum.S +++ b/kernel/x86_64/zsum.S @@ -56,9 +56,9 @@ fldz testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) salq $ZBASE_SHIFT, INCX @@ -66,14 +66,14 @@ fldz fldz cmpq $SIZE * 2, INCX - jne .L40 + jne L(40) movq M, I sarq $2, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -101,33 +101,33 @@ faddp %st, %st(1) decq I - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): andq $3, M - jle .L998 + jle L(998) ALIGN_4 -.L21: +L(21): FLD 0 * SIZE(X) FLD 1 * SIZE(X) faddp %st,%st(3) faddp %st,%st(1) addq $2 * SIZE, X decq M - jg .L21 - jmp .L998 + jg L(21) + jmp L(998) ALIGN_4 -.L40: +L(40): movq M, I sarq $2, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): FLD 0 * SIZE(X) FLD 1 * SIZE(X) addq INCX, X @@ -153,32 +153,32 @@ faddp %st, %st(1) decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): andq $3, M - jle .L998 + jle L(998) ALIGN_4 -.L61: +L(61): FLD 0 * SIZE(X) FLD 1 * SIZE(X) addq INCX, X faddp %st,%st(3) faddp %st,%st(1) decq M - jg .L61 + jg L(61) ALIGN_4 -.L998: +L(998): faddp %st,%st(2) faddp %st,%st(1) faddp %st,%st(1) ALIGN_4 -.L999: +L(999): ret EPILOGUE diff --git a/kernel/x86_64/zsum_sse.S b/kernel/x86_64/zsum_sse.S index b679b42b0e..eeacae1d97 100644 --- a/kernel/x86_64/zsum_sse.S +++ b/kernel/x86_64/zsum_sse.S @@ -54,9 +54,9 @@ pxor %xmm0, %xmm0 testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) pxor %xmm1, %xmm1 pxor %xmm2, %xmm2 @@ -65,25 +65,25 @@ salq $ZBASE_SHIFT, INCX cmpq $2 * SIZE, INCX - jne .L100 + jne L(100) subq $-32 * SIZE, X addq M, M cmpq $3, M - jle .L18 + jle L(18) testq $4, X - je .L05 + je L(05) movss -32 * SIZE(X), %xmm0 addq $SIZE, X decq M - jle .L998 + jle L(998) ALIGN_3 -.L05: +L(05): testq $8, X - je .L10 + je L(10) #ifdef movsd xorps %xmm1, %xmm1 @@ -91,13 +91,13 @@ movsd -32 * SIZE(X), %xmm1 addq $2 * SIZE, X subq $2, M - jle .L998 + jle L(998) ALIGN_3 -.L10: +L(10): movq M, I sarq $5, I - jle .L14 + jle L(14) movaps -32 * SIZE(X), %xmm4 movaps -28 * SIZE(X), %xmm5 @@ -109,10 +109,10 @@ movaps -8 * SIZE(X), %xmm10 movaps -4 * SIZE(X), %xmm11 decq I - jle .L12 + jle L(12) ALIGN_3 -.L11: +L(11): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -147,10 +147,10 @@ subq $-32 * SIZE, X decq I - jg .L11 + jg L(11) ALIGN_3 -.L12: +L(12): addps %xmm4, %xmm0 addps %xmm5, %xmm1 @@ -166,13 +166,13 @@ addq $32 * SIZE, X ALIGN_3 -.L14: +L(14): testq $31, M - jle .L998 + jle L(998) -.L15: +L(15): testq $16, M - je .L16 + je L(16) movaps -32 * SIZE(X), %xmm4 addps %xmm4, %xmm0 @@ -189,9 +189,9 @@ addq $16 * SIZE, X ALIGN_3 -.L16: +L(16): testq $8, M - je .L17 + je L(17) movaps -32 * SIZE(X), %xmm4 addps %xmm4, %xmm0 @@ -202,18 +202,18 @@ addq $8 * SIZE, X ALIGN_3 -.L17: +L(17): testq $4, M - je .L18 + je L(18) movaps -32 * SIZE(X), %xmm6 addps %xmm6, %xmm2 addq $4 * SIZE, X ALIGN_3 -.L18: +L(18): testq $2, M - je .L19 + je L(19) #ifdef movsd xorps %xmm7, %xmm7 @@ -223,22 +223,22 @@ addq $2 * SIZE, X ALIGN_3 -.L19: +L(19): testq $1, M - je .L998 + je L(998) movss -32 * SIZE(X), %xmm6 addps %xmm6, %xmm2 - jmp .L998 + jmp L(998) ALIGN_4 -.L100: +L(100): movq M, I sarq $2, I - jle .L105 + jle L(105) ALIGN_4 -.L101: +L(101): movsd (X), %xmm4 addq INCX, X movhps (X), %xmm4 @@ -254,26 +254,26 @@ addps %xmm5, %xmm1 decq I - jg .L101 + jg L(101) ALIGN_4 -.L105: +L(105): #ifdef movsd xorps %xmm4, %xmm4 #endif andq $3, M - jle .L998 + jle L(998) ALIGN_4 -.L106: +L(106): movsd (X), %xmm4 addps %xmm4, %xmm0 addq INCX, X decq M - jg .L106 + jg L(106) ALIGN_4 -.L998: +L(998): addps %xmm1, %xmm0 addps %xmm3, %xmm2 addps %xmm2, %xmm0 @@ -291,7 +291,7 @@ #endif ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/zsum_sse2.S b/kernel/x86_64/zsum_sse2.S index 6f667164dc..7f9ef34221 100644 --- a/kernel/x86_64/zsum_sse2.S +++ b/kernel/x86_64/zsum_sse2.S @@ -54,9 +54,9 @@ xorps %xmm0, %xmm0 testq M, M - jle .L999 + jle L(999) testq INCX, INCX - jle .L999 + jle L(999) xorps %xmm1, %xmm1 xorps %xmm2, %xmm2 @@ -65,13 +65,13 @@ salq $ZBASE_SHIFT, INCX cmpq $2 * SIZE, INCX - jne .L40 + jne L(40) subq $-16 * SIZE, X addq M, M testq $SIZE, X - je .L05 + je L(05) #ifdef movsd xorps %xmm0, %xmm0 @@ -80,13 +80,13 @@ addq $SIZE, X subq $1, M - jle .L999 + jle L(999) ALIGN_3 -.L05: +L(05): movq M, I sarq $4, I - jle .L20 + jle L(20) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -99,10 +99,10 @@ movaps -2 * SIZE(X), %xmm11 decq I - jle .L11 + jle L(11) ALIGN_4 -.L10: +L(10): #ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -137,10 +137,10 @@ subq $-16 * SIZE, X decq I - jg .L10 + jg L(10) ALIGN_4 -.L11: +L(11): addpd %xmm4, %xmm0 addpd %xmm5, %xmm1 @@ -155,12 +155,12 @@ subq $-16 * SIZE, X ALIGN_3 -.L20: +L(20): andq $15, M - jle .L998 + jle L(998) testq $8, M - je .L21 + je L(21) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -174,9 +174,9 @@ addq $8 * SIZE, X ALIGN_3 -.L21: +L(21): testq $4, M - je .L22 + je L(22) movaps -16 * SIZE(X), %xmm4 movaps -14 * SIZE(X), %xmm5 @@ -187,34 +187,34 @@ addq $4 * SIZE, X ALIGN_3 -.L22: +L(22): testq $2, M - je .L23 + je L(23) movaps -16 * SIZE(X), %xmm6 addpd %xmm6, %xmm3 addq $2 * SIZE, X -.L23: +L(23): testq $1, M - je .L998 + je L(998) #ifdef movsd xorps %xmm4, %xmm4 #endif movsd -16 * SIZE(X), %xmm4 addsd %xmm4, %xmm0 - jmp .L998 + jmp L(998) ALIGN_3 -.L40: +L(40): movq M, I sarq $2, I - jle .L60 + jle L(60) ALIGN_4 -.L50: +L(50): #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) prefetcht0 PREFETCHSIZE * SIZE(X) #endif @@ -244,25 +244,25 @@ addpd %xmm7, %xmm3 decq I - jg .L50 + jg L(50) ALIGN_4 -.L60: +L(60): andq $3, M - jle .L998 + jle L(998) ALIGN_4 -.L61: +L(61): movsd 0 * SIZE(X), %xmm4 movhpd 1 * SIZE(X), %xmm4 addpd %xmm4, %xmm0 addq INCX, X decq M - jg .L61 + jg L(61) ALIGN_4 -.L998: +L(998): addpd %xmm1, %xmm0 addpd %xmm3, %xmm2 addpd %xmm2, %xmm0 @@ -275,7 +275,7 @@ #endif ALIGN_4 -.L999: +L(999): RESTOREREGISTERS ret diff --git a/kernel/x86_64/zswap.S b/kernel/x86_64/zswap.S index 68568f7ddd..b9ab1f4b9e 100644 --- a/kernel/x86_64/zswap.S +++ b/kernel/x86_64/zswap.S @@ -82,16 +82,16 @@ salq $ZBASE_SHIFT, INCY cmpq $2 * SIZE, INCX - jne .L14 + jne L(14) cmpq $2 * SIZE, INCY - jne .L14 + jne L(14) movq N, %rax sarq $2, %rax - jle .L15 + jle L(15) ALIGN_3 -.L16: +L(16): #ifdef XDOUBLE movq 0(X), %mm0 movq 8(X), %mm1 @@ -233,16 +233,16 @@ addq $8 * SIZE, X addq $8 * SIZE, Y decq %rax - jg .L16 + jg L(16) ALIGN_3 -.L15: +L(15): movq N, %rax andq $3, %rax - jle .L27 + jle L(27) ALIGN_3 -.L22: +L(22): #ifdef XDOUBLE movq 0(X), %mm0 movq 8(X), %mm1 @@ -279,21 +279,21 @@ addq $2 * SIZE, X addq $2 * SIZE, Y decq %rax - jg .L22 - jmp .L27 + jg L(22) + jmp L(27) ALIGN_3 /* INCX != 1 or INCY != 1 */ -.L14: +L(14): movq N, %rax movq X, XX movq Y, YY sarq $1, %rax - jle .L28 + jle L(28) ALIGN_2 -.L29: +L(29): #ifdef XDOUBLE movq 0(X), %mm0 movq 8(X), %mm1 @@ -388,16 +388,16 @@ addq INCY, YY #endif decq %rax - jg .L29 + jg L(29) ALIGN_3 -.L28: +L(28): movq N, %rax andq $1, %rax - jle .L27 + jle L(27) ALIGN_3 -.L35: +L(35): #ifdef XDOUBLE movq 0(X), %mm0 movq 8(X), %mm1 @@ -436,10 +436,10 @@ addq INCY, Y decq %rax - jg .L35 + jg L(35) ALIGN_3 -.L27: +L(27): EMMS xorq %rax,%rax diff --git a/kernel/x86_64/zswap_sse.S b/kernel/x86_64/zswap_sse.S index 12f9875ba8..720a676cd2 100644 --- a/kernel/x86_64/zswap_sse.S +++ b/kernel/x86_64/zswap_sse.S @@ -75,12 +75,12 @@ salq $ZBASE_SHIFT, INCY testq M, M - jle .L19 + jle L(19) cmpq $2 * SIZE, INCX - jne .L50 + jne L(50) cmpq $2 * SIZE, INCY - jne .L50 + jne L(50) addq M, M @@ -88,10 +88,10 @@ subq $-32 * SIZE, Y cmpq $3, M - jle .L16 + jle L(16) testq $SIZE, Y - je .L05 + je L(05) movss -32 * SIZE(X), %xmm0 movss -32 * SIZE(Y), %xmm1 @@ -104,9 +104,9 @@ decq M ALIGN_3 -.L05: +L(05): testq $2 * SIZE, Y - je .L10 + je L(10) movsd -32 * SIZE(X), %xmm0 movsd -32 * SIZE(Y), %xmm1 @@ -117,25 +117,25 @@ addq $2 * SIZE, X addq $2 * SIZE, Y subq $2, M - jle .L19 + jle L(19) ALIGN_3 -.L10: +L(10): cmpq $3, M - jle .L16 + jle L(16) testq $2 * SIZE, X - jne .L30 + jne L(30) testq $1 * SIZE, X - jne .L20 + jne L(20) movq M, %rax sarq $5, %rax - jle .L13 + jle L(13) ALIGN_3 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -204,12 +204,12 @@ subq $-32 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L13: +L(13): testq $16, M - jle .L14 + jle L(14) movaps -32 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -239,9 +239,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $8, M - jle .L15 + jle L(15) movaps -32 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -259,9 +259,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $4, M - jle .L16 + jle L(16) movaps -32 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -273,9 +273,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L16: +L(16): testq $2, M - jle .L17 + jle L(17) movsd -32 * SIZE(X), %xmm0 movsd -32 * SIZE(Y), %xmm1 @@ -286,9 +286,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L17: +L(17): testq $1, M - jle .L19 + jle L(19) movss -32 * SIZE(X), %xmm0 movss -32 * SIZE(Y), %xmm1 @@ -297,7 +297,7 @@ movss %xmm0, -32 * SIZE(Y) ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -309,7 +309,7 @@ ret ALIGN_3 -.L20: +L(20): movaps -33 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -321,10 +321,10 @@ movq M, %rax sarq $5, %rax - jle .L23 + jle L(23) ALIGN_4 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -425,12 +425,12 @@ subq $-32 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L23: +L(23): testq $16, M - jle .L24 + jle L(24) movaps -29 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -476,9 +476,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $8, M - jle .L25 + jle L(25) movaps -29 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -504,9 +504,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $4, M - jle .L26 + jle L(26) movaps -29 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -525,7 +525,7 @@ addq $4 * SIZE, Y ALIGN_3 -.L26: +L(26): pshufd $0x39, %xmm0, %xmm2 pshufd $0xff, %xmm0, %xmm0 @@ -533,7 +533,7 @@ movss %xmm0, -30 * SIZE(Y) testq $2, M - jle .L27 + jle L(27) movsd -29 * SIZE(X), %xmm0 movsd -29 * SIZE(Y), %xmm1 @@ -545,9 +545,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L27: +L(27): testq $1, M - jle .L29 + jle L(29) movss -29 * SIZE(X), %xmm0 movss -29 * SIZE(Y), %xmm1 @@ -556,7 +556,7 @@ movss %xmm1, -29 * SIZE(X) ALIGN_3 -.L29: +L(29): xorq %rax,%rax RESTOREREGISTERS @@ -568,9 +568,9 @@ ret ALIGN_3 -.L30: +L(30): testq $1 * SIZE, X - jne .L40 + jne L(40) movhps -32 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -580,10 +580,10 @@ movq M, %rax sarq $5, %rax - jle .L33 + jle L(33) ALIGN_4 -.L31: +L(31): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -668,12 +668,12 @@ subq $-32 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L33: +L(33): testq $16, M - jle .L34 + jle L(34) movaps -30 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -711,9 +711,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L34: +L(34): testq $8, M - jle .L35 + jle L(35) movaps -30 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -735,9 +735,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L35: +L(35): testq $4, M - jle .L36 + jle L(36) movaps -30 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -754,11 +754,11 @@ addq $4 * SIZE, Y ALIGN_3 -.L36: +L(36): movhps %xmm0, -32 * SIZE(Y) testq $2, M - jle .L37 + jle L(37) movsd -30 * SIZE(X), %xmm0 movsd -30 * SIZE(Y), %xmm1 @@ -770,9 +770,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L37: +L(37): testq $1, M - jle .L39 + jle L(39) movss -30 * SIZE(X), %xmm0 movss -30 * SIZE(Y), %xmm1 @@ -781,7 +781,7 @@ movss %xmm1, -30 * SIZE(X) ALIGN_3 -.L39: +L(39): xorq %rax,%rax RESTOREREGISTERS @@ -793,7 +793,7 @@ ret ALIGN_3 -.L40: +L(40): movaps -35 * SIZE(X), %xmm0 movaps -32 * SIZE(Y), %xmm1 @@ -803,10 +803,10 @@ movq M, %rax sarq $5, %rax - jle .L43 + jle L(43) ALIGN_4 -.L41: +L(41): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -907,12 +907,12 @@ subq $-32 * SIZE, Y decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L43: +L(43): testq $16, M - jle .L44 + jle L(44) movaps -31 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -958,9 +958,9 @@ addq $16 * SIZE, Y ALIGN_3 -.L44: +L(44): testq $8, M - jle .L45 + jle L(45) movaps -31 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -986,9 +986,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L45: +L(45): testq $4, M - jle .L46 + jle L(46) movaps -31 * SIZE(X), %xmm2 movaps -28 * SIZE(Y), %xmm3 @@ -1007,7 +1007,7 @@ addq $4 * SIZE, Y ALIGN_3 -.L46: +L(46): movsd -31 * SIZE(X), %xmm2 pshufd $0x39, %xmm1, %xmm1 @@ -1022,7 +1022,7 @@ addq $3 * SIZE, Y testq $2, M - jle .L47 + jle L(47) movsd -32 * SIZE(X), %xmm0 movsd -32 * SIZE(Y), %xmm1 @@ -1034,9 +1034,9 @@ addq $2 * SIZE, Y ALIGN_3 -.L47: +L(47): testq $1, M - jle .L49 + jle L(49) movss -32 * SIZE(X), %xmm0 movss -32 * SIZE(Y), %xmm1 @@ -1045,7 +1045,7 @@ movss %xmm1, -32 * SIZE(X) ALIGN_3 -.L49: +L(49): xorq %rax,%rax RESTOREREGISTERS @@ -1057,13 +1057,13 @@ ret ALIGN_3 -.L50: +L(50): movq M, %rax sarq $2, %rax - jle .L55 + jle L(55) ALIGN_3 -.L51: +L(51): movsd (X), %xmm0 movsd (Y), %xmm1 @@ -1097,16 +1097,16 @@ addq INCY, Y decq %rax - jg .L51 + jg L(51) ALIGN_3 -.L55: +L(55): movq M, %rax andq $3, %rax - jle .L57 + jle L(57) ALIGN_3 -.L56: +L(56): movsd (X), %xmm0 movsd (Y), %xmm1 @@ -1116,10 +1116,10 @@ addq INCY, Y decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L57: +L(57): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/zswap_sse2.S b/kernel/x86_64/zswap_sse2.S index c505014dd6..8072e4f21b 100644 --- a/kernel/x86_64/zswap_sse2.S +++ b/kernel/x86_64/zswap_sse2.S @@ -75,28 +75,28 @@ salq $ZBASE_SHIFT, INCY testq M, M - jle .L19 + jle L(19) cmpq $2 * SIZE, INCX - jne .L50 + jne L(50) cmpq $2 * SIZE, INCY - jne .L50 + jne L(50) subq $-16 * SIZE, X subq $-16 * SIZE, Y testq $SIZE, Y - jne .L30 + jne L(30) testq $SIZE, X - jne .L20 + jne L(20) movq M, %rax sarq $3, %rax - jle .L13 + jle L(13) ALIGN_3 -.L11: +L(11): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -165,12 +165,12 @@ subq $-16 * SIZE, X decq %rax - jg .L11 + jg L(11) ALIGN_3 -.L13: +L(13): testq $4, M - jle .L14 + jle L(14) movaps -16 * SIZE(X), %xmm0 movaps -16 * SIZE(Y), %xmm1 @@ -200,9 +200,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L14: +L(14): testq $2, M - jle .L15 + jle L(15) movaps -16 * SIZE(X), %xmm0 movaps -16 * SIZE(Y), %xmm1 @@ -220,9 +220,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L15: +L(15): testq $1, M - jle .L19 + jle L(19) movaps -16 * SIZE(X), %xmm0 movaps -16 * SIZE(Y), %xmm1 @@ -234,7 +234,7 @@ addq $2 * SIZE, Y ALIGN_3 -.L19: +L(19): xorq %rax,%rax RESTOREREGISTERS @@ -245,20 +245,20 @@ ret ALIGN_3 -.L20: +L(20): movhps -16 * SIZE(X), %xmm0 movaps -16 * SIZE(Y), %xmm1 movlps %xmm1, -16 * SIZE(X) decq M - jle .L29 + jle L(29) movq M, %rax sarq $3, %rax - jle .L23 + jle L(23) ALIGN_4 -.L21: +L(21): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -343,12 +343,12 @@ subq $-16 * SIZE, Y decq %rax - jg .L21 + jg L(21) ALIGN_3 -.L23: +L(23): testq $4, M - jle .L24 + jle L(24) movaps -15 * SIZE(X), %xmm2 movaps -14 * SIZE(Y), %xmm3 @@ -386,9 +386,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L24: +L(24): testq $2, M - jle .L25 + jle L(25) movaps -15 * SIZE(X), %xmm2 movaps -14 * SIZE(Y), %xmm3 @@ -410,9 +410,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L25: +L(25): testq $1, M - jle .L29 + jle L(29) movaps -15 * SIZE(X), %xmm2 movaps -14 * SIZE(Y), %xmm3 @@ -429,7 +429,7 @@ addq $2 * SIZE, Y ALIGN_3 -.L29: +L(29): movaps -15 * SIZE(X), %xmm2 movhps %xmm1, -15 * SIZE(X) @@ -447,23 +447,23 @@ ret ALIGN_3 -.L30: +L(30): testq $SIZE, X - jne .L40 + jne L(40) movhps -16 * SIZE(Y), %xmm0 movaps -16 * SIZE(X), %xmm1 movlps %xmm1, -16 * SIZE(Y) decq M - jle .L39 + jle L(39) movq M, %rax sarq $3, %rax - jle .L33 + jle L(33) ALIGN_4 -.L31: +L(31): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(Y) #endif @@ -548,12 +548,12 @@ subq $-16 * SIZE, Y decq %rax - jg .L31 + jg L(31) ALIGN_3 -.L33: +L(33): testq $4, M - jle .L34 + jle L(34) movaps -15 * SIZE(Y), %xmm2 movaps -14 * SIZE(X), %xmm3 @@ -591,9 +591,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L34: +L(34): testq $2, M - jle .L35 + jle L(35) movaps -15 * SIZE(Y), %xmm2 movaps -14 * SIZE(X), %xmm3 @@ -615,9 +615,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L35: +L(35): testq $1, M - jle .L39 + jle L(39) movaps -15 * SIZE(Y), %xmm2 movaps -14 * SIZE(X), %xmm3 @@ -634,7 +634,7 @@ addq $2 * SIZE, Y ALIGN_3 -.L39: +L(39): movaps -15 * SIZE(Y), %xmm2 movhps %xmm1, -15 * SIZE(Y) @@ -652,7 +652,7 @@ ret ALIGN_3 -.L40: +L(40): movsd -16 * SIZE(X), %xmm0 movsd -16 * SIZE(Y), %xmm1 @@ -662,14 +662,14 @@ addq $SIZE, X addq $SIZE, Y decq M - jle .L49 + jle L(49) movq M, %rax sarq $3, %rax - jle .L43 + jle L(43) ALIGN_3 -.L41: +L(41): #ifdef PREFETCHW PREFETCHW (PREFETCHSIZE + 0) - PREOFFSET(X) #endif @@ -738,12 +738,12 @@ subq $-16 * SIZE, X decq %rax - jg .L41 + jg L(41) ALIGN_3 -.L43: +L(43): testq $4, M - jle .L44 + jle L(44) movaps -16 * SIZE(X), %xmm0 movaps -16 * SIZE(Y), %xmm1 @@ -773,9 +773,9 @@ addq $8 * SIZE, Y ALIGN_3 -.L44: +L(44): testq $2, M - jle .L45 + jle L(45) movaps -16 * SIZE(X), %xmm0 movaps -16 * SIZE(Y), %xmm1 @@ -793,9 +793,9 @@ addq $4 * SIZE, Y ALIGN_3 -.L45: +L(45): testq $1, M - jle .L49 + jle L(49) movaps -16 * SIZE(X), %xmm0 movaps -16 * SIZE(Y), %xmm1 @@ -807,7 +807,7 @@ addq $2 * SIZE, Y ALIGN_3 -.L49: +L(49): movsd -16 * SIZE(X), %xmm0 movsd -16 * SIZE(Y), %xmm1 @@ -824,18 +824,18 @@ ret ALIGN_3 -.L50: +L(50): testq $SIZE, X - jne .L60 + jne L(60) testq $SIZE, Y - jne .L60 + jne L(60) movq M, %rax sarq $2, %rax - jle .L55 + jle L(55) ALIGN_3 -.L51: +L(51): movaps (X), %xmm0 movaps (Y), %xmm1 @@ -869,16 +869,16 @@ addq INCY, Y decq %rax - jg .L51 + jg L(51) ALIGN_3 -.L55: +L(55): movq M, %rax andq $3, %rax - jle .L57 + jle L(57) ALIGN_3 -.L56: +L(56): movaps (X), %xmm0 movaps (Y), %xmm1 @@ -888,10 +888,10 @@ addq INCY, Y decq %rax - jg .L56 + jg L(56) ALIGN_3 -.L57: +L(57): xorq %rax, %rax RESTOREREGISTERS @@ -903,13 +903,13 @@ ret ALIGN_3 -.L60: +L(60): movq M, %rax sarq $2, %rax - jle .L65 + jle L(65) ALIGN_3 -.L61: +L(61): movsd 0 * SIZE(X), %xmm0 movhps 1 * SIZE(X), %xmm0 movsd 0 * SIZE(Y), %xmm1 @@ -959,16 +959,16 @@ addq INCY, Y decq %rax - jg .L61 + jg L(61) ALIGN_3 -.L65: +L(65): movq M, %rax andq $3, %rax - jle .L67 + jle L(67) ALIGN_3 -.L66: +L(66): movsd 0 * SIZE(X), %xmm0 movhps 1 * SIZE(X), %xmm0 movsd 0 * SIZE(Y), %xmm1 @@ -982,10 +982,10 @@ addq INCY, Y decq %rax - jg .L66 + jg L(66) ALIGN_3 -.L67: +L(67): xorq %rax, %rax RESTOREREGISTERS diff --git a/kernel/x86_64/zsymv_L_sse.S b/kernel/x86_64/zsymv_L_sse.S index df190c64c3..0199890eb6 100644 --- a/kernel/x86_64/zsymv_L_sse.S +++ b/kernel/x86_64/zsymv_L_sse.S @@ -188,7 +188,7 @@ salq $ZBASE_SHIFT, LDA testq M, M - jle .L999 + jle L(999) pcmpeqb %xmm2, %xmm2 xorpd %xmm3, %xmm3 @@ -203,10 +203,10 @@ movq M, %rax sarq $2, %rax - jle .L02 + jle L(02) ALIGN_3 -.L01: +L(01): MOVDDUP(0 * SIZE, X, %xmm3) MOVDDUP(1 * SIZE, X, %xmm4) addq INCX, X @@ -259,16 +259,16 @@ subq $-16 * SIZE, XX decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L02: +L(02): movq M, %rax andq $3, %rax - jle .L05 + jle L(05) ALIGN_3 -.L03: +L(03): MOVDDUP(0 * SIZE, X, %xmm3) MOVDDUP(1 * SIZE, X, %xmm4) addq INCX, X @@ -285,10 +285,10 @@ addq $4 * SIZE, XX decq %rax - jg .L03 + jg L(03) ALIGN_3 -.L05: +L(05): /* now we don't need original X */ movq Y, NEW_Y @@ -296,17 +296,17 @@ andq $-512, XX cmpq $2 * SIZE, INCY - je .L10 + je L(10) movq Y, YY movq XX, NEW_Y movq M, %rax sarq $2, %rax - jle .L07 + jle L(07) ALIGN_3 -.L06: +L(06): movsd 0 * SIZE(YY), %xmm0 movhpd 1 * SIZE(YY), %xmm0 addq INCY, YY @@ -327,16 +327,16 @@ addq $8 * SIZE, XX decq %rax - jg .L06 + jg L(06) ALIGN_3 -.L07: +L(07): movq M, %rax andq $3, %rax - jle .L10 + jle L(10) ALIGN_3 -.L08: +L(08): movsd 0 * SIZE(YY), %xmm0 movhpd 1 * SIZE(YY), %xmm0 addq INCY, YY @@ -345,17 +345,17 @@ addq $2 * SIZE, XX decq %rax - jg .L08 + jg L(08) ALIGN_3 -.L10: +L(10): xorq IS, IS # is = 0 cmpq $2, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): movq A, A1 leaq (A, LDA, 1), A2 leaq 4 * SIZE(A, LDA, 2), A @@ -421,10 +421,10 @@ subq IS, I subq $2, I sarq $2, I - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): movapd xtemp1, xt1 mulpd a1, xt1 mulpd atemp1, a1 @@ -579,15 +579,15 @@ addq $ 8 * SIZE, A2 decq I - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): movq M, I subq IS, I subq $2, I testq $2, I - jle .L16 + jle L(16) movapd xtemp1, xt1 mulpd a1, xt1 @@ -663,9 +663,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L16: +L(16): testq $1, M - jle .L18 + jle L(18) MOVDDUP(1 * SIZE, A1, a2) @@ -701,7 +701,7 @@ movhpd yy1, 1 * SIZE(YY) ALIGN_3 -.L18: +L(18): leaq (, IS, SIZE), I movsd 0 * SIZE(NEW_Y, I, 2), yy1 @@ -722,13 +722,13 @@ movq IS, I addq $2, I cmpq M, I - jle .L11 + jle L(11) ALIGN_3 -.L20: +L(20): HALT testq $1, N - jle .L990 + jle L(990) leaq (, IS, SIZE), I @@ -750,16 +750,16 @@ movhpd yy1, 1 * SIZE(NEW_Y, I, 2) ALIGN_3 -.L990: +L(990): cmpq $2 * SIZE, INCY - je .L999 + je L(999) movq M, %rax sarq $2, %rax - jle .L997 + jle L(997) ALIGN_3 -.L996: +L(996): movapd 0 * SIZE(NEW_Y), %xmm0 movapd 2 * SIZE(NEW_Y), %xmm1 movapd 4 * SIZE(NEW_Y), %xmm2 @@ -780,16 +780,16 @@ addq $8 * SIZE, NEW_Y decq %rax - jg .L996 + jg L(996) ALIGN_3 -.L997: +L(997): movq M, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_3 -.L998: +L(998): movapd 0 * SIZE(NEW_Y), %xmm0 movsd %xmm0, 0 * SIZE(Y) @@ -799,10 +799,10 @@ addq $2 * SIZE, NEW_Y decq %rax - jg .L998 + jg L(998) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zsymv_L_sse2.S b/kernel/x86_64/zsymv_L_sse2.S index acc5ad592b..dd2d4c0250 100644 --- a/kernel/x86_64/zsymv_L_sse2.S +++ b/kernel/x86_64/zsymv_L_sse2.S @@ -224,7 +224,7 @@ salq $ZBASE_SHIFT, LDA testq M, M - jle .L999 + jle L(999) pcmpeqb %xmm2, %xmm2 xorpd %xmm3, %xmm3 @@ -239,10 +239,10 @@ movq M, %rax sarq $2, %rax - jle .L02 + jle L(02) ALIGN_3 -.L01: +L(01): MOVDDUP(0 * SIZE, X, %xmm3) MOVDDUP(1 * SIZE, X, %xmm4) addq INCX, X @@ -295,16 +295,16 @@ subq $-16 * SIZE, XX decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L02: +L(02): movq M, %rax andq $3, %rax - jle .L05 + jle L(05) ALIGN_3 -.L03: +L(03): MOVDDUP(0 * SIZE, X, %xmm3) MOVDDUP(1 * SIZE, X, %xmm4) addq INCX, X @@ -321,10 +321,10 @@ addq $4 * SIZE, XX decq %rax - jg .L03 + jg L(03) ALIGN_3 -.L05: +L(05): /* now we don't need original X */ movq Y, NEW_Y @@ -332,17 +332,17 @@ andq $-512, XX cmpq $2 * SIZE, INCY - je .L10 + je L(10) movq Y, YY movq XX, NEW_Y movq M, %rax sarq $2, %rax - jle .L07 + jle L(07) ALIGN_3 -.L06: +L(06): movsd 0 * SIZE(YY), %xmm0 movhpd 1 * SIZE(YY), %xmm0 addq INCY, YY @@ -363,16 +363,16 @@ addq $8 * SIZE, XX decq %rax - jg .L06 + jg L(06) ALIGN_3 -.L07: +L(07): movq M, %rax andq $3, %rax - jle .L10 + jle L(10) ALIGN_3 -.L08: +L(08): movsd 0 * SIZE(YY), %xmm0 movhpd 1 * SIZE(YY), %xmm0 addq INCY, YY @@ -381,17 +381,17 @@ addq $2 * SIZE, XX decq %rax - jg .L08 + jg L(08) ALIGN_3 -.L10: +L(10): xorq IS, IS # is = 0 cmpq $2, N - jl .L20 + jl L(20) ALIGN_3 -.L11: +L(11): movq A, A1 leaq (A, LDA, 1), A2 leaq 4 * SIZE(A, LDA, 2), A @@ -465,18 +465,18 @@ subq IS, I subq $2, I sarq $2, I - jle .L14 + jle L(14) MOVDDUP(6 * SIZE - (4 * SIZE), A2, a2) ALIGN_3 -.L12_prep: +L(12_prep): movsd 0 * SIZE(YY), yy1 movhpd 1 * SIZE(YY), yy1 movsd 2 * SIZE(YY), yy2 movhpd 3 * SIZE(YY), yy2 -.L12: +L(12): movapd xtemp1, xt1 mulpd a1, xt1 mulpd atemp1, a1 @@ -635,32 +635,32 @@ addq $ 8 * SIZE, A2 decq I - jg .L12_prep - jmp .L15 + jg L(12_prep) + jmp L(15) ALIGN_3 -.L14: +L(14): movq M, I subq IS, I subq $2, I testq $2, I - jle .L16 + jle L(16) MOVDDUP(6 * SIZE - (4 * SIZE), A2, a2) -.L15: +L(15): movq M, I subq IS, I subq $2, I testq $2, I - jle .L16 + jle L(16) movsd 0 * SIZE(YY), yy1 movhpd 1 * SIZE(YY), yy1 movsd 2 * SIZE(YY), yy2 movhpd 3 * SIZE(YY), yy2 -.L15_pastcheck: +L(15_pastcheck): movapd xtemp1, xt1 mulpd a1, xt1 mulpd atemp1, a1 @@ -731,9 +731,9 @@ addq $4 * SIZE, A2 ALIGN_3 -.L16: +L(16): testq $1, M - jle .L18 + jle L(18) MOVDDUP(1 * SIZE, A1, a2) @@ -772,7 +772,7 @@ movhpd yy1, 1 * SIZE(YY) ALIGN_3 -.L18: +L(18): leaq (, IS, SIZE), I movsd 0 * SIZE(NEW_Y, I, 2), yy1 @@ -793,12 +793,12 @@ movq IS, I addq $2, I cmpq N, I - jle .L11 + jle L(11) ALIGN_3 -.L20: +L(20): testq $1, N - jle .L990 + jle L(990) leaq (, IS, SIZE), I @@ -827,16 +827,16 @@ movhpd yy1, 1 * SIZE(NEW_Y, I, 2) ALIGN_3 -.L990: +L(990): cmpq $2 * SIZE, INCY - je .L999 + je L(999) movq M, %rax sarq $2, %rax - jle .L997 + jle L(997) ALIGN_3 -.L996: +L(996): movapd 0 * SIZE(NEW_Y), %xmm0 movapd 2 * SIZE(NEW_Y), %xmm1 movapd 4 * SIZE(NEW_Y), %xmm2 @@ -857,16 +857,16 @@ addq $8 * SIZE, NEW_Y decq %rax - jg .L996 + jg L(996) ALIGN_3 -.L997: +L(997): movq M, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_3 -.L998: +L(998): movapd 0 * SIZE(NEW_Y), %xmm0 movsd %xmm0, 0 * SIZE(Y) @@ -876,10 +876,10 @@ addq $2 * SIZE, NEW_Y decq %rax - jg .L998 + jg L(998) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zsymv_U_sse.S b/kernel/x86_64/zsymv_U_sse.S index 13176ce9c3..cf88ce2c5e 100644 --- a/kernel/x86_64/zsymv_U_sse.S +++ b/kernel/x86_64/zsymv_U_sse.S @@ -217,7 +217,7 @@ salq $ZBASE_SHIFT, LDA testq M, M - jle .L999 + jle L(999) negq IS addq M, IS @@ -243,10 +243,10 @@ movq M, %rax sarq $2, %rax - jle .L02 + jle L(02) ALIGN_3 -.L01: +L(01): movsd 0 * SIZE(X), %xmm4 addq INCX, X movhps 0 * SIZE(X), %xmm4 @@ -283,12 +283,12 @@ subq $-16 * SIZE, XX decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L02: +L(02): testq $2, M - jle .L03 + jle L(03) movsd 0 * SIZE(X), %xmm4 addq INCX, X @@ -312,9 +312,9 @@ subq $-8 * SIZE, XX ALIGN_3 -.L03: +L(03): testq $1, M - jle .L05 + jle L(05) movsd 0 * SIZE(X), %xmm4 addq INCX, X @@ -336,7 +336,7 @@ subq $-4 * SIZE, XX ALIGN_3 -.L05: +L(05): /* now we don't need original X */ movq Y, NEW_Y @@ -344,17 +344,17 @@ andq $-512, XX cmpq $2 * SIZE, INCY - je .L10 + je L(10) movq Y, YY movq XX, NEW_Y movq M, %rax sarq $2, %rax - jle .L07 + jle L(07) ALIGN_3 -.L06: +L(06): movsd 0 * SIZE(YY), %xmm0 addq INCY, YY movhps 0 * SIZE(YY), %xmm0 @@ -369,16 +369,16 @@ addq $8 * SIZE, XX decq %rax - jg .L06 + jg L(06) ALIGN_3 -.L07: +L(07): movq M, %rax andq $3, %rax - jle .L10 + jle L(10) ALIGN_3 -.L08: +L(08): movsd 0 * SIZE(YY), %xmm0 addq INCY, YY @@ -386,17 +386,17 @@ addq $2 * SIZE, XX decq %rax - jg .L08 + jg L(08) ALIGN_3 -.L10: +L(10): movq IS, I addq $2, I cmpq M, I - jg .L20 + jg L(20) ALIGN_3 -.L11: +L(11): movq A, A1 leaq (A, LDA, 1), A2 leaq (A, LDA, 2), A @@ -423,10 +423,10 @@ movq IS, I sarq $2, I - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): HALT subq $-16 * SIZE, XX @@ -435,12 +435,12 @@ addq $ 8 * SIZE, A2 decq I - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): testq $2, IS - jle .L18 + jle L(18) movsd 0 * SIZE(YY), yy1 movhps 2 * SIZE(YY), yy1 @@ -489,7 +489,7 @@ addq $4 * SIZE, A2 ALIGN_3 -.L18: +L(18): leaq (, IS, 4), I movaps 0 * SIZE(NEW_X, I, SIZE), atemp1 @@ -530,24 +530,24 @@ movq IS, I addq $2, I cmpq M, I - jle .L11 + jle L(11) ALIGN_3 -.L20: +L(20): testq $1, M - jle .L990 + jle L(990) -.L990: +L(990): cmpq $2 * SIZE, INCY - je .L999 + je L(999) movq M, %rax sarq $2, %rax - jle .L997 + jle L(997) ALIGN_3 -.L996: +L(996): movaps 0 * SIZE(NEW_Y), %xmm0 movaps 4 * SIZE(NEW_Y), %xmm1 @@ -562,16 +562,16 @@ addq $8 * SIZE, NEW_Y decq %rax - jg .L996 + jg L(996) ALIGN_3 -.L997: +L(997): movq M, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_3 -.L998: +L(998): movlps 0 * SIZE(NEW_Y), %xmm0 addq $2 * SIZE, NEW_Y @@ -579,10 +579,10 @@ addq INCY, Y decq %rax - jg .L998 + jg L(998) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/zsymv_U_sse2.S b/kernel/x86_64/zsymv_U_sse2.S index fa1fe9fe9e..6743d81ce3 100644 --- a/kernel/x86_64/zsymv_U_sse2.S +++ b/kernel/x86_64/zsymv_U_sse2.S @@ -223,7 +223,7 @@ salq $ZBASE_SHIFT, LDA testq M, M - jle .L999 + jle L(999) negq IS addq M, IS @@ -245,10 +245,10 @@ movq M, %rax sarq $2, %rax - jle .L02 + jle L(02) ALIGN_3 -.L01: +L(01): MOVDDUP(0 * SIZE, X, %xmm3) MOVDDUP(1 * SIZE, X, %xmm4) addq INCX, X @@ -301,16 +301,16 @@ subq $-16 * SIZE, XX decq %rax - jg .L01 + jg L(01) ALIGN_3 -.L02: +L(02): movq M, %rax andq $3, %rax - jle .L05 + jle L(05) ALIGN_3 -.L03: +L(03): MOVDDUP(0 * SIZE, X, %xmm3) MOVDDUP(1 * SIZE, X, %xmm4) addq INCX, X @@ -327,10 +327,10 @@ addq $4 * SIZE, XX decq %rax - jg .L03 + jg L(03) ALIGN_3 -.L05: +L(05): /* now we don't need original X */ movq Y, NEW_Y @@ -338,17 +338,17 @@ andq $-512, XX cmpq $2 * SIZE, INCY - je .L10 + je L(10) movq Y, YY movq XX, NEW_Y movq M, %rax sarq $2, %rax - jle .L07 + jle L(07) ALIGN_3 -.L06: +L(06): movsd 0 * SIZE(YY), %xmm0 movhpd 1 * SIZE(YY), %xmm0 addq INCY, YY @@ -369,16 +369,16 @@ addq $8 * SIZE, XX decq %rax - jg .L06 + jg L(06) ALIGN_3 -.L07: +L(07): movq M, %rax andq $3, %rax - jle .L10 + jle L(10) ALIGN_3 -.L08: +L(08): movsd 0 * SIZE(YY), %xmm0 movhpd 1 * SIZE(YY), %xmm0 addq INCY, YY @@ -387,17 +387,17 @@ addq $2 * SIZE, XX decq %rax - jg .L08 + jg L(08) ALIGN_3 -.L10: +L(10): movq IS, I addq $2, I cmpq M, I - jg .L20 + jg L(20) ALIGN_3 -.L11: +L(11): movq A, A1 leaq (A, LDA, 1), A2 leaq (A, LDA, 2), A @@ -431,10 +431,10 @@ movq IS, I sarq $2, I - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): movapd xtemp1, xt1 mulpd a1, xt1 mulpd atemp1, a1 @@ -597,12 +597,12 @@ addq $ 8 * SIZE, A2 decq I - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): testq $2, IS - jle .L18 + jle L(18) movapd xtemp1, xt1 mulpd a1, xt1 @@ -673,7 +673,7 @@ addq $4 * SIZE, A2 ALIGN_3 -.L18: +L(18): MOVDDUP(0 * SIZE, A1, a1) MOVDDUP(0 * SIZE, A2, a2) @@ -733,12 +733,12 @@ movq IS, I addq $2, I cmpq M, I - jle .L11 + jle L(11) ALIGN_3 -.L20: +L(20): testq $1, M - jle .L990 + jle L(990) movq A, A1 leaq (, IS, 4), I @@ -767,10 +767,10 @@ movq IS, I sarq $1, I - jle .L28 + jle L(28) ALIGN_3 -.L22: +L(22): movapd xtemp1, xt1 movapd 8 * SIZE(XX), xtemp1 mulpd a1, xt1 @@ -818,10 +818,10 @@ addq $4 * SIZE, A1 decq I - jg .L22 + jg L(22) ALIGN_3 -.L28: +L(28): MOVDDUP(0 * SIZE, A1, a1) #ifndef HEMV @@ -844,16 +844,16 @@ movhpd yy1, 1 * SIZE(YY) ALIGN_3 -.L990: +L(990): cmpq $2 * SIZE, INCY - je .L999 + je L(999) movq M, %rax sarq $2, %rax - jle .L997 + jle L(997) ALIGN_3 -.L996: +L(996): movapd 0 * SIZE(NEW_Y), %xmm0 movapd 2 * SIZE(NEW_Y), %xmm1 movapd 4 * SIZE(NEW_Y), %xmm2 @@ -874,16 +874,16 @@ addq $8 * SIZE, NEW_Y decq %rax - jg .L996 + jg L(996) ALIGN_3 -.L997: +L(997): movq M, %rax andq $3, %rax - jle .L999 + jle L(999) ALIGN_3 -.L998: +L(998): movapd 0 * SIZE(NEW_Y), %xmm0 movsd %xmm0, 0 * SIZE(Y) @@ -893,10 +893,10 @@ addq $2 * SIZE, NEW_Y decq %rax - jg .L998 + jg L(998) ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_LN_2x1_atom.S b/kernel/x86_64/ztrsm_kernel_LN_2x1_atom.S index d3bedff12c..3da8f1d8dc 100644 --- a/kernel/x86_64/ztrsm_kernel_LN_2x1_atom.S +++ b/kernel/x86_64/ztrsm_kernel_LN_2x1_atom.S @@ -177,10 +177,10 @@ movq N, J testq N, N - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -213,7 +213,7 @@ leaq (B, %rax), BB testq $1, M - jle .L20 + jle L(20) #ifdef LN movq K, %rax @@ -253,10 +253,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADDSD2 %xmm2, %xmm9 movaps %xmm0, %xmm2 @@ -334,10 +334,10 @@ addq $8 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -347,10 +347,10 @@ andq $3, %rax BRANCH BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): ADDSD2 %xmm2, %xmm9 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -373,10 +373,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): ADDSD2 %xmm2, %xmm9 ADDSD4 %xmm6, %xmm11 @@ -481,13 +481,13 @@ #endif ALIGN_4 -.L20: +L(20): movq M, I sarq $1, I - jle .L99 + jle L(99) ALIGN_4 -.L10: +L(10): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -535,10 +535,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): ADDSD2 %xmm2, %xmm13 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps %xmm0, %xmm2 @@ -681,10 +681,10 @@ mulsd %xmm3, %xmm6 movsd 1 * SIZE(BO), %xmm3 - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -694,10 +694,10 @@ andq $3, %rax BRANCH BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): ADDSD2 %xmm2, %xmm13 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -736,10 +736,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): ADDSD2 %xmm2, %xmm13 ADDSD3 %xmm7, %xmm14 ADDSD4 %xmm6, %xmm15 @@ -942,10 +942,10 @@ addq %rax, AORIG #endif decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -963,10 +963,10 @@ subq $1, KK #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_LN_2x2_core2.S b/kernel/x86_64/ztrsm_kernel_LN_2x2_core2.S index 80485c0def..dc7f2a9eb5 100644 --- a/kernel/x86_64/ztrsm_kernel_LN_2x2_core2.S +++ b/kernel/x86_64/ztrsm_kernel_LN_2x2_core2.S @@ -196,10 +196,10 @@ movq N, J sarq $1, J # j = (n >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L01: +L(01): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -234,12 +234,12 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): prefetcht0 (PREFETCH_R + 0) * SIZE(B) movddup -16 * SIZE(B), %xmm8 @@ -268,10 +268,10 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -280,10 +280,10 @@ #endif andq $3, %rax BRANCH - jle .L05 + jle L(05) ALIGN_4 -.L04: +L(04): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 movddup -14 * SIZE(B), %xmm10 @@ -298,10 +298,10 @@ addq $ 8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L05: +L(05): #if defined(LT) || defined(RN) movq A, AO #else @@ -321,7 +321,7 @@ #endif testq $1, M - jle .L30 + jle L(30) #ifdef LN movq K, %rax @@ -356,9 +356,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L42 + je L(42) -.L41: +L(41): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -428,9 +428,9 @@ subq $ -8 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L41 + jne L(41) -.L42: +L(42): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -441,9 +441,9 @@ andq $3, %rax # if (k & 1) BRANCH - jle .L44 + jle L(44) -.L43: +L(43): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -463,10 +463,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -681,13 +681,13 @@ #endif ALIGN_4 -.L30: +L(30): movq M, I sarq $1, I # i = (m >> 2) - jle .L99 + jle L(99) ALIGN_4 -.L10: +L(10): leaq (PREFETCH_R + 0) * SIZE(B), BB #ifdef LN @@ -749,10 +749,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -873,10 +873,10 @@ subq $1, %rax BRANCH BRANCH - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -888,10 +888,10 @@ andq $3, %rax BRANCH BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): ADD1 %xmm2, %xmm10 ADD1 %xmm3, %xmm14 ADD2 %xmm4, %xmm11 @@ -928,10 +928,10 @@ addq $8 * SIZE, BO subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): ADD1 %xmm2, %xmm10 ADD1 %xmm3, %xmm14 ADD2 %xmm4, %xmm11 @@ -1309,10 +1309,10 @@ #endif decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1334,13 +1334,13 @@ #endif decq J # j -- - jg .L01 + jg L(01) -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) -.L101: +L(101): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -1375,10 +1375,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 movddup -14 * SIZE(B), %xmm10 @@ -1400,10 +1400,10 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1412,10 +1412,10 @@ #endif andq $3, %rax BRANCH - jle .L105 + jle L(105) ALIGN_4 -.L104: +L(104): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 @@ -1425,10 +1425,10 @@ addq $4 * SIZE, BO addq $2 * SIZE, B decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq A, AO #else @@ -1445,10 +1445,10 @@ #endif testq $1, M - jle .L130 + jle L(130) ALIGN_4 -.L140: +L(140): #ifdef LN movq K, %rax salq $0 + ZBASE_SHIFT, %rax @@ -1482,9 +1482,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L142 + je L(142) -.L141: +L(141): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1524,9 +1524,9 @@ subq $ -8 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L141 + jne L(141) -.L142: +L(142): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1537,9 +1537,9 @@ andq $3, %rax # if (k & 1) BRANCH - jle .L144 + jle L(144) -.L143: +L(143): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -1553,10 +1553,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L143 + jg L(143) ALIGN_4 -.L144: +L(144): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -1714,13 +1714,13 @@ #endif ALIGN_4 -.L130: +L(130): movq M, I sarq $1, I # i = (m >> 2) - jle .L199 + jle L(199) ALIGN_4 -.L110: +L(110): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1755,9 +1755,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L112 + je L(112) -.L111: +L(111): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1835,10 +1835,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1848,9 +1848,9 @@ movapd POSINV, %xmm7 andq $3, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): movapd -16 * SIZE(AO), %xmm0 movapd -14 * SIZE(AO), %xmm1 @@ -1872,10 +1872,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2105,10 +2105,10 @@ #endif decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L199: +L(199): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2131,7 +2131,7 @@ ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/ztrsm_kernel_LN_2x2_penryn.S b/kernel/x86_64/ztrsm_kernel_LN_2x2_penryn.S index 3a691ca8cc..ad4cc0ad57 100644 --- a/kernel/x86_64/ztrsm_kernel_LN_2x2_penryn.S +++ b/kernel/x86_64/ztrsm_kernel_LN_2x2_penryn.S @@ -171,10 +171,10 @@ movq N, J sarq $1, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -211,7 +211,7 @@ testq $1, M BRANCH - jle .L20 + jle L(20) ALIGN_4 #ifdef LN @@ -250,10 +250,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_4 -.L22: +L(22): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -319,10 +319,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -331,10 +331,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -355,10 +355,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -556,14 +556,14 @@ #endif ALIGN_4 -.L20: +L(20): movq M, I sarq $1, I NOBRANCH - jle .L39 + jle L(39) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -619,10 +619,10 @@ #endif sarq $3, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 @@ -868,10 +868,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -880,10 +880,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 @@ -919,10 +919,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax subq $2, %rax @@ -1266,10 +1266,10 @@ decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1289,13 +1289,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(LT) || defined(RN) movq A, AO @@ -1332,7 +1332,7 @@ testq $1, M BRANCH - jle .L60 + jle L(60) ALIGN_4 #ifdef LN @@ -1366,10 +1366,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x4e, %xmm2, %xmm7 @@ -1412,10 +1412,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1424,10 +1424,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm7 @@ -1442,10 +1442,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -1588,14 +1588,14 @@ #endif ALIGN_4 -.L60: +L(60): movq M, I sarq $1, I # i = (m >> 2) NOBRANCH - jle .L79 + jle L(79) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1633,10 +1633,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_4 -.L52: +L(52): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -1711,10 +1711,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1723,10 +1723,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -1749,10 +1749,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1965,10 +1965,10 @@ #endif decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -1987,7 +1987,7 @@ #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_LN_2x2_sse2.S b/kernel/x86_64/ztrsm_kernel_LN_2x2_sse2.S index 542bd59474..df3bd43a63 100644 --- a/kernel/x86_64/ztrsm_kernel_LN_2x2_sse2.S +++ b/kernel/x86_64/ztrsm_kernel_LN_2x2_sse2.S @@ -319,10 +319,10 @@ movq N, J sarq $1, J # j = (n >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L01: +L(01): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -358,12 +358,12 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): PREFETCHNTA 56 * SIZE(B) movlpd 0 * SIZE(B), %xmm0 @@ -395,10 +395,10 @@ subq $-16 * SIZE, BO addq $ 8 * SIZE, B decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -407,10 +407,10 @@ #endif andq $3, %rax BRANCH - jle .L05 + jle L(05) ALIGN_4 -.L04: +L(04): movlpd 0 * SIZE(B), %xmm0 movlpd 1 * SIZE(B), %xmm1 movlpd 2 * SIZE(B), %xmm2 @@ -429,10 +429,10 @@ addq $ 8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L05: +L(05): #if defined(LT) || defined(RN) movq A, AO #else @@ -452,7 +452,7 @@ #endif testq $1, M - jle .L30 + jle L(30) #ifdef LN movq K, %rax @@ -487,9 +487,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L42 + je L(42) -.L41: +L(41): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 @@ -565,9 +565,9 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L41 + jne L(41) -.L42: +L(42): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -577,9 +577,9 @@ movapd POSINV, %xmm15 andq $3, %rax # if (k & 1) BRANCH - jle .L44 + jle L(44) -.L43: +L(43): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 @@ -602,10 +602,10 @@ addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -833,13 +833,13 @@ #endif ALIGN_4 -.L30: +L(30): movq M, I sarq $1, I # i = (m >> 2) - jle .L99 + jle L(99) ALIGN_4 -.L10: +L(10): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -897,8 +897,8 @@ #endif andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -916,7 +916,7 @@ KERNEL7(16 * 1) KERNEL8(16 * 1) cmpq $64 * 2, %rax - jle .L12 + jle L(12) KERNEL1(16 * 2) KERNEL2(16 * 2) KERNEL3(16 * 2) @@ -934,7 +934,7 @@ KERNEL7(16 * 3) KERNEL8(16 * 3) cmpq $64 * 4, %rax - jle .L12 + jle L(12) KERNEL1(16 * 4) KERNEL2(16 * 4) KERNEL3(16 * 4) @@ -952,7 +952,7 @@ KERNEL7(16 * 5) KERNEL8(16 * 5) cmpq $64 * 6, %rax - jle .L12 + jle L(12) KERNEL1(16 * 6) KERNEL2(16 * 6) KERNEL3(16 * 6) @@ -973,14 +973,14 @@ addq $16 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $64 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 4), BO # * 64 ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -990,10 +990,10 @@ movapd POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -1022,10 +1022,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1421,11 +1421,11 @@ #endif decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1447,13 +1447,13 @@ #endif decq J # j -- - jg .L01 + jg L(01) -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) -.L101: +L(101): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -1489,10 +1489,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movlpd 0 * SIZE(B), %xmm0 movlpd 1 * SIZE(B), %xmm1 movlpd 2 * SIZE(B), %xmm2 @@ -1522,10 +1522,10 @@ subq $-16 * SIZE, BO addq $ 8 * SIZE, B decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1534,10 +1534,10 @@ #endif andq $3, %rax BRANCH - jle .L105 + jle L(105) ALIGN_4 -.L104: +L(104): movlpd 0 * SIZE(B), %xmm0 movlpd 1 * SIZE(B), %xmm1 @@ -1549,10 +1549,10 @@ addq $4 * SIZE, BO addq $2 * SIZE, B decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq A, AO #else @@ -1569,10 +1569,10 @@ #endif testq $1, M - jle .L130 + jle L(130) ALIGN_4 -.L140: +L(140): #ifdef LN movq K, %rax salq $0 + ZBASE_SHIFT, %rax @@ -1606,9 +1606,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L142 + je L(142) -.L141: +L(141): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -1640,9 +1640,9 @@ addq $8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L141 + jne L(141) -.L142: +L(142): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -1656,9 +1656,9 @@ #endif andq $3, %rax # if (k & 1) BRANCH - jle .L144 + jle L(144) -.L143: +L(143): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -1669,10 +1669,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L143 + jg L(143) ALIGN_4 -.L144: +L(144): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1834,13 +1834,13 @@ #endif ALIGN_4 -.L130: +L(130): movq M, I sarq $1, I # i = (m >> 2) - jle .L199 + jle L(199) ALIGN_4 -.L110: +L(110): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1880,9 +1880,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L112 + je L(112) -.L111: +L(111): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -1942,10 +1942,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1955,9 +1955,9 @@ movapd POSINV, %xmm15 andq $3, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -1975,10 +1975,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2223,10 +2223,10 @@ #endif decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L199: +L(199): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2248,7 +2248,7 @@ #endif ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/ztrsm_kernel_LN_2x2_sse3.S b/kernel/x86_64/ztrsm_kernel_LN_2x2_sse3.S index 7547421efa..063d01cefd 100644 --- a/kernel/x86_64/ztrsm_kernel_LN_2x2_sse3.S +++ b/kernel/x86_64/ztrsm_kernel_LN_2x2_sse3.S @@ -408,10 +408,10 @@ movq N, J sarq $1, J # j = (n >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -443,7 +443,7 @@ #endif testq $1, M - jle .L30 + jle L(30) #ifdef LN movq K, %rax @@ -477,9 +477,9 @@ subq KK, %rax #endif sarq $3, %rax - je .L42 + je L(42) -.L41: +L(41): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -589,9 +589,9 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L41 + jne L(41) -.L42: +L(42): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -600,9 +600,9 @@ #endif andq $7, %rax # if (k & 1) BRANCH - jle .L44 + jle L(44) -.L43: +L(43): mulpd %xmm8, %xmm9 ADD1 %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -620,10 +620,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm3, %xmm3 @@ -863,13 +863,13 @@ #endif ALIGN_4 -.L30: +L(30): movq M, I sarq $1, I # i = (m >> 2) - jle .L99 + jle L(99) ALIGN_4 -.L10: +L(10): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -920,9 +920,9 @@ #endif andq $-8, %rax salq $4, %rax - je .L12 + je L(12) -.L1X: +L(1X): KERNEL1 (16 * 0) KERNEL2 (16 * 0) KERNEL3 (16 * 0) @@ -941,7 +941,7 @@ KERNEL16(16 * 0) cmpq $128 * 1, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 1) KERNEL2 (16 * 1) KERNEL3 (16 * 1) @@ -960,7 +960,7 @@ KERNEL16(16 * 1) cmpq $128 * 2, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 2) KERNEL2 (16 * 2) KERNEL3 (16 * 2) @@ -979,7 +979,7 @@ KERNEL16(16 * 2) cmpq $128 * 3, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 3) KERNEL2 (16 * 3) KERNEL3 (16 * 3) @@ -998,7 +998,7 @@ KERNEL16(16 * 3) cmpq $128 * 4, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 4) KERNEL2 (16 * 4) KERNEL3 (16 * 4) @@ -1017,7 +1017,7 @@ KERNEL16(16 * 4) cmpq $128 * 5, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 5) KERNEL2 (16 * 5) KERNEL3 (16 * 5) @@ -1036,7 +1036,7 @@ KERNEL16(16 * 5) cmpq $128 * 6, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 6) KERNEL2 (16 * 6) KERNEL3 (16 * 6) @@ -1055,7 +1055,7 @@ KERNEL16(16 * 6) cmpq $128 * 7, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 7) KERNEL2 (16 * 7) KERNEL3 (16 * 7) @@ -1076,14 +1076,14 @@ addq $32 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $128 * 8, %rax - jg .L1X + jg L(1X) -.L11: +L(11): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 2), BO # * 64 ALIGN_4 -.L12: +L(12): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1092,10 +1092,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L14 + je L(14) ALIGN_4 -.L13: +L(13): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 ADD1 %xmm9, %xmm0 @@ -1126,10 +1126,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm3, %xmm3 SHUFPD_1 %xmm5, %xmm5 @@ -1402,10 +1402,10 @@ addq %rax, AORIG #endif decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1423,13 +1423,13 @@ subq $2, KK #endif decq J # j -- - jg .L01 + jg L(01) -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) -.L101: +L(101): #if defined(LT) || defined(RN) movq A, AO #else @@ -1457,7 +1457,7 @@ #endif testq $1, M - jle .L130 + jle L(130) ALIGN_4 #ifdef LN @@ -1492,9 +1492,9 @@ subq KK, %rax #endif sarq $3, %rax - je .L142 + je L(142) -.L141: +L(141): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -1556,9 +1556,9 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L141 + jne L(141) -.L142: +L(142): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1567,9 +1567,9 @@ #endif andq $7, %rax # if (k & 1) BRANCH - jle .L144 + jle L(144) -.L143: +L(143): mulpd %xmm8, %xmm9 ADD1 %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -1581,10 +1581,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L143 + jg L(143) ALIGN_4 -.L144: +L(144): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -1735,13 +1735,13 @@ #endif ALIGN_4 -.L130: +L(130): movq M, I sarq $1, I # i = (m >> 2) - jle .L149 + jle L(149) ALIGN_4 -.L110: +L(110): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1780,9 +1780,9 @@ subq KK, %rax #endif sarq $3, %rax - je .L112 + je L(112) -.L111: +L(111): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -1901,10 +1901,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1913,9 +1913,9 @@ #endif andq $7, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 ADD1 %xmm9, %xmm0 @@ -1934,10 +1934,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm5, %xmm5 @@ -2152,10 +2152,10 @@ #endif decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L149: +L(149): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2174,7 +2174,7 @@ #endif ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_LN_2x4_nehalem.S b/kernel/x86_64/ztrsm_kernel_LN_2x4_nehalem.S index 5d931cea85..2e15200dfc 100644 --- a/kernel/x86_64/ztrsm_kernel_LN_2x4_nehalem.S +++ b/kernel/x86_64/ztrsm_kernel_LN_2x4_nehalem.S @@ -171,10 +171,10 @@ movq N, J sarq $2, J NOBRANCH - jle .L30 + jle L(30) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -211,7 +211,7 @@ testq $1, M BRANCH - jle .L20 + jle L(20) #ifdef LN movq K, %rax @@ -251,10 +251,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -324,10 +324,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -336,10 +336,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -361,10 +361,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -782,14 +782,14 @@ #endif ALIGN_4 -.L20: +L(20): movq M, I sarq $1, I NOBRANCH - jle .L29 + jle L(29) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -839,10 +839,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm12 @@ -969,10 +969,10 @@ subq $-16 * SIZE, AO subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -981,10 +981,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm1, %xmm12 movaps -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm13 @@ -1020,10 +1020,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1584,10 +1584,10 @@ decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L29: +L(29): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -1608,13 +1608,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L30: +L(30): testq $2, N BRANCH - jle .L50 + jle L(50) #if defined(LT) || defined(RN) movq A, AO @@ -1648,7 +1648,7 @@ testq $1, M BRANCH - jle .L40 + jle L(40) #ifdef LN movq K, %rax @@ -1688,10 +1688,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1733,10 +1733,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1745,10 +1745,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1763,10 +1763,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1980,14 +1980,14 @@ #endif ALIGN_4 -.L40: +L(40): movq M, I sarq $1, I NOBRANCH - jle .L49 + jle L(49) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -2027,10 +2027,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -2094,10 +2094,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2106,10 +2106,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): ADD1 %xmm1, %xmm8 movaps -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -2129,10 +2129,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2443,10 +2443,10 @@ decq I # i -- BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L49: +L(49): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -2466,10 +2466,10 @@ #endif ALIGN_4 -.L50: +L(50): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(LT) || defined(RN) movq A, AO @@ -2501,7 +2501,7 @@ testq $1, M BRANCH - jle .L60 + jle L(60) #ifdef LN movq K, %rax @@ -2536,10 +2536,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2581,10 +2581,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2593,10 +2593,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2611,10 +2611,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -2730,14 +2730,14 @@ #endif ALIGN_4 -.L60: +L(60): movq M, I sarq $1, I NOBRANCH - jle .L69 + jle L(69) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -2776,10 +2776,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -2819,10 +2819,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2831,10 +2831,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): ADD1 %xmm1, %xmm8 movddup -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -2848,10 +2848,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3064,10 +3064,10 @@ decq I # i -- BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L69: +L(69): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -3087,7 +3087,7 @@ #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_LN_4x2_sse.S b/kernel/x86_64/ztrsm_kernel_LN_4x2_sse.S index 02b5098a23..45478766bb 100644 --- a/kernel/x86_64/ztrsm_kernel_LN_4x2_sse.S +++ b/kernel/x86_64/ztrsm_kernel_LN_4x2_sse.S @@ -332,10 +332,10 @@ movq N, J sarq $1, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -370,10 +370,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 @@ -423,10 +423,10 @@ addq $16 * SIZE, B addq $64 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -435,10 +435,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movaps 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -454,10 +454,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -477,7 +477,7 @@ #endif testq $1, M - je .L20 + je L(20) #ifdef LN movq K, %rax @@ -522,10 +522,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -643,10 +643,10 @@ addq $128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -656,10 +656,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -678,10 +678,10 @@ addq $ 2 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm3, %xmm3 @@ -956,9 +956,9 @@ #endif ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) #ifdef LN movq K, %rax @@ -1003,10 +1003,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -1121,10 +1121,10 @@ addq $128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1134,10 +1134,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1154,10 +1154,10 @@ addq $ 4 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm3, %xmm3 @@ -1516,13 +1516,13 @@ #endif ALIGN_4 -.L30: +L(30): movq M, I sarq $2, I # i = (m >> 2) - jle .L39 + jle L(39) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + ZBASE_SHIFT, %rax @@ -1573,8 +1573,8 @@ #endif andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -1595,14 +1595,14 @@ addq $32 * 2 * SIZE, AO addq $64 * 2 * SIZE, BO subq $64 * 2, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO leaq (BO, %rax, 4), BO ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1612,10 +1612,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1644,10 +1644,10 @@ addq $ 8 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm3, %xmm3 shufps $0xb1, %xmm5, %xmm5 @@ -2365,10 +2365,10 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2 * COMPSIZE), B @@ -2390,12 +2390,12 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $1, N - je .L999 + je L(999) ALIGN_4 #ifdef LN @@ -2432,10 +2432,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 @@ -2462,10 +2462,10 @@ addq $ 8 * SIZE, B addq $32 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2474,10 +2474,10 @@ #endif andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movsd 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -2489,10 +2489,10 @@ addq $2 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): #if defined(LT) || defined(RN) movq A, AO #else @@ -2510,7 +2510,7 @@ #endif testq $1, M - je .L60 + je L(60) #ifdef LN movq K, %rax @@ -2555,10 +2555,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -2631,10 +2631,10 @@ addq $64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2644,10 +2644,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -2659,10 +2659,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -2806,9 +2806,9 @@ #endif ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) #ifdef LN movq K, %rax @@ -2851,10 +2851,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2918,10 +2918,10 @@ addq $64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2931,10 +2931,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -2945,10 +2945,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -3196,13 +3196,13 @@ #endif ALIGN_4 -.L70: +L(70): movq M, I sarq $2, I # i = (m >> 2) - jle .L79 + jle L(79) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + ZBASE_SHIFT, %rax @@ -3253,10 +3253,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3378,10 +3378,10 @@ decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3391,10 +3391,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -3411,10 +3411,10 @@ addq $ 8 * SIZE, AO # aoffset += 4 addq $ 8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm5, %xmm5 @@ -3957,10 +3957,10 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, COMPSIZE), B @@ -3982,7 +3982,7 @@ #endif ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/ztrsm_kernel_LT_1x4_nehalem.S b/kernel/x86_64/ztrsm_kernel_LT_1x4_nehalem.S index 874d34db34..6756395456 100644 --- a/kernel/x86_64/ztrsm_kernel_LT_1x4_nehalem.S +++ b/kernel/x86_64/ztrsm_kernel_LT_1x4_nehalem.S @@ -168,15 +168,15 @@ #endif testq M, M - jle .L999 + jle L(999) movq N, J sarq $2, J NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -214,7 +214,7 @@ movq M, I ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -264,10 +264,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm12 @@ -391,10 +391,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -403,10 +403,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm1, %xmm12 movaps -16 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm13 @@ -442,10 +442,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -819,7 +819,7 @@ decq I BRANCH - jg .L11 + jg L(11) #ifdef LN movq K, %rax @@ -841,13 +841,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L20: +L(20): testq $2, N BRANCH - jle .L30 + jle L(30) #if defined(LT) || defined(RN) movq A, AO @@ -882,7 +882,7 @@ movq M, I ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -922,10 +922,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -997,10 +997,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1009,10 +1009,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): ADD1 %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -1034,10 +1034,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1239,7 +1239,7 @@ decq I BRANCH - jg .L21 + jg L(21) #ifdef LN movq K, %rax @@ -1260,10 +1260,10 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(LT) || defined(RN) movq A, AO @@ -1295,7 +1295,7 @@ movq M, I ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -1332,10 +1332,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -1375,13 +1375,13 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 ALIGN_3 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1390,10 +1390,10 @@ #endif andq $3, %rax BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): ADD1 %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -1407,10 +1407,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -1536,7 +1536,7 @@ #endif decq I BRANCH - jg .L31 + jg L(31) #ifdef LN movq K, %rax @@ -1557,7 +1557,7 @@ #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_LT_2x1_atom.S b/kernel/x86_64/ztrsm_kernel_LT_2x1_atom.S index 92dc6366b9..90f80829d1 100644 --- a/kernel/x86_64/ztrsm_kernel_LT_2x1_atom.S +++ b/kernel/x86_64/ztrsm_kernel_LT_2x1_atom.S @@ -177,10 +177,10 @@ movq N, J testq N, N - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -214,10 +214,10 @@ movq M, I sarq $1, I - jle .L20 + jle L(20) ALIGN_4 -.L10: +L(10): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -265,10 +265,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L15 + je L(15) ALIGN_4 -.L12: +L(12): ADDSD2 %xmm2, %xmm13 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movaps %xmm0, %xmm2 @@ -411,10 +411,10 @@ mulsd %xmm3, %xmm6 movsd 1 * SIZE(BO), %xmm3 - jne .L12 + jne L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -424,10 +424,10 @@ andq $3, %rax BRANCH BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): ADDSD2 %xmm2, %xmm13 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -466,10 +466,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): ADDSD2 %xmm2, %xmm13 ADDSD3 %xmm7, %xmm14 ADDSD4 %xmm6, %xmm15 @@ -672,12 +672,12 @@ addq %rax, AORIG #endif decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L20: +L(20): testq $1, M - jle .L99 + jle L(99) #ifdef LN movq K, %rax @@ -717,10 +717,10 @@ subq KK, %rax #endif sarq $2, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADDSD2 %xmm2, %xmm9 movaps %xmm0, %xmm2 @@ -798,10 +798,10 @@ addq $8 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -811,10 +811,10 @@ andq $3, %rax BRANCH BRANCH - je .L29 + je L(29) ALIGN_4 -.L26: +L(26): ADDSD2 %xmm2, %xmm9 movaps %xmm0, %xmm2 mulsd %xmm1, %xmm0 @@ -837,10 +837,10 @@ addq $2 * SIZE, BO decq %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L29: +L(29): ADDSD2 %xmm2, %xmm9 ADDSD4 %xmm6, %xmm11 @@ -945,7 +945,7 @@ #endif ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -963,10 +963,10 @@ subq $1, KK #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_LT_2x2_core2.S b/kernel/x86_64/ztrsm_kernel_LT_2x2_core2.S index 6cf850609d..87c4f531ee 100644 --- a/kernel/x86_64/ztrsm_kernel_LT_2x2_core2.S +++ b/kernel/x86_64/ztrsm_kernel_LT_2x2_core2.S @@ -196,10 +196,10 @@ movq N, J sarq $1, J # j = (n >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L01: +L(01): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -234,12 +234,12 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): prefetcht0 (PREFETCH_R + 0) * SIZE(B) movddup -16 * SIZE(B), %xmm8 @@ -268,10 +268,10 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -280,10 +280,10 @@ #endif andq $3, %rax BRANCH - jle .L05 + jle L(05) ALIGN_4 -.L04: +L(04): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 movddup -14 * SIZE(B), %xmm10 @@ -298,10 +298,10 @@ addq $ 8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L05: +L(05): #if defined(LT) || defined(RN) movq A, AO #else @@ -322,10 +322,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L30 + jle L(30) ALIGN_4 -.L10: +L(10): leaq (PREFETCH_R + 0) * SIZE(B), BB #ifdef LN @@ -387,10 +387,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -511,10 +511,10 @@ subq $1, %rax BRANCH BRANCH - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -526,10 +526,10 @@ andq $3, %rax BRANCH BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): ADD1 %xmm2, %xmm10 ADD1 %xmm3, %xmm14 ADD2 %xmm4, %xmm11 @@ -566,10 +566,10 @@ addq $8 * SIZE, BO subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): ADD1 %xmm2, %xmm10 ADD1 %xmm3, %xmm14 ADD2 %xmm4, %xmm11 @@ -947,12 +947,12 @@ #endif decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L30: +L(30): testq $1, M - jle .L99 + jle L(99) #ifdef LN movq K, %rax @@ -987,9 +987,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L42 + je L(42) -.L41: +L(41): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1059,9 +1059,9 @@ subq $ -8 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L41 + jne L(41) -.L42: +L(42): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1072,9 +1072,9 @@ andq $3, %rax # if (k & 1) BRANCH - jle .L44 + jle L(44) -.L43: +L(43): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -1094,10 +1094,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1312,7 +1312,7 @@ #endif ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1334,13 +1334,13 @@ #endif decq J # j -- - jg .L01 + jg L(01) -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) -.L101: +L(101): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -1375,10 +1375,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 movddup -14 * SIZE(B), %xmm10 @@ -1400,10 +1400,10 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1412,10 +1412,10 @@ #endif andq $3, %rax BRANCH - jle .L105 + jle L(105) ALIGN_4 -.L104: +L(104): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 @@ -1425,10 +1425,10 @@ addq $4 * SIZE, BO addq $2 * SIZE, B decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq A, AO #else @@ -1446,10 +1446,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L130 + jle L(130) ALIGN_4 -.L110: +L(110): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1484,9 +1484,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L112 + je L(112) -.L111: +L(111): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1564,10 +1564,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1577,9 +1577,9 @@ movapd POSINV, %xmm7 andq $3, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): movapd -16 * SIZE(AO), %xmm0 movapd -14 * SIZE(AO), %xmm1 @@ -1601,10 +1601,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1834,15 +1834,15 @@ #endif decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L130: +L(130): testq $1, M - jle .L199 + jle L(199) ALIGN_4 -.L140: +L(140): #ifdef LN movq K, %rax salq $0 + ZBASE_SHIFT, %rax @@ -1876,9 +1876,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L142 + je L(142) -.L141: +L(141): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1918,9 +1918,9 @@ subq $ -8 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L141 + jne L(141) -.L142: +L(142): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1931,9 +1931,9 @@ andq $3, %rax # if (k & 1) BRANCH - jle .L144 + jle L(144) -.L143: +L(143): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -1947,10 +1947,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L143 + jg L(143) ALIGN_4 -.L144: +L(144): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -2108,7 +2108,7 @@ #endif ALIGN_4 -.L199: +L(199): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2131,7 +2131,7 @@ ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/ztrsm_kernel_LT_2x2_penryn.S b/kernel/x86_64/ztrsm_kernel_LT_2x2_penryn.S index 0078117587..4277220c73 100644 --- a/kernel/x86_64/ztrsm_kernel_LT_2x2_penryn.S +++ b/kernel/x86_64/ztrsm_kernel_LT_2x2_penryn.S @@ -171,10 +171,10 @@ movq N, J sarq $1, J NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -212,10 +212,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -271,10 +271,10 @@ #endif sarq $3, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 @@ -520,10 +520,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -532,10 +532,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 @@ -571,10 +571,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax subq $2, %rax @@ -918,13 +918,13 @@ decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M BRANCH - jle .L39 + jle L(39) ALIGN_4 #ifdef LN @@ -963,10 +963,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_4 -.L22: +L(22): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -1032,10 +1032,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1044,10 +1044,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -1068,10 +1068,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1269,7 +1269,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1289,13 +1289,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(LT) || defined(RN) movq A, AO @@ -1333,10 +1333,10 @@ movq M, I sarq $1, I # i = (m >> 2) NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1374,10 +1374,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_4 -.L52: +L(52): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -1452,10 +1452,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1464,10 +1464,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -1490,10 +1490,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1706,13 +1706,13 @@ #endif decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L79 + jle L(79) ALIGN_4 #ifdef LN @@ -1746,10 +1746,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x4e, %xmm2, %xmm7 @@ -1792,10 +1792,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1804,10 +1804,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm7 @@ -1822,10 +1822,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -1968,7 +1968,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -1987,7 +1987,7 @@ #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_LT_2x2_sse2.S b/kernel/x86_64/ztrsm_kernel_LT_2x2_sse2.S index d90bfd8a55..0dd1fd3f51 100644 --- a/kernel/x86_64/ztrsm_kernel_LT_2x2_sse2.S +++ b/kernel/x86_64/ztrsm_kernel_LT_2x2_sse2.S @@ -320,10 +320,10 @@ movq N, J sarq $1, J # j = (n >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L01: +L(01): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -358,12 +358,12 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): PREFETCHNTA 56 * SIZE(B) movlpd 0 * SIZE(B), %xmm0 @@ -395,10 +395,10 @@ subq $-16 * SIZE, BO addq $ 8 * SIZE, B decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -407,10 +407,10 @@ #endif andq $3, %rax BRANCH - jle .L05 + jle L(05) ALIGN_4 -.L04: +L(04): movlpd 0 * SIZE(B), %xmm0 movlpd 1 * SIZE(B), %xmm1 movlpd 2 * SIZE(B), %xmm2 @@ -429,10 +429,10 @@ addq $ 8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L05: +L(05): #if defined(LT) || defined(RN) movq A, AO #else @@ -453,10 +453,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L30 + jle L(30) ALIGN_4 -.L10: +L(10): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -507,8 +507,8 @@ #endif andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -526,7 +526,7 @@ KERNEL7(16 * 1) KERNEL8(16 * 1) cmpq $64 * 2, %rax - jle .L12 + jle L(12) KERNEL1(16 * 2) KERNEL2(16 * 2) KERNEL3(16 * 2) @@ -544,7 +544,7 @@ KERNEL7(16 * 3) KERNEL8(16 * 3) cmpq $64 * 4, %rax - jle .L12 + jle L(12) KERNEL1(16 * 4) KERNEL2(16 * 4) KERNEL3(16 * 4) @@ -562,7 +562,7 @@ KERNEL7(16 * 5) KERNEL8(16 * 5) cmpq $64 * 6, %rax - jle .L12 + jle L(12) KERNEL1(16 * 6) KERNEL2(16 * 6) KERNEL3(16 * 6) @@ -583,14 +583,14 @@ addq $16 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $64 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 4), BO # * 64 ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -600,10 +600,10 @@ movapd POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -632,10 +632,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1031,12 +1031,12 @@ #endif decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L30: +L(30): testq $1, M - jle .L99 + jle L(99) #ifdef LN movq K, %rax @@ -1071,9 +1071,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L42 + je L(42) -.L41: +L(41): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 @@ -1149,9 +1149,9 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L41 + jne L(41) -.L42: +L(42): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1161,9 +1161,9 @@ movapd POSINV, %xmm15 andq $3, %rax # if (k & 1) BRANCH - jle .L44 + jle L(44) -.L43: +L(43): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 @@ -1186,10 +1186,10 @@ addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1417,7 +1417,7 @@ #endif ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1439,13 +1439,13 @@ #endif decq J # j -- - jg .L01 + jg L(01) -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) -.L101: +L(101): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -1481,10 +1481,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movlpd 0 * SIZE(B), %xmm0 movlpd 1 * SIZE(B), %xmm1 movlpd 2 * SIZE(B), %xmm2 @@ -1514,10 +1514,10 @@ subq $-16 * SIZE, BO addq $ 8 * SIZE, B decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1526,10 +1526,10 @@ #endif andq $3, %rax BRANCH - jle .L105 + jle L(105) ALIGN_4 -.L104: +L(104): movlpd 0 * SIZE(B), %xmm0 movlpd 1 * SIZE(B), %xmm1 @@ -1541,10 +1541,10 @@ addq $4 * SIZE, BO addq $2 * SIZE, B decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq A, AO #else @@ -1562,10 +1562,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L130 + jle L(130) ALIGN_4 -.L110: +L(110): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1600,9 +1600,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L112 + je L(112) -.L111: +L(111): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -1662,10 +1662,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1675,9 +1675,9 @@ movapd POSINV, %xmm15 andq $3, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -1695,10 +1695,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1943,15 +1943,15 @@ #endif decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L130: +L(130): testq $1, M - jle .L199 + jle L(199) ALIGN_4 -.L140: +L(140): #ifdef LN movq K, %rax salq $0 + ZBASE_SHIFT, %rax @@ -1985,9 +1985,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L142 + je L(142) -.L141: +L(141): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -2019,9 +2019,9 @@ addq $8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L141 + jne L(141) -.L142: +L(142): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -2035,9 +2035,9 @@ #endif andq $3, %rax # if (k & 1) BRANCH - jle .L144 + jle L(144) -.L143: +L(143): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -2048,10 +2048,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L143 + jg L(143) ALIGN_4 -.L144: +L(144): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2213,7 +2213,7 @@ #endif ALIGN_4 -.L199: +L(199): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2236,7 +2236,7 @@ ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/ztrsm_kernel_LT_2x2_sse3.S b/kernel/x86_64/ztrsm_kernel_LT_2x2_sse3.S index c52b058bf6..1ad609874b 100644 --- a/kernel/x86_64/ztrsm_kernel_LT_2x2_sse3.S +++ b/kernel/x86_64/ztrsm_kernel_LT_2x2_sse3.S @@ -408,10 +408,10 @@ movq N, J sarq $1, J # j = (n >> 2) - jle .L100 + jle L(100) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -444,10 +444,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L30 + jle L(30) ALIGN_4 -.L10: +L(10): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -493,9 +493,9 @@ #endif andq $-8, %rax salq $4, %rax - je .L12 + je L(12) -.L1X: +L(1X): KERNEL1 (16 * 0) KERNEL2 (16 * 0) KERNEL3 (16 * 0) @@ -514,7 +514,7 @@ KERNEL16(16 * 0) cmpq $128 * 1, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 1) KERNEL2 (16 * 1) KERNEL3 (16 * 1) @@ -533,7 +533,7 @@ KERNEL16(16 * 1) cmpq $128 * 2, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 2) KERNEL2 (16 * 2) KERNEL3 (16 * 2) @@ -552,7 +552,7 @@ KERNEL16(16 * 2) cmpq $128 * 3, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 3) KERNEL2 (16 * 3) KERNEL3 (16 * 3) @@ -571,7 +571,7 @@ KERNEL16(16 * 3) cmpq $128 * 4, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 4) KERNEL2 (16 * 4) KERNEL3 (16 * 4) @@ -590,7 +590,7 @@ KERNEL16(16 * 4) cmpq $128 * 5, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 5) KERNEL2 (16 * 5) KERNEL3 (16 * 5) @@ -609,7 +609,7 @@ KERNEL16(16 * 5) cmpq $128 * 6, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 6) KERNEL2 (16 * 6) KERNEL3 (16 * 6) @@ -628,7 +628,7 @@ KERNEL16(16 * 6) cmpq $128 * 7, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 7) KERNEL2 (16 * 7) KERNEL3 (16 * 7) @@ -649,14 +649,14 @@ addq $32 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $128 * 8, %rax - jg .L1X + jg L(1X) -.L11: +L(11): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 2), BO # * 64 ALIGN_4 -.L12: +L(12): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -665,10 +665,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L14 + je L(14) ALIGN_4 -.L13: +L(13): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 ADD1 %xmm9, %xmm0 @@ -699,10 +699,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm3, %xmm3 SHUFPD_1 %xmm5, %xmm5 @@ -975,12 +975,12 @@ addq %rax, AORIG #endif decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L30: +L(30): testq $1, M - jle .L99 + jle L(99) #ifdef LN movq K, %rax @@ -1014,9 +1014,9 @@ subq KK, %rax #endif sarq $3, %rax - je .L42 + je L(42) -.L41: +L(41): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -1126,9 +1126,9 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L41 + jne L(41) -.L42: +L(42): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1137,9 +1137,9 @@ #endif andq $7, %rax # if (k & 1) BRANCH - jle .L44 + jle L(44) -.L43: +L(43): mulpd %xmm8, %xmm9 ADD1 %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -1157,10 +1157,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm3, %xmm3 @@ -1400,7 +1400,7 @@ #endif ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1418,13 +1418,13 @@ subq $2, KK #endif decq J # j -- - jg .L01 + jg L(01) -.L100: +L(100): testq $1, N - jle .L999 + jle L(999) -.L101: +L(101): #if defined(LT) || defined(RN) movq A, AO #else @@ -1453,10 +1453,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L130 + jle L(130) ALIGN_4 -.L110: +L(110): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1491,9 +1491,9 @@ subq KK, %rax #endif sarq $3, %rax - je .L112 + je L(112) -.L111: +L(111): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -1612,10 +1612,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1624,9 +1624,9 @@ #endif andq $7, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 ADD1 %xmm9, %xmm0 @@ -1645,10 +1645,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm5, %xmm5 @@ -1863,12 +1863,12 @@ #endif decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L130: +L(130): testq $1, M - jle .L149 + jle L(149) ALIGN_4 #ifdef LN @@ -1903,9 +1903,9 @@ subq KK, %rax #endif sarq $3, %rax - je .L142 + je L(142) -.L141: +L(141): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -1967,9 +1967,9 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L141 + jne L(141) -.L142: +L(142): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1978,9 +1978,9 @@ #endif andq $7, %rax # if (k & 1) BRANCH - jle .L144 + jle L(144) -.L143: +L(143): mulpd %xmm8, %xmm9 ADD1 %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -1992,10 +1992,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L143 + jg L(143) ALIGN_4 -.L144: +L(144): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -2146,7 +2146,7 @@ #endif ALIGN_4 -.L149: +L(149): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -2165,7 +2165,7 @@ #endif ALIGN_3 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_LT_2x4_nehalem.S b/kernel/x86_64/ztrsm_kernel_LT_2x4_nehalem.S index 0d6531ac85..49aa90673a 100644 --- a/kernel/x86_64/ztrsm_kernel_LT_2x4_nehalem.S +++ b/kernel/x86_64/ztrsm_kernel_LT_2x4_nehalem.S @@ -171,10 +171,10 @@ movq N, J sarq $2, J NOBRANCH - jle .L30 + jle L(30) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -212,10 +212,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -265,10 +265,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm12 @@ -395,10 +395,10 @@ subq $-16 * SIZE, AO subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -407,10 +407,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm1, %xmm12 movaps -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm13 @@ -446,10 +446,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1010,13 +1010,13 @@ decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M BRANCH - jle .L29 + jle L(29) #ifdef LN movq K, %rax @@ -1056,10 +1056,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1129,10 +1129,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1141,10 +1141,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1166,10 +1166,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1587,7 +1587,7 @@ #endif ALIGN_4 -.L29: +L(29): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -1608,13 +1608,13 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L30: +L(30): testq $2, N BRANCH - jle .L50 + jle L(50) #if defined(LT) || defined(RN) movq A, AO @@ -1649,10 +1649,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1692,10 +1692,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -1759,10 +1759,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1771,10 +1771,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): ADD1 %xmm1, %xmm8 movaps -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -1794,10 +1794,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2108,13 +2108,13 @@ decq I # i -- BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L40: +L(40): testq $1, M BRANCH - jle .L49 + jle L(49) #ifdef LN movq K, %rax @@ -2154,10 +2154,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2199,10 +2199,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2211,10 +2211,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2229,10 +2229,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2446,7 +2446,7 @@ #endif ALIGN_4 -.L49: +L(49): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -2466,10 +2466,10 @@ #endif ALIGN_4 -.L50: +L(50): testq $1, N BRANCH - jle .L999 + jle L(999) #if defined(LT) || defined(RN) movq A, AO @@ -2502,10 +2502,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -2544,10 +2544,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -2587,10 +2587,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2599,10 +2599,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): ADD1 %xmm1, %xmm8 movddup -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -2616,10 +2616,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2832,13 +2832,13 @@ decq I # i -- BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L69 + jle L(69) #ifdef LN movq K, %rax @@ -2873,10 +2873,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2918,10 +2918,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2930,10 +2930,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2948,10 +2948,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -3067,7 +3067,7 @@ #endif ALIGN_4 -.L69: +L(69): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -3087,7 +3087,7 @@ #endif ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_LT_4x2_sse.S b/kernel/x86_64/ztrsm_kernel_LT_4x2_sse.S index 0c3a052a17..15850c5542 100644 --- a/kernel/x86_64/ztrsm_kernel_LT_4x2_sse.S +++ b/kernel/x86_64/ztrsm_kernel_LT_4x2_sse.S @@ -332,10 +332,10 @@ movq N, J sarq $1, J # j = (n >> 2) - jle .L40 + jle L(40) ALIGN_4 -.L01: +L(01): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -370,10 +370,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 @@ -423,10 +423,10 @@ addq $16 * SIZE, B addq $64 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -435,10 +435,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movaps 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -454,10 +454,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -478,10 +478,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + ZBASE_SHIFT, %rax @@ -532,8 +532,8 @@ #endif andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -554,14 +554,14 @@ addq $32 * 2 * SIZE, AO addq $64 * 2 * SIZE, BO subq $64 * 2, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO leaq (BO, %rax, 4), BO ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -571,10 +571,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -603,10 +603,10 @@ addq $ 8 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm3, %xmm3 shufps $0xb1, %xmm5, %xmm5 @@ -1324,12 +1324,12 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) #ifdef LN movq K, %rax @@ -1374,10 +1374,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -1492,10 +1492,10 @@ addq $128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1505,10 +1505,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1525,10 +1525,10 @@ addq $ 4 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm3, %xmm3 @@ -1887,9 +1887,9 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #ifdef LN movq K, %rax @@ -1934,10 +1934,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -2055,10 +2055,10 @@ addq $128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2068,10 +2068,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -2090,10 +2090,10 @@ addq $ 2 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm3, %xmm3 @@ -2368,7 +2368,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2 * COMPSIZE), B @@ -2390,12 +2390,12 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L40: +L(40): testq $1, N - je .L999 + je L(999) ALIGN_4 #ifdef LN @@ -2432,10 +2432,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 @@ -2462,10 +2462,10 @@ addq $ 8 * SIZE, B addq $32 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2474,10 +2474,10 @@ #endif andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movsd 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -2489,10 +2489,10 @@ addq $2 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): #if defined(LT) || defined(RN) movq A, AO #else @@ -2511,10 +2511,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + ZBASE_SHIFT, %rax @@ -2565,10 +2565,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -2690,10 +2690,10 @@ decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2703,10 +2703,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -2723,10 +2723,10 @@ addq $ 8 * SIZE, AO # aoffset += 4 addq $ 8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm5, %xmm5 @@ -3269,12 +3269,12 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) #ifdef LN movq K, %rax @@ -3317,10 +3317,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -3384,10 +3384,10 @@ addq $64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3397,10 +3397,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -3411,10 +3411,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -3662,9 +3662,9 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) #ifdef LN movq K, %rax @@ -3709,10 +3709,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -3785,10 +3785,10 @@ addq $64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3798,10 +3798,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -3813,10 +3813,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -3960,7 +3960,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, COMPSIZE), B @@ -3982,7 +3982,7 @@ #endif ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/ztrsm_kernel_RT_1x4_nehalem.S b/kernel/x86_64/ztrsm_kernel_RT_1x4_nehalem.S index a65c2718ba..465a02a955 100644 --- a/kernel/x86_64/ztrsm_kernel_RT_1x4_nehalem.S +++ b/kernel/x86_64/ztrsm_kernel_RT_1x4_nehalem.S @@ -168,11 +168,11 @@ #endif testq M, M - jle .L999 + jle L(999) testq $1, N BRANCH - jle .L20 + jle L(20) #if defined(LT) || defined(RN) movq A, AO @@ -204,7 +204,7 @@ movq M, I ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -241,10 +241,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -284,13 +284,13 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 ALIGN_3 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -299,10 +299,10 @@ #endif andq $3, %rax BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): ADD1 %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -316,10 +316,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -445,7 +445,7 @@ #endif decq I BRANCH - jg .L31 + jg L(31) #ifdef LN movq K, %rax @@ -466,10 +466,10 @@ #endif ALIGN_4 -.L20: +L(20): testq $2, N BRANCH - jle .L30 + jle L(30) #if defined(LT) || defined(RN) movq A, AO @@ -504,7 +504,7 @@ movq M, I ALIGN_4 -.L21: +L(21): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -544,10 +544,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -619,10 +619,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -631,10 +631,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): ADD1 %xmm1, %xmm8 movaps -16 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -656,10 +656,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -861,7 +861,7 @@ decq I BRANCH - jg .L21 + jg L(21) #ifdef LN movq K, %rax @@ -882,14 +882,14 @@ #endif ALIGN_4 -.L30: +L(30): movq N, J sarq $2, J NOBRANCH - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -927,7 +927,7 @@ movq M, I ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -977,10 +977,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm12 @@ -1104,10 +1104,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1116,10 +1116,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm1, %xmm12 movaps -16 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm13 @@ -1155,10 +1155,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1532,7 +1532,7 @@ decq I BRANCH - jg .L11 + jg L(11) #ifdef LN movq K, %rax @@ -1554,10 +1554,10 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_RT_2x2_core2.S b/kernel/x86_64/ztrsm_kernel_RT_2x2_core2.S index 0702b00887..ce09e346c4 100644 --- a/kernel/x86_64/ztrsm_kernel_RT_2x2_core2.S +++ b/kernel/x86_64/ztrsm_kernel_RT_2x2_core2.S @@ -195,9 +195,9 @@ #endif testq $1, N - jle .L100 + jle L(100) -.L101: +L(101): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -232,10 +232,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 movddup -14 * SIZE(B), %xmm10 @@ -257,10 +257,10 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -269,10 +269,10 @@ #endif andq $3, %rax BRANCH - jle .L105 + jle L(105) ALIGN_4 -.L104: +L(104): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 @@ -282,10 +282,10 @@ addq $4 * SIZE, BO addq $2 * SIZE, B decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq A, AO #else @@ -303,10 +303,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L130 + jle L(130) ALIGN_4 -.L110: +L(110): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -341,9 +341,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L112 + je L(112) -.L111: +L(111): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -421,10 +421,10 @@ subq $-16 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -434,9 +434,9 @@ movapd POSINV, %xmm7 andq $3, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): movapd -16 * SIZE(AO), %xmm0 movapd -14 * SIZE(AO), %xmm1 @@ -458,10 +458,10 @@ addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -691,15 +691,15 @@ #endif decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L130: +L(130): testq $1, M - jle .L199 + jle L(199) ALIGN_4 -.L140: +L(140): #ifdef LN movq K, %rax salq $0 + ZBASE_SHIFT, %rax @@ -733,9 +733,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L142 + je L(142) -.L141: +L(141): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -775,9 +775,9 @@ subq $ -8 * SIZE, AO subq $-16 * SIZE, BO subq $1, %rax - jne .L141 + jne L(141) -.L142: +L(142): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -788,9 +788,9 @@ andq $3, %rax # if (k & 1) BRANCH - jle .L144 + jle L(144) -.L143: +L(143): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -804,10 +804,10 @@ addq $2 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax - jg .L143 + jg L(143) ALIGN_4 -.L144: +L(144): addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 @@ -965,7 +965,7 @@ #endif ALIGN_4 -.L199: +L(199): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -987,13 +987,13 @@ #endif ALIGN_4 -.L100: +L(100): movq N, J sarq $1, J # j = (n >> 2) - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -1028,12 +1028,12 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): prefetcht0 (PREFETCH_R + 0) * SIZE(B) movddup -16 * SIZE(B), %xmm8 @@ -1062,10 +1062,10 @@ addq $ 8 * SIZE, B subq $-16 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1074,10 +1074,10 @@ #endif andq $3, %rax BRANCH - jle .L05 + jle L(05) ALIGN_4 -.L04: +L(04): movddup -16 * SIZE(B), %xmm8 movddup -15 * SIZE(B), %xmm9 movddup -14 * SIZE(B), %xmm10 @@ -1092,10 +1092,10 @@ addq $ 8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L05: +L(05): #if defined(LT) || defined(RN) movq A, AO #else @@ -1116,10 +1116,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L30 + jle L(30) ALIGN_4 -.L10: +L(10): leaq (PREFETCH_R + 0) * SIZE(B), BB #ifdef LN @@ -1181,10 +1181,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_4 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1305,10 +1305,10 @@ subq $1, %rax BRANCH BRANCH - jg .L12 + jg L(12) ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1320,10 +1320,10 @@ andq $3, %rax BRANCH BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): ADD1 %xmm2, %xmm10 ADD1 %xmm3, %xmm14 ADD2 %xmm4, %xmm11 @@ -1360,10 +1360,10 @@ addq $8 * SIZE, BO subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): ADD1 %xmm2, %xmm10 ADD1 %xmm3, %xmm14 ADD2 %xmm4, %xmm11 @@ -1741,12 +1741,12 @@ #endif decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L30: +L(30): testq $1, M - jle .L99 + jle L(99) #ifdef LN movq K, %rax @@ -1781,9 +1781,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L42 + je L(42) -.L41: +L(41): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) movapd -16 * SIZE(AO), %xmm0 @@ -1853,9 +1853,9 @@ subq $ -8 * SIZE, AO subq $-32 * SIZE, BO subq $1, %rax - jne .L41 + jne L(41) -.L42: +L(42): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1866,9 +1866,9 @@ andq $3, %rax # if (k & 1) BRANCH - jle .L44 + jle L(44) -.L43: +L(43): movapd -16 * SIZE(AO), %xmm0 movapd -16 * SIZE(BO), %xmm2 movapd -14 * SIZE(BO), %xmm3 @@ -1888,10 +1888,10 @@ addq $2 * SIZE, AO addq $8 * SIZE, BO subq $1, %rax - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2106,7 +2106,7 @@ #endif ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -2128,10 +2128,10 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq %r15, %rsp movq 0(%rsp), %rbx diff --git a/kernel/x86_64/ztrsm_kernel_RT_2x2_penryn.S b/kernel/x86_64/ztrsm_kernel_RT_2x2_penryn.S index 7770f5d9cc..1b0db5bca6 100644 --- a/kernel/x86_64/ztrsm_kernel_RT_2x2_penryn.S +++ b/kernel/x86_64/ztrsm_kernel_RT_2x2_penryn.S @@ -170,7 +170,7 @@ testq $1, N BRANCH - jle .L40 + jle L(40) #if defined(LT) || defined(RN) movq A, AO @@ -204,10 +204,10 @@ movq M, I sarq $1, I # i = (m >> 2) NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -242,10 +242,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_4 -.L52: +L(52): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -320,10 +320,10 @@ subq $ -8 * SIZE, BO subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -332,10 +332,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 @@ -358,10 +358,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -574,13 +574,13 @@ #endif decq I BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L79 + jle L(79) ALIGN_4 #ifdef LN @@ -614,10 +614,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_4 -.L62: +L(62): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) pshufd $0x4e, %xmm2, %xmm7 @@ -660,10 +660,10 @@ subq $-8 * SIZE, BO subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -672,10 +672,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm0, %xmm7 @@ -690,10 +690,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -836,7 +836,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -855,14 +855,14 @@ #endif ALIGN_4 -.L40: +L(40): movq N, J sarq $1, J NOBRANCH - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -901,10 +901,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -960,10 +960,10 @@ #endif sarq $3, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 @@ -1209,10 +1209,10 @@ subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1221,10 +1221,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 @@ -1260,10 +1260,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax subq $2, %rax @@ -1607,13 +1607,13 @@ decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M BRANCH - jle .L39 + jle L(39) ALIGN_4 #ifdef LN @@ -1652,10 +1652,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_4 -.L22: +L(22): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -1721,10 +1721,10 @@ subq $-16 * SIZE, BO subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1733,10 +1733,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 pshufd $0x4e, %xmm2, %xmm7 @@ -1757,10 +1757,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1958,7 +1958,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -1978,10 +1978,10 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_RT_2x2_sse2.S b/kernel/x86_64/ztrsm_kernel_RT_2x2_sse2.S index 2dffe2d327..c27af76346 100644 --- a/kernel/x86_64/ztrsm_kernel_RT_2x2_sse2.S +++ b/kernel/x86_64/ztrsm_kernel_RT_2x2_sse2.S @@ -318,9 +318,9 @@ #endif testq $1, N - jle .L100 + jle L(100) -.L101: +L(101): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -356,10 +356,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L103 + jle L(103) ALIGN_4 -.L102: +L(102): movlpd 0 * SIZE(B), %xmm0 movlpd 1 * SIZE(B), %xmm1 movlpd 2 * SIZE(B), %xmm2 @@ -389,10 +389,10 @@ subq $-16 * SIZE, BO addq $ 8 * SIZE, B decq %rax - jne .L102 + jne L(102) ALIGN_4 -.L103: +L(103): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -401,10 +401,10 @@ #endif andq $3, %rax BRANCH - jle .L105 + jle L(105) ALIGN_4 -.L104: +L(104): movlpd 0 * SIZE(B), %xmm0 movlpd 1 * SIZE(B), %xmm1 @@ -416,10 +416,10 @@ addq $4 * SIZE, BO addq $2 * SIZE, B decq %rax - jne .L104 + jne L(104) ALIGN_4 -.L105: +L(105): #if defined(LT) || defined(RN) movq A, AO #else @@ -437,10 +437,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L130 + jle L(130) ALIGN_4 -.L110: +L(110): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -475,9 +475,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L112 + je L(112) -.L111: +L(111): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -537,10 +537,10 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -550,9 +550,9 @@ movapd POSINV, %xmm15 andq $3, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -570,10 +570,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -818,15 +818,15 @@ #endif decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L130: +L(130): testq $1, M - jle .L199 + jle L(199) ALIGN_4 -.L140: +L(140): #ifdef LN movq K, %rax salq $0 + ZBASE_SHIFT, %rax @@ -860,9 +860,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L142 + je L(142) -.L141: +L(141): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -894,9 +894,9 @@ addq $8 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L141 + jne L(141) -.L142: +L(142): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -910,9 +910,9 @@ #endif andq $3, %rax # if (k & 1) BRANCH - jle .L144 + jle L(144) -.L143: +L(143): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 mulpd %xmm8, %xmm9 @@ -923,10 +923,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L143 + jg L(143) ALIGN_4 -.L144: +L(144): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1088,7 +1088,7 @@ #endif ALIGN_4 -.L199: +L(199): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -1110,13 +1110,13 @@ #endif ALIGN_4 -.L100: +L(100): movq N, J sarq $1, J # j = (n >> 2) - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -1152,12 +1152,12 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) addq %rax, %rax ALIGN_4 -.L02: +L(02): PREFETCHNTA 56 * SIZE(B) movlpd 0 * SIZE(B), %xmm0 @@ -1189,10 +1189,10 @@ subq $-16 * SIZE, BO addq $ 8 * SIZE, B decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1201,10 +1201,10 @@ #endif andq $3, %rax BRANCH - jle .L05 + jle L(05) ALIGN_4 -.L04: +L(04): movlpd 0 * SIZE(B), %xmm0 movlpd 1 * SIZE(B), %xmm1 movlpd 2 * SIZE(B), %xmm2 @@ -1223,10 +1223,10 @@ addq $ 8 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L05: +L(05): #if defined(LT) || defined(RN) movq A, AO #else @@ -1247,10 +1247,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L30 + jle L(30) ALIGN_4 -.L10: +L(10): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1301,8 +1301,8 @@ #endif andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1(16 * 0) KERNEL2(16 * 0) KERNEL3(16 * 0) @@ -1320,7 +1320,7 @@ KERNEL7(16 * 1) KERNEL8(16 * 1) cmpq $64 * 2, %rax - jle .L12 + jle L(12) KERNEL1(16 * 2) KERNEL2(16 * 2) KERNEL3(16 * 2) @@ -1338,7 +1338,7 @@ KERNEL7(16 * 3) KERNEL8(16 * 3) cmpq $64 * 4, %rax - jle .L12 + jle L(12) KERNEL1(16 * 4) KERNEL2(16 * 4) KERNEL3(16 * 4) @@ -1356,7 +1356,7 @@ KERNEL7(16 * 5) KERNEL8(16 * 5) cmpq $64 * 6, %rax - jle .L12 + jle L(12) KERNEL1(16 * 6) KERNEL2(16 * 6) KERNEL3(16 * 6) @@ -1377,14 +1377,14 @@ addq $16 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $64 * 8, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 4), BO # * 64 ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1394,10 +1394,10 @@ movapd POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L19 + je L(19) ALIGN_4 -.L16: +L(16): mulpd %xmm8, %xmm9 addpd %xmm9, %xmm0 movapd 2 * SIZE(BO), %xmm9 @@ -1426,10 +1426,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L19: +L(19): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1825,12 +1825,12 @@ #endif decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L30: +L(30): testq $1, M - jle .L99 + jle L(99) #ifdef LN movq K, %rax @@ -1865,9 +1865,9 @@ subq KK, %rax #endif sarq $2, %rax - je .L42 + je L(42) -.L41: +L(41): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 @@ -1943,9 +1943,9 @@ addq $ 8 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L41 + jne L(41) -.L42: +L(42): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1955,9 +1955,9 @@ movapd POSINV, %xmm15 andq $3, %rax # if (k & 1) BRANCH - jle .L44 + jle L(44) -.L43: +L(43): movapd 0 * SIZE(AO), %xmm8 movapd 0 * SIZE(BO), %xmm9 @@ -1980,10 +1980,10 @@ addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2211,7 +2211,7 @@ #endif ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -2233,10 +2233,10 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_3 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp diff --git a/kernel/x86_64/ztrsm_kernel_RT_2x2_sse3.S b/kernel/x86_64/ztrsm_kernel_RT_2x2_sse3.S index a473df4dcd..fb322a45e8 100644 --- a/kernel/x86_64/ztrsm_kernel_RT_2x2_sse3.S +++ b/kernel/x86_64/ztrsm_kernel_RT_2x2_sse3.S @@ -407,9 +407,9 @@ #endif testq $1, N - jle .L100 + jle L(100) -.L101: +L(101): #if defined(LT) || defined(RN) movq A, AO #else @@ -438,10 +438,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L130 + jle L(130) ALIGN_4 -.L110: +L(110): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -476,9 +476,9 @@ subq KK, %rax #endif sarq $3, %rax - je .L112 + je L(112) -.L111: +L(111): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -597,10 +597,10 @@ addq $32 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L111 + jne L(111) ALIGN_4 -.L112: +L(112): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -609,9 +609,9 @@ #endif andq $7, %rax # if (k & 1) BRANCH - jle .L114 + jle L(114) -.L113: +L(113): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 ADD1 %xmm9, %xmm0 @@ -630,10 +630,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L113 + jg L(113) ALIGN_4 -.L114: +L(114): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm5, %xmm5 @@ -848,12 +848,12 @@ #endif decq I # i -- - jg .L110 + jg L(110) ALIGN_4 -.L130: +L(130): testq $1, M - jle .L149 + jle L(149) ALIGN_4 #ifdef LN @@ -888,9 +888,9 @@ subq KK, %rax #endif sarq $3, %rax - je .L142 + je L(142) -.L141: +L(141): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -952,9 +952,9 @@ addq $16 * SIZE, AO addq $16 * SIZE, BO decq %rax - jne .L141 + jne L(141) -.L142: +L(142): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -963,9 +963,9 @@ #endif andq $7, %rax # if (k & 1) BRANCH - jle .L144 + jle L(144) -.L143: +L(143): mulpd %xmm8, %xmm9 ADD1 %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -977,10 +977,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $2 * SIZE, BO # boffset1 += 8 decq %rax - jg .L143 + jg L(143) ALIGN_4 -.L144: +L(144): addpd %xmm2, %xmm0 addpd %xmm3, %xmm1 @@ -1131,7 +1131,7 @@ #endif ALIGN_4 -.L149: +L(149): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2), B @@ -1151,13 +1151,13 @@ ALIGN_3 -.L100: +L(100): movq N, J sarq $1, J # j = (n >> 2) - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -1190,10 +1190,10 @@ movq M, I sarq $1, I # i = (m >> 2) - jle .L30 + jle L(30) ALIGN_4 -.L10: +L(10): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1239,9 +1239,9 @@ #endif andq $-8, %rax salq $4, %rax - je .L12 + je L(12) -.L1X: +L(1X): KERNEL1 (16 * 0) KERNEL2 (16 * 0) KERNEL3 (16 * 0) @@ -1260,7 +1260,7 @@ KERNEL16(16 * 0) cmpq $128 * 1, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 1) KERNEL2 (16 * 1) KERNEL3 (16 * 1) @@ -1279,7 +1279,7 @@ KERNEL16(16 * 1) cmpq $128 * 2, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 2) KERNEL2 (16 * 2) KERNEL3 (16 * 2) @@ -1298,7 +1298,7 @@ KERNEL16(16 * 2) cmpq $128 * 3, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 3) KERNEL2 (16 * 3) KERNEL3 (16 * 3) @@ -1317,7 +1317,7 @@ KERNEL16(16 * 3) cmpq $128 * 4, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 4) KERNEL2 (16 * 4) KERNEL3 (16 * 4) @@ -1336,7 +1336,7 @@ KERNEL16(16 * 4) cmpq $128 * 5, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 5) KERNEL2 (16 * 5) KERNEL3 (16 * 5) @@ -1355,7 +1355,7 @@ KERNEL16(16 * 5) cmpq $128 * 6, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 6) KERNEL2 (16 * 6) KERNEL3 (16 * 6) @@ -1374,7 +1374,7 @@ KERNEL16(16 * 6) cmpq $128 * 7, %rax NOBRANCH - jle .L11 + jle L(11) KERNEL1 (16 * 7) KERNEL2 (16 * 7) KERNEL3 (16 * 7) @@ -1395,14 +1395,14 @@ addq $32 * 8 * SIZE, AO addq $32 * 8 * SIZE, BO subq $128 * 8, %rax - jg .L1X + jg L(1X) -.L11: +L(11): leaq (AO, %rax, 2), AO # * 16 leaq (BO, %rax, 2), BO # * 64 ALIGN_4 -.L12: +L(12): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1411,10 +1411,10 @@ #endif andq $7, %rax # if (k & 1) BRANCH - je .L14 + je L(14) ALIGN_4 -.L13: +L(13): mulpd %xmm8, %xmm9 movapd 2 * SIZE(AO), %xmm10 ADD1 %xmm9, %xmm0 @@ -1445,10 +1445,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L13 + jg L(13) ALIGN_4 -.L14: +L(14): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm3, %xmm3 SHUFPD_1 %xmm5, %xmm5 @@ -1721,12 +1721,12 @@ addq %rax, AORIG #endif decq I # i -- - jg .L10 + jg L(10) ALIGN_4 -.L30: +L(30): testq $1, M - jle .L99 + jle L(99) #ifdef LN movq K, %rax @@ -1760,9 +1760,9 @@ subq KK, %rax #endif sarq $3, %rax - je .L42 + je L(42) -.L41: +L(41): mulpd %xmm8, %xmm9 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm9, %xmm0 @@ -1872,9 +1872,9 @@ addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax - jne .L41 + jne L(41) -.L42: +L(42): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1883,9 +1883,9 @@ #endif andq $7, %rax # if (k & 1) BRANCH - jle .L44 + jle L(44) -.L43: +L(43): mulpd %xmm8, %xmm9 ADD1 %xmm9, %xmm0 movddup 1 * SIZE(BO), %xmm9 @@ -1903,10 +1903,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax - jg .L43 + jg L(43) ALIGN_4 -.L44: +L(44): SHUFPD_1 %xmm1, %xmm1 SHUFPD_1 %xmm3, %xmm3 @@ -2146,7 +2146,7 @@ #endif ALIGN_4 -.L99: +L(99): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 4), B @@ -2164,10 +2164,10 @@ subq $2, KK #endif decq J # j -- - jg .L01 + jg L(01) -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_RT_2x4_nehalem.S b/kernel/x86_64/ztrsm_kernel_RT_2x4_nehalem.S index ddb5fe0982..768a9b2fa6 100644 --- a/kernel/x86_64/ztrsm_kernel_RT_2x4_nehalem.S +++ b/kernel/x86_64/ztrsm_kernel_RT_2x4_nehalem.S @@ -170,7 +170,7 @@ testq $1, N BRANCH - jle .L30 + jle L(30) #if defined(LT) || defined(RN) movq A, AO @@ -203,10 +203,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -245,10 +245,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L55 + jle L(55) ALIGN_3 -.L52: +L(52): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -288,10 +288,10 @@ subq $1, %rax BRANCH - jg .L52 + jg L(52) ALIGN_3 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -300,10 +300,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_3 -.L56: +L(56): ADD1 %xmm1, %xmm8 movddup -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -317,10 +317,10 @@ subq $1, %rax BRANCH - jg .L56 + jg L(56) ALIGN_3 -.L58: +L(58): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -533,13 +533,13 @@ decq I # i -- BRANCH - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $1, M BRANCH - jle .L69 + jle L(69) #ifdef LN movq K, %rax @@ -574,10 +574,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L65 + jle L(65) ALIGN_3 -.L62: +L(62): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -619,10 +619,10 @@ subq $1, %rax BRANCH - jg .L62 + jg L(62) ALIGN_3 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -631,10 +631,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_3 -.L66: +L(66): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -649,10 +649,10 @@ subq $1, %rax BRANCH - jg .L66 + jg L(66) ALIGN_3 -.L68: +L(68): #if defined(LN) || defined(RT) movq KK, %rax subq $1, %rax @@ -768,7 +768,7 @@ #endif ALIGN_4 -.L69: +L(69): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -788,10 +788,10 @@ #endif ALIGN_4 -.L30: +L(30): testq $2, N BRANCH - jle .L50 + jle L(50) #if defined(LT) || defined(RN) movq A, AO @@ -826,10 +826,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L40 + jle L(40) ALIGN_4 -.L31: +L(31): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -869,10 +869,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L35 + jle L(35) ALIGN_3 -.L32: +L(32): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm8 @@ -936,10 +936,10 @@ subq $1, %rax BRANCH - jg .L32 + jg L(32) ALIGN_3 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -948,10 +948,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_3 -.L36: +L(36): ADD1 %xmm1, %xmm8 movaps -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm9 @@ -971,10 +971,10 @@ subq $1, %rax BRANCH - jg .L36 + jg L(36) ALIGN_3 -.L38: +L(38): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1285,13 +1285,13 @@ decq I # i -- BRANCH - jg .L31 + jg L(31) ALIGN_4 -.L40: +L(40): testq $1, M BRANCH - jle .L49 + jle L(49) #ifdef LN movq K, %rax @@ -1331,10 +1331,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L45 + jle L(45) ALIGN_3 -.L42: +L(42): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1376,10 +1376,10 @@ subq $1, %rax BRANCH - jg .L42 + jg L(42) ALIGN_3 -.L45: +L(45): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1388,10 +1388,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L48 + je L(48) ALIGN_3 -.L46: +L(46): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -1406,10 +1406,10 @@ subq $1, %rax BRANCH - jg .L46 + jg L(46) ALIGN_3 -.L48: +L(48): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -1623,7 +1623,7 @@ #endif ALIGN_4 -.L49: +L(49): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -1643,14 +1643,14 @@ #endif ALIGN_4 -.L50: +L(50): movq N, J sarq $2, J NOBRANCH - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #if defined(LT) || defined(RN) movq A, AO #else @@ -1688,10 +1688,10 @@ movq M, I sarq $1, I NOBRANCH - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax @@ -1741,10 +1741,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L15 + jle L(15) ALIGN_3 -.L12: +L(12): PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) ADD1 %xmm1, %xmm12 @@ -1871,10 +1871,10 @@ subq $-16 * SIZE, AO subq $1, %rax BRANCH - jg .L12 + jg L(12) ALIGN_3 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1883,10 +1883,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_3 -.L16: +L(16): ADD1 %xmm1, %xmm12 movaps -32 * SIZE(BO), %xmm1 ADD2 %xmm2, %xmm13 @@ -1922,10 +1922,10 @@ subq $1, %rax BRANCH - jg .L16 + jg L(16) ALIGN_3 -.L18: +L(18): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -2486,13 +2486,13 @@ decq I # i -- BRANCH - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $1, M BRANCH - jle .L29 + jle L(29) #ifdef LN movq K, %rax @@ -2532,10 +2532,10 @@ #endif sarq $2, %rax NOBRANCH - jle .L25 + jle L(25) ALIGN_3 -.L22: +L(22): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2605,10 +2605,10 @@ subq $1, %rax BRANCH - jg .L22 + jg L(22) ALIGN_3 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2617,10 +2617,10 @@ #endif andq $3, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_3 -.L26: +L(26): ADD1 %xmm1, %xmm8 pshufd $0xa0, %xmm5, %xmm1 mulps %xmm0, %xmm1 @@ -2642,10 +2642,10 @@ subq $1, %rax BRANCH - jg .L26 + jg L(26) ALIGN_3 -.L28: +L(28): #if defined(LN) || defined(RT) movq KK, %rax #ifdef LN @@ -3063,7 +3063,7 @@ #endif ALIGN_4 -.L29: +L(29): #ifdef LN movq K, %rax salq $ZBASE_SHIFT, %rax @@ -3084,10 +3084,10 @@ subq $1, J BRANCH - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq 0(%rsp), %rbx movq 8(%rsp), %rbp movq 16(%rsp), %r12 diff --git a/kernel/x86_64/ztrsm_kernel_RT_4x2_sse.S b/kernel/x86_64/ztrsm_kernel_RT_4x2_sse.S index 518e1b4fef..eda16264bb 100644 --- a/kernel/x86_64/ztrsm_kernel_RT_4x2_sse.S +++ b/kernel/x86_64/ztrsm_kernel_RT_4x2_sse.S @@ -332,7 +332,7 @@ #endif testq $1, N - je .L40 + je L(40) ALIGN_4 #ifdef LN @@ -369,10 +369,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L43 + jle L(43) ALIGN_4 -.L42: +L(42): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 @@ -399,10 +399,10 @@ addq $ 8 * SIZE, B addq $32 * SIZE, BO decq %rax - jne .L42 + jne L(42) ALIGN_4 -.L43: +L(43): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -411,10 +411,10 @@ #endif andq $3, %rax BRANCH - jle .L50 + jle L(50) ALIGN_4 -.L44: +L(44): movlps 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -426,10 +426,10 @@ addq $2 * SIZE, B addq $8 * SIZE, BO decq %rax - jne .L44 + jne L(44) ALIGN_4 -.L50: +L(50): #if defined(LT) || defined(RN) movq A, AO #else @@ -448,10 +448,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L60 + jle L(60) ALIGN_4 -.L51: +L(51): #ifdef LN movq K, %rax salq $2 + ZBASE_SHIFT, %rax @@ -502,10 +502,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L55 + je L(55) ALIGN_4 -.L52: +L(52): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -627,10 +627,10 @@ decq %rax - jne .L52 + jne L(52) ALIGN_4 -.L55: +L(55): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -640,10 +640,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L58 + je L(58) ALIGN_4 -.L56: +L(56): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -660,10 +660,10 @@ addq $ 8 * SIZE, AO # aoffset += 4 addq $ 8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L56 + jg L(56) ALIGN_4 -.L58: +L(58): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm5, %xmm5 @@ -1206,12 +1206,12 @@ #endif decq I # i -- - jg .L51 + jg L(51) ALIGN_4 -.L60: +L(60): testq $2, M - je .L70 + je L(70) #ifdef LN movq K, %rax @@ -1254,10 +1254,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L65 + je L(65) ALIGN_4 -.L62: +L(62): mulps %xmm8, %xmm9 #if defined(OPTERON) && defined(HAVE_PREFETCH) PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) @@ -1321,10 +1321,10 @@ addq $64 * SIZE, BO decq %rax - jne .L62 + jne L(62) ALIGN_4 -.L65: +L(65): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1334,10 +1334,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L68 + je L(68) ALIGN_4 -.L66: +L(66): mulps %xmm8, %xmm9 mulps 4 * SIZE(BO), %xmm8 addps %xmm9, %xmm0 @@ -1348,10 +1348,10 @@ addq $4 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L66 + jg L(66) ALIGN_4 -.L68: +L(68): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -1599,9 +1599,9 @@ #endif ALIGN_4 -.L70: +L(70): testq $1, M - je .L79 + je L(79) #ifdef LN movq K, %rax @@ -1646,10 +1646,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L75 + je L(75) ALIGN_4 -.L72: +L(72): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -1722,10 +1722,10 @@ addq $64 * SIZE, BO decq %rax - jne .L72 + jne L(72) ALIGN_4 -.L75: +L(75): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -1735,10 +1735,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L78 + je L(78) ALIGN_4 -.L76: +L(76): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -1750,10 +1750,10 @@ addq $2 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax - jg .L76 + jg L(76) ALIGN_4 -.L78: +L(78): addps %xmm2, %xmm0 addps %xmm3, %xmm1 @@ -1897,7 +1897,7 @@ #endif ALIGN_4 -.L79: +L(79): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, COMPSIZE), B @@ -1919,13 +1919,13 @@ #endif ALIGN_4 -.L40: +L(40): movq N, J sarq $1, J # j = (n >> 2) - jle .L999 + jle L(999) ALIGN_4 -.L01: +L(01): #ifdef LN movq OFFSET, %rax addq M, %rax @@ -1960,10 +1960,10 @@ subq KK, %rax #endif sarq $2, %rax - jle .L03 + jle L(03) ALIGN_4 -.L02: +L(02): movaps 0 * SIZE(B), %xmm3 movaps 4 * SIZE(B), %xmm7 @@ -2013,10 +2013,10 @@ addq $16 * SIZE, B addq $64 * SIZE, BO decq %rax - jne .L02 + jne L(02) ALIGN_4 -.L03: +L(03): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2025,10 +2025,10 @@ #endif andq $3, %rax BRANCH - jle .L10 + jle L(10) ALIGN_4 -.L04: +L(04): movaps 0 * SIZE(B), %xmm3 pshufd $0x00, %xmm3, %xmm0 @@ -2044,10 +2044,10 @@ addq $ 4 * SIZE, B addq $16 * SIZE, BO decq %rax - jne .L04 + jne L(04) ALIGN_4 -.L10: +L(10): #if defined(LT) || defined(RN) movq A, AO #else @@ -2068,10 +2068,10 @@ movq M, I sarq $2, I # i = (m >> 2) - jle .L20 + jle L(20) ALIGN_4 -.L11: +L(11): #ifdef LN movq K, %rax salq $2 + ZBASE_SHIFT, %rax @@ -2122,8 +2122,8 @@ #endif andq $-8, %rax salq $4, %rax - je .L15 -.L1X: + je L(15) +L(1X): KERNEL1(32 * 0) KERNEL2(32 * 0) KERNEL3(32 * 0) @@ -2144,14 +2144,14 @@ addq $32 * 2 * SIZE, AO addq $64 * 2 * SIZE, BO subq $64 * 2, %rax - jg .L1X + jg L(1X) -.L12: +L(12): leaq (AO, %rax, 2), AO leaq (BO, %rax, 4), BO ALIGN_4 -.L15: +L(15): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -2161,10 +2161,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L18 + je L(18) ALIGN_4 -.L16: +L(16): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -2193,10 +2193,10 @@ addq $ 8 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L16 + jg L(16) ALIGN_4 -.L18: +L(18): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm3, %xmm3 shufps $0xb1, %xmm5, %xmm5 @@ -2914,12 +2914,12 @@ #endif decq I # i -- - jg .L11 + jg L(11) ALIGN_4 -.L20: +L(20): testq $2, M - je .L30 + je L(30) #ifdef LN movq K, %rax @@ -2964,10 +2964,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L25 + je L(25) ALIGN_4 -.L22: +L(22): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -3082,10 +3082,10 @@ addq $128 * SIZE, BO decq %rax - jne .L22 + jne L(22) ALIGN_4 -.L25: +L(25): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3095,10 +3095,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L28 + je L(28) ALIGN_4 -.L26: +L(26): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -3115,10 +3115,10 @@ addq $ 4 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L26 + jg L(26) ALIGN_4 -.L28: +L(28): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm3, %xmm3 @@ -3477,9 +3477,9 @@ #endif ALIGN_4 -.L30: +L(30): testq $1, M - je .L39 + je L(39) #ifdef LN movq K, %rax @@ -3524,10 +3524,10 @@ subq KK, %rax #endif sarq $3, %rax - je .L35 + je L(35) ALIGN_4 -.L32: +L(32): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 #if defined(OPTERON) && defined(HAVE_PREFETCH) @@ -3645,10 +3645,10 @@ addq $128 * SIZE, BO decq %rax - jne .L32 + jne L(32) ALIGN_4 -.L35: +L(35): #if defined(LT) || defined(RN) movq KK, %rax #else @@ -3658,10 +3658,10 @@ movaps POSINV, %xmm15 andq $7, %rax # if (k & 1) BRANCH - je .L38 + je L(38) ALIGN_4 -.L36: +L(36): mulps %xmm8, %xmm9 addps %xmm9, %xmm0 movaps 4 * SIZE(BO), %xmm9 @@ -3680,10 +3680,10 @@ addq $ 2 * SIZE, AO # aoffset += 4 addq $16 * SIZE, BO # boffset1 += 8 decq %rax - jg .L36 + jg L(36) ALIGN_4 -.L38: +L(38): shufps $0xb1, %xmm1, %xmm1 shufps $0xb1, %xmm3, %xmm3 @@ -3958,7 +3958,7 @@ #endif ALIGN_4 -.L39: +L(39): #ifdef LN leaq (, K, SIZE), %rax leaq (B, %rax, 2 * COMPSIZE), B @@ -3980,10 +3980,10 @@ #endif decq J # j -- - jg .L01 + jg L(01) ALIGN_4 -.L999: +L(999): movq %rbx, %rsp movq 0(%rsp), %rbx movq 8(%rsp), %rbp From 13b9f7a7975ae8f8e346d2b9e503ccb6f783a251 Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Thu, 1 Oct 2026 23:14:10 -0400 Subject: [PATCH 3/5] arm64: spell the local labels of the assembly kernels L(name) No change to any object: on ELF and COFF L(name) is .Lname, as before, and on Mach-O, where it is Lname, the labels become local to the assembler without changing the code (the arm64 kernels do not use .subsections_via_symbols, so the linker did not act on the label symbols). Preparation for using that directive on Darwin, which needs the labels of a kernel to be local: with it, the assembler cannot even branch conditionally to a label that is a symbol ("conditional branch requires assembler-local label"). Generated by asm-labels/use_local_label_macro.py: 6565 labels in 76 files. The two SME kernels, which do not include common.h, are not touched. For an ELF target the 476 kernel objects of a DYNAMIC_ARCH build are byte for byte the same, and on Darwin their text is. Generated-by: asm-labels/use_local_label_macro.py Co-Authored-By: Claude Fable 5.1 --- kernel/arm64/amax.S | 50 +- kernel/arm64/asum.S | 40 +- kernel/arm64/axpy.S | 42 +- kernel/arm64/casum.S | 40 +- kernel/arm64/cgemm_kernel_4x4.S | 284 +++++----- kernel/arm64/cgemm_kernel_8x4.S | 350 ++++++------- kernel/arm64/cgemm_kernel_8x4_thunderx2t99.S | 350 ++++++------- kernel/arm64/cgemm_kernel_sve_v1x4.S | 110 ++-- kernel/arm64/copy.S | 40 +- kernel/arm64/csum.S | 40 +- kernel/arm64/ctrmm_kernel_4x4.S | 258 +++++----- kernel/arm64/ctrmm_kernel_8x4.S | 350 ++++++------- kernel/arm64/ctrmm_kernel_sve_v1x4.S | 110 ++-- kernel/arm64/daxpy_thunderx2t99.S | 52 +- kernel/arm64/dgemm_beta.S | 54 +- kernel/arm64/dgemm_kernel_4x4.S | 286 +++++------ kernel/arm64/dgemm_kernel_4x8.S | 352 ++++++------- kernel/arm64/dgemm_kernel_6x8_cortexa72.S | 228 ++++----- kernel/arm64/dgemm_kernel_8x4.S | 338 ++++++------ kernel/arm64/dgemm_kernel_8x4_thunderx2t99.S | 338 ++++++------ kernel/arm64/dgemm_kernel_inter_sve_v2x8.S | 264 +++++----- kernel/arm64/dgemm_kernel_sve_v1x8.S | 138 ++--- kernel/arm64/dgemm_kernel_sve_v2x8.S | 264 +++++----- kernel/arm64/dgemm_ncopy_4.S | 72 +-- kernel/arm64/dgemm_ncopy_8.S | 96 ++-- kernel/arm64/dgemm_tcopy_4.S | 72 +-- kernel/arm64/dgemm_tcopy_8.S | 112 ++-- kernel/arm64/dot.S | 40 +- kernel/arm64/dtrmm_kernel_4x4.S | 258 +++++----- kernel/arm64/dtrmm_kernel_4x8.S | 352 ++++++------- kernel/arm64/dtrmm_kernel_6x8_cortexa72.S | 228 ++++----- kernel/arm64/dtrmm_kernel_8x4.S | 338 ++++++------ kernel/arm64/dtrmm_kernel_sve_v1x8.S | 138 ++--- kernel/arm64/gemv_n.S | 62 +-- kernel/arm64/gemv_t.S | 62 +-- kernel/arm64/iamax.S | 48 +- kernel/arm64/izamax.S | 48 +- kernel/arm64/nrm2.S | 32 +- kernel/arm64/rot.S | 40 +- kernel/arm64/scal.S | 50 +- kernel/arm64/sgemm_beta.S | 54 +- kernel/arm64/sgemm_kernel_16x4.S | 442 ++++++++-------- kernel/arm64/sgemm_kernel_16x4_thunderx2t99.S | 442 ++++++++-------- kernel/arm64/sgemm_kernel_4x4.S | 310 +++++------ kernel/arm64/sgemm_kernel_8x8.S | 482 ++++++++--------- kernel/arm64/sgemm_kernel_8x8_cortexa53.S | 484 +++++++++--------- kernel/arm64/sgemm_kernel_inter_sve_v2x8.S | 264 +++++----- kernel/arm64/sgemm_kernel_sve_v1x8.S | 138 ++--- kernel/arm64/sgemm_kernel_sve_v2x8.S | 264 +++++----- kernel/arm64/sgemm_ncopy_4.S | 72 +-- kernel/arm64/sgemm_ncopy_8.S | 118 ++--- kernel/arm64/sgemm_tcopy_16.S | 128 ++--- kernel/arm64/sgemm_tcopy_8.S | 112 ++-- kernel/arm64/strmm_kernel_16x4.S | 442 ++++++++-------- kernel/arm64/strmm_kernel_4x4.S | 260 +++++----- kernel/arm64/strmm_kernel_8x8.S | 482 ++++++++--------- kernel/arm64/strmm_kernel_8x8_cortexa53.S | 484 +++++++++--------- kernel/arm64/strmm_kernel_sve_v1x8.S | 138 ++--- kernel/arm64/sum.S | 40 +- kernel/arm64/swap.S | 42 +- kernel/arm64/swap_thunderx2t99.S | 44 +- kernel/arm64/zamax.S | 50 +- kernel/arm64/zasum.S | 40 +- kernel/arm64/zaxpy.S | 46 +- kernel/arm64/zdot.S | 40 +- kernel/arm64/zgemm_kernel_4x4.S | 260 +++++----- kernel/arm64/zgemm_kernel_4x4_thunderx2t99.S | 260 +++++----- kernel/arm64/zgemm_kernel_sve_v1x4.S | 110 ++-- kernel/arm64/zgemv_n.S | 52 +- kernel/arm64/zgemv_t.S | 52 +- kernel/arm64/znrm2.S | 32 +- kernel/arm64/zrot.S | 40 +- kernel/arm64/zscal.S | 70 +-- kernel/arm64/zsum.S | 40 +- kernel/arm64/ztrmm_kernel_4x4.S | 260 +++++----- kernel/arm64/ztrmm_kernel_sve_v1x4.S | 110 ++-- 76 files changed, 6565 insertions(+), 6565 deletions(-) diff --git a/kernel/arm64/amax.S b/kernel/arm64/amax.S index f535ddf273..08a69cef24 100644 --- a/kernel/arm64/amax.S +++ b/kernel/arm64/amax.S @@ -160,62 +160,62 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Lamax_kernel_zero + ble L(amax_kernel_zero) cmp INC_X, xzr - ble .Lamax_kernel_zero + ble L(amax_kernel_zero) cmp INC_X, #1 - bne .Lamax_kernel_S_BEGIN + bne L(amax_kernel_S_BEGIN) -.Lamax_kernel_F_BEGIN: +L(amax_kernel_F_BEGIN): asr I, N, #2 cmp I, xzr - beq .Lamax_kernel_F1_INIT + beq L(amax_kernel_F1_INIT) INIT_F4 subs I, I, #1 - beq .Lamax_kernel_F1 + beq L(amax_kernel_F1) -.Lamax_kernel_F4: +L(amax_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lamax_kernel_F4 + bne L(amax_kernel_F4) -.Lamax_kernel_F1: +L(amax_kernel_F1): ands I, N, #3 - ble .Lamax_kernel_L999 + ble L(amax_kernel_L999) -.Lamax_kernel_F10: +L(amax_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lamax_kernel_F10 + bne L(amax_kernel_F10) ret -.Lamax_kernel_F1_INIT: +L(amax_kernel_F1_INIT): INIT_F1 subs N, N, #1 - b .Lamax_kernel_F1 + b L(amax_kernel_F1) -.Lamax_kernel_S_BEGIN: +L(amax_kernel_S_BEGIN): INIT_S subs N, N, #1 - ble .Lamax_kernel_L999 + ble L(amax_kernel_L999) asr I, N, #2 cmp I, xzr - ble .Lamax_kernel_S1 + ble L(amax_kernel_S1) -.Lamax_kernel_S4: +L(amax_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -223,25 +223,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lamax_kernel_S4 + bne L(amax_kernel_S4) -.Lamax_kernel_S1: +L(amax_kernel_S1): ands I, N, #3 - ble .Lamax_kernel_L999 + ble L(amax_kernel_L999) -.Lamax_kernel_S10: +L(amax_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lamax_kernel_S10 + bne L(amax_kernel_S10) -.Lamax_kernel_L999: +L(amax_kernel_L999): ret -.Lamax_kernel_zero: +L(amax_kernel_zero): fmov MAXF, REG0 ret diff --git a/kernel/arm64/asum.S b/kernel/arm64/asum.S index e88eb07c2e..9ca92c392a 100644 --- a/kernel/arm64/asum.S +++ b/kernel/arm64/asum.S @@ -122,52 +122,52 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif cmp N, xzr - ble .Lasum_kernel_L999 + ble L(asum_kernel_L999) cmp INC_X, xzr - ble .Lasum_kernel_L999 + ble L(asum_kernel_L999) cmp INC_X, #1 - bne .Lasum_kernel_S_BEGIN + bne L(asum_kernel_S_BEGIN) -.Lasum_kernel_F_BEGIN: +L(asum_kernel_F_BEGIN): asr I, N, #3 cmp I, xzr - beq .Lasum_kernel_F1 + beq L(asum_kernel_F1) -.Lasum_kernel_F8: +L(asum_kernel_F8): KERNEL_F8 subs I, I, #1 - bne .Lasum_kernel_F8 + bne L(asum_kernel_F8) KERNEL_F8_FINALIZE -.Lasum_kernel_F1: +L(asum_kernel_F1): ands I, N, #7 - ble .Lasum_kernel_L999 + ble L(asum_kernel_L999) -.Lasum_kernel_F10: +L(asum_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lasum_kernel_F10 + bne L(asum_kernel_F10) -.Lasum_kernel_L999: +L(asum_kernel_L999): ret -.Lasum_kernel_S_BEGIN: +L(asum_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lasum_kernel_S1 + ble L(asum_kernel_S1) -.Lasum_kernel_S4: +L(asum_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -175,19 +175,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lasum_kernel_S4 + bne L(asum_kernel_S4) -.Lasum_kernel_S1: +L(asum_kernel_S1): ands I, N, #3 - ble .Lasum_kernel_L999 + ble L(asum_kernel_L999) -.Lasum_kernel_S10: +L(asum_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lasum_kernel_S10 + bne L(asum_kernel_S10) ret diff --git a/kernel/arm64/axpy.S b/kernel/arm64/axpy.S index 8094351105..9aa1a4f6a3 100644 --- a/kernel/arm64/axpy.S +++ b/kernel/arm64/axpy.S @@ -135,53 +135,53 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Laxpy_kernel_L999 + ble L(axpy_kernel_L999) fcmp DA, #0.0 - beq .Laxpy_kernel_L999 + beq L(axpy_kernel_L999) cmp INC_X, #1 - bne .Laxpy_kernel_S_BEGIN + bne L(axpy_kernel_S_BEGIN) cmp INC_Y, #1 - bne .Laxpy_kernel_S_BEGIN + bne L(axpy_kernel_S_BEGIN) -.Laxpy_kernel_F_BEGIN: +L(axpy_kernel_F_BEGIN): asr I, N, #3 cmp I, xzr - beq .Laxpy_kernel_F1 + beq L(axpy_kernel_F1) -.Laxpy_kernel_F8: +L(axpy_kernel_F8): KERNEL_F8 subs I, I, #1 - bne .Laxpy_kernel_F8 + bne L(axpy_kernel_F8) -.Laxpy_kernel_F1: +L(axpy_kernel_F1): ands I, N, #7 - ble .Laxpy_kernel_L999 + ble L(axpy_kernel_L999) -.Laxpy_kernel_F10: +L(axpy_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Laxpy_kernel_F10 + bne L(axpy_kernel_F10) mov w0, wzr ret -.Laxpy_kernel_S_BEGIN: +L(axpy_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Laxpy_kernel_S1 + ble L(axpy_kernel_S1) -.Laxpy_kernel_S4: +L(axpy_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -189,21 +189,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Laxpy_kernel_S4 + bne L(axpy_kernel_S4) -.Laxpy_kernel_S1: +L(axpy_kernel_S1): ands I, N, #3 - ble .Laxpy_kernel_L999 + ble L(axpy_kernel_L999) -.Laxpy_kernel_S10: +L(axpy_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Laxpy_kernel_S10 + bne L(axpy_kernel_S10) -.Laxpy_kernel_L999: +L(axpy_kernel_L999): mov w0, wzr ret diff --git a/kernel/arm64/casum.S b/kernel/arm64/casum.S index 7c82827a54..8311fed45f 100644 --- a/kernel/arm64/casum.S +++ b/kernel/arm64/casum.S @@ -98,52 +98,52 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. fmov s1, SUMF cmp N, xzr - ble .Lcasum_kernel_L999 + ble L(casum_kernel_L999) cmp INC_X, xzr - ble .Lcasum_kernel_L999 + ble L(casum_kernel_L999) cmp INC_X, #1 - bne .Lcasum_kernel_S_BEGIN + bne L(casum_kernel_S_BEGIN) -.Lcasum_kernel_F_BEGIN: +L(casum_kernel_F_BEGIN): asr I, N, #3 cmp I, xzr - beq .Lcasum_kernel_F1 + beq L(casum_kernel_F1) -.Lcasum_kernel_F8: +L(casum_kernel_F8): KERNEL_F8 subs I, I, #1 - bne .Lcasum_kernel_F8 + bne L(casum_kernel_F8) KERNEL_F8_FINALIZE -.Lcasum_kernel_F1: +L(casum_kernel_F1): ands I, N, #7 - ble .Lcasum_kernel_L999 + ble L(casum_kernel_L999) -.Lcasum_kernel_F10: +L(casum_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lcasum_kernel_F10 + bne L(casum_kernel_F10) -.Lcasum_kernel_L999: +L(casum_kernel_L999): ret -.Lcasum_kernel_S_BEGIN: +L(casum_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lcasum_kernel_S1 + ble L(casum_kernel_S1) -.Lcasum_kernel_S4: +L(casum_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -151,19 +151,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lcasum_kernel_S4 + bne L(casum_kernel_S4) -.Lcasum_kernel_S1: +L(casum_kernel_S1): ands I, N, #3 - ble .Lcasum_kernel_L999 + ble L(casum_kernel_L999) -.Lcasum_kernel_S10: +L(casum_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lcasum_kernel_S10 + bne L(casum_kernel_S10) ret diff --git a/kernel/arm64/cgemm_kernel_4x4.S b/kernel/arm64/cgemm_kernel_4x4.S index bbf0c7537c..252138e150 100644 --- a/kernel/arm64/cgemm_kernel_4x4.S +++ b/kernel/arm64/cgemm_kernel_4x4.S @@ -1072,11 +1072,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lcgemm_kernel_L2_BEGIN + ble L(cgemm_kernel_L2_BEGIN) /******************************************************************************/ -.Lcgemm_kernel_L4_BEGIN: +L(cgemm_kernel_L4_BEGIN): mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #2 @@ -1084,96 +1084,96 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array add ppA, temp, pA -.Lcgemm_kernel_L4_M8_BEGIN: +L(cgemm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lcgemm_kernel_L4_M4_BEGIN + ble L(cgemm_kernel_L4_M4_BEGIN) -.Lcgemm_kernel_L4_M8_20: +L(cgemm_kernel_L4_M8_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lcgemm_kernel_L4_M8_32 + blt L(cgemm_kernel_L4_M8_32) KERNEL8x4_I // do one in the K KERNEL8x4_M2 // do another in the K subs counterL, counterL, #2 // subtract 2 - ble .Lcgemm_kernel_L4_M8_22a + ble L(cgemm_kernel_L4_M8_22a) .align 5 -.Lcgemm_kernel_L4_M8_22: +L(cgemm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M8_22 + bgt L(cgemm_kernel_L4_M8_22) -.Lcgemm_kernel_L4_M8_22a: +L(cgemm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_E - b .Lcgemm_kernel_L4_M8_44 + b L(cgemm_kernel_L4_M8_44) -.Lcgemm_kernel_L4_M8_32: +L(cgemm_kernel_L4_M8_32): tst counterL, #1 - ble .Lcgemm_kernel_L4_M8_40 + ble L(cgemm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_E - b .Lcgemm_kernel_L4_M8_44 + b L(cgemm_kernel_L4_M8_44) -.Lcgemm_kernel_L4_M8_40: +L(cgemm_kernel_L4_M8_40): INIT8x4 -.Lcgemm_kernel_L4_M8_44: +L(cgemm_kernel_L4_M8_44): ands counterL , origK, #1 - ble .Lcgemm_kernel_L4_M8_100 + ble L(cgemm_kernel_L4_M8_100) -.Lcgemm_kernel_L4_M8_46: +L(cgemm_kernel_L4_M8_46): KERNEL8x4_SUB -.Lcgemm_kernel_L4_M8_100: +L(cgemm_kernel_L4_M8_100): SAVE8x4 -.Lcgemm_kernel_L4_M8_END: +L(cgemm_kernel_L4_M8_END): lsl temp, origK, #5 // k * 4 * 8 add pA, pA, temp add ppA, ppA, temp subs counterI, counterI, #1 - bne .Lcgemm_kernel_L4_M8_20 + bne L(cgemm_kernel_L4_M8_20) -.Lcgemm_kernel_L4_M4_BEGIN: +L(cgemm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lcgemm_kernel_L4_END + ble L(cgemm_kernel_L4_END) tst counterI, #4 - ble .Lcgemm_kernel_L4_M2_BEGIN + ble L(cgemm_kernel_L4_M2_BEGIN) -.Lcgemm_kernel_L4_M4_20: +L(cgemm_kernel_L4_M4_20): INIT4x4 mov pB, origPB asr counterL, origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lcgemm_kernel_L4_M4_40 + ble L(cgemm_kernel_L4_M4_40) -.Lcgemm_kernel_L4_M4_22: +L(cgemm_kernel_L4_M4_22): KERNEL4x4_SUB KERNEL4x4_SUB @@ -1186,47 +1186,47 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M4_22 + bgt L(cgemm_kernel_L4_M4_22) -.Lcgemm_kernel_L4_M4_40: +L(cgemm_kernel_L4_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L4_M4_100 + ble L(cgemm_kernel_L4_M4_100) -.Lcgemm_kernel_L4_M4_42: +L(cgemm_kernel_L4_M4_42): KERNEL4x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M4_42 + bgt L(cgemm_kernel_L4_M4_42) -.Lcgemm_kernel_L4_M4_100: +L(cgemm_kernel_L4_M4_100): SAVE4x4 -.Lcgemm_kernel_L4_M4_END: +L(cgemm_kernel_L4_M4_END): -.Lcgemm_kernel_L4_M2_BEGIN: +L(cgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lcgemm_kernel_L4_END + ble L(cgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lcgemm_kernel_L4_M1_BEGIN + ble L(cgemm_kernel_L4_M1_BEGIN) -.Lcgemm_kernel_L4_M2_20: +L(cgemm_kernel_L4_M2_20): INIT2x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L4_M2_40 + ble L(cgemm_kernel_L4_M2_40) -.Lcgemm_kernel_L4_M2_22: +L(cgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1239,43 +1239,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M2_22 + bgt L(cgemm_kernel_L4_M2_22) -.Lcgemm_kernel_L4_M2_40: +L(cgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L4_M2_100 + ble L(cgemm_kernel_L4_M2_100) -.Lcgemm_kernel_L4_M2_42: +L(cgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M2_42 + bgt L(cgemm_kernel_L4_M2_42) -.Lcgemm_kernel_L4_M2_100: +L(cgemm_kernel_L4_M2_100): SAVE2x4 -.Lcgemm_kernel_L4_M2_END: +L(cgemm_kernel_L4_M2_END): -.Lcgemm_kernel_L4_M1_BEGIN: +L(cgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lcgemm_kernel_L4_END + ble L(cgemm_kernel_L4_END) -.Lcgemm_kernel_L4_M1_20: +L(cgemm_kernel_L4_M1_20): INIT1x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L4_M1_40 + ble L(cgemm_kernel_L4_M1_40) -.Lcgemm_kernel_L4_M1_22: +L(cgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1287,45 +1287,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M1_22 + bgt L(cgemm_kernel_L4_M1_22) -.Lcgemm_kernel_L4_M1_40: +L(cgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L4_M1_100 + ble L(cgemm_kernel_L4_M1_100) -.Lcgemm_kernel_L4_M1_42: +L(cgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M1_42 + bgt L(cgemm_kernel_L4_M1_42) -.Lcgemm_kernel_L4_M1_100: +L(cgemm_kernel_L4_M1_100): SAVE1x4 -.Lcgemm_kernel_L4_END: +L(cgemm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 subs counterJ, counterJ , #1 // j-- - bgt .Lcgemm_kernel_L4_BEGIN + bgt L(cgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Lcgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(cgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lcgemm_kernel_L999 // error, N was less than 4? + ble L(cgemm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Lcgemm_kernel_L1_BEGIN + ble L(cgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1335,24 +1335,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lcgemm_kernel_L2_M4_BEGIN: +L(cgemm_kernel_L2_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI,#0 - ble .Lcgemm_kernel_L2_M2_BEGIN + ble L(cgemm_kernel_L2_M2_BEGIN) -.Lcgemm_kernel_L2_M4_20: +L(cgemm_kernel_L2_M4_20): INIT4x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lcgemm_kernel_L2_M4_40 + ble L(cgemm_kernel_L2_M4_40) .align 5 -.Lcgemm_kernel_L2_M4_22: +L(cgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1364,50 +1364,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M4_22 + bgt L(cgemm_kernel_L2_M4_22) -.Lcgemm_kernel_L2_M4_40: +L(cgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_M4_100 + ble L(cgemm_kernel_L2_M4_100) -.Lcgemm_kernel_L2_M4_42: +L(cgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M4_42 + bgt L(cgemm_kernel_L2_M4_42) -.Lcgemm_kernel_L2_M4_100: +L(cgemm_kernel_L2_M4_100): SAVE4x2 -.Lcgemm_kernel_L2_M4_END: +L(cgemm_kernel_L2_M4_END): subs counterI, counterI, #1 - bgt .Lcgemm_kernel_L2_M4_20 + bgt L(cgemm_kernel_L2_M4_20) -.Lcgemm_kernel_L2_M2_BEGIN: +L(cgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lcgemm_kernel_L2_END + ble L(cgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lcgemm_kernel_L2_M1_BEGIN + ble L(cgemm_kernel_L2_M1_BEGIN) -.Lcgemm_kernel_L2_M2_20: +L(cgemm_kernel_L2_M2_20): INIT2x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lcgemm_kernel_L2_M2_40 + ble L(cgemm_kernel_L2_M2_40) -.Lcgemm_kernel_L2_M2_22: +L(cgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1420,43 +1420,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M2_22 + bgt L(cgemm_kernel_L2_M2_22) -.Lcgemm_kernel_L2_M2_40: +L(cgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_M2_100 + ble L(cgemm_kernel_L2_M2_100) -.Lcgemm_kernel_L2_M2_42: +L(cgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M2_42 + bgt L(cgemm_kernel_L2_M2_42) -.Lcgemm_kernel_L2_M2_100: +L(cgemm_kernel_L2_M2_100): SAVE2x2 -.Lcgemm_kernel_L2_M2_END: +L(cgemm_kernel_L2_M2_END): -.Lcgemm_kernel_L2_M1_BEGIN: +L(cgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lcgemm_kernel_L2_END + ble L(cgemm_kernel_L2_END) -.Lcgemm_kernel_L2_M1_20: +L(cgemm_kernel_L2_M1_20): INIT1x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lcgemm_kernel_L2_M1_40 + ble L(cgemm_kernel_L2_M1_40) -.Lcgemm_kernel_L2_M1_22: +L(cgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1468,36 +1468,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M1_22 + bgt L(cgemm_kernel_L2_M1_22) -.Lcgemm_kernel_L2_M1_40: +L(cgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_M1_100 + ble L(cgemm_kernel_L2_M1_100) -.Lcgemm_kernel_L2_M1_42: +L(cgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M1_42 + bgt L(cgemm_kernel_L2_M1_42) -.Lcgemm_kernel_L2_M1_100: +L(cgemm_kernel_L2_M1_100): SAVE1x2 -.Lcgemm_kernel_L2_END: +L(cgemm_kernel_L2_END): add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 /******************************************************************************/ -.Lcgemm_kernel_L1_BEGIN: +L(cgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lcgemm_kernel_L999 // done + ble L(cgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1507,24 +1507,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lcgemm_kernel_L1_M4_BEGIN: +L(cgemm_kernel_L1_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lcgemm_kernel_L1_M2_BEGIN + ble L(cgemm_kernel_L1_M2_BEGIN) -.Lcgemm_kernel_L1_M4_20: +L(cgemm_kernel_L1_M4_20): INIT4x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_M4_40 + ble L(cgemm_kernel_L1_M4_40) .align 5 -.Lcgemm_kernel_L1_M4_22: +L(cgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1536,50 +1536,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M4_22 + bgt L(cgemm_kernel_L1_M4_22) -.Lcgemm_kernel_L1_M4_40: +L(cgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_M4_100 + ble L(cgemm_kernel_L1_M4_100) -.Lcgemm_kernel_L1_M4_42: +L(cgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M4_42 + bgt L(cgemm_kernel_L1_M4_42) -.Lcgemm_kernel_L1_M4_100: +L(cgemm_kernel_L1_M4_100): SAVE4x1 -.Lcgemm_kernel_L1_M4_END: +L(cgemm_kernel_L1_M4_END): subs counterI, counterI, #1 - bgt .Lcgemm_kernel_L1_M4_20 + bgt L(cgemm_kernel_L1_M4_20) -.Lcgemm_kernel_L1_M2_BEGIN: +L(cgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lcgemm_kernel_L1_END + ble L(cgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lcgemm_kernel_L1_M1_BEGIN + ble L(cgemm_kernel_L1_M1_BEGIN) -.Lcgemm_kernel_L1_M2_20: +L(cgemm_kernel_L1_M2_20): INIT2x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_M2_40 + ble L(cgemm_kernel_L1_M2_40) -.Lcgemm_kernel_L1_M2_22: +L(cgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1592,43 +1592,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M2_22 + bgt L(cgemm_kernel_L1_M2_22) -.Lcgemm_kernel_L1_M2_40: +L(cgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_M2_100 + ble L(cgemm_kernel_L1_M2_100) -.Lcgemm_kernel_L1_M2_42: +L(cgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M2_42 + bgt L(cgemm_kernel_L1_M2_42) -.Lcgemm_kernel_L1_M2_100: +L(cgemm_kernel_L1_M2_100): SAVE2x1 -.Lcgemm_kernel_L1_M2_END: +L(cgemm_kernel_L1_M2_END): -.Lcgemm_kernel_L1_M1_BEGIN: +L(cgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lcgemm_kernel_L1_END + ble L(cgemm_kernel_L1_END) -.Lcgemm_kernel_L1_M1_20: +L(cgemm_kernel_L1_M1_20): INIT1x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_M1_40 + ble L(cgemm_kernel_L1_M1_40) -.Lcgemm_kernel_L1_M1_22: +L(cgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1640,30 +1640,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M1_22 + bgt L(cgemm_kernel_L1_M1_22) -.Lcgemm_kernel_L1_M1_40: +L(cgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_M1_100 + ble L(cgemm_kernel_L1_M1_100) -.Lcgemm_kernel_L1_M1_42: +L(cgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M1_42 + bgt L(cgemm_kernel_L1_M1_42) -.Lcgemm_kernel_L1_M1_100: +L(cgemm_kernel_L1_M1_100): SAVE1x1 -.Lcgemm_kernel_L1_END: +L(cgemm_kernel_L1_END): -.Lcgemm_kernel_L999: +L(cgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/cgemm_kernel_8x4.S b/kernel/arm64/cgemm_kernel_8x4.S index f100adc7af..0107d83f5e 100644 --- a/kernel/arm64/cgemm_kernel_8x4.S +++ b/kernel/arm64/cgemm_kernel_8x4.S @@ -1407,11 +1407,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lcgemm_kernel_L2_BEGIN + ble L(cgemm_kernel_L2_BEGIN) /******************************************************************************/ -.Lcgemm_kernel_L4_BEGIN: +L(cgemm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -1421,21 +1421,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lcgemm_kernel_L4_M8_BEGIN: +L(cgemm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lcgemm_kernel_L4_M4_BEGIN + ble L(cgemm_kernel_L4_M4_BEGIN) .align 5 -.Lcgemm_kernel_L4_M8_20: +L(cgemm_kernel_L4_M8_20): mov pB, origPB asr counterL , origK, #3 cmp counterL , #2 - blt .Lcgemm_kernel_L4_M8_32 + blt L(cgemm_kernel_L4_M8_32) KERNEL8x4_I KERNEL8x4_M2 @@ -1447,10 +1447,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lcgemm_kernel_L4_M8_22a + ble L(cgemm_kernel_L4_M8_22a) .align 5 -.Lcgemm_kernel_L4_M8_22: +L(cgemm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 @@ -1462,10 +1462,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M8_22 + bgt L(cgemm_kernel_L4_M8_22) .align 5 -.Lcgemm_kernel_L4_M8_22a: +L(cgemm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_M2 @@ -1476,13 +1476,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Lcgemm_kernel_L4_M8_44 + b L(cgemm_kernel_L4_M8_44) .align 5 -.Lcgemm_kernel_L4_M8_32: +L(cgemm_kernel_L4_M8_32): tst counterL, #1 - ble .Lcgemm_kernel_L4_M8_40 + ble L(cgemm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_M2 @@ -1493,116 +1493,116 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Lcgemm_kernel_L4_M8_44 + b L(cgemm_kernel_L4_M8_44) -.Lcgemm_kernel_L4_M8_40: +L(cgemm_kernel_L4_M8_40): INIT8x4 -.Lcgemm_kernel_L4_M8_44: +L(cgemm_kernel_L4_M8_44): ands counterL , origK, #7 - ble .Lcgemm_kernel_L4_M8_100 + ble L(cgemm_kernel_L4_M8_100) .align 5 -.Lcgemm_kernel_L4_M8_46: +L(cgemm_kernel_L4_M8_46): KERNEL8x4_SUB subs counterL, counterL, #1 - bne .Lcgemm_kernel_L4_M8_46 + bne L(cgemm_kernel_L4_M8_46) -.Lcgemm_kernel_L4_M8_100: +L(cgemm_kernel_L4_M8_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVE8x4 -.Lcgemm_kernel_L4_M8_END: +L(cgemm_kernel_L4_M8_END): subs counterI, counterI, #1 - bne .Lcgemm_kernel_L4_M8_20 + bne L(cgemm_kernel_L4_M8_20) -.Lcgemm_kernel_L4_M4_BEGIN: +L(cgemm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lcgemm_kernel_L4_END + ble L(cgemm_kernel_L4_END) tst counterI, #4 - ble .Lcgemm_kernel_L4_M2_BEGIN + ble L(cgemm_kernel_L4_M2_BEGIN) -.Lcgemm_kernel_L4_M4_20: +L(cgemm_kernel_L4_M4_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lcgemm_kernel_L4_M4_32 + blt L(cgemm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lcgemm_kernel_L4_M4_22a + ble L(cgemm_kernel_L4_M4_22a) .align 5 -.Lcgemm_kernel_L4_M4_22: +L(cgemm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M4_22 + bgt L(cgemm_kernel_L4_M4_22) -.Lcgemm_kernel_L4_M4_22a: +L(cgemm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lcgemm_kernel_L4_M4_44 -.Lcgemm_kernel_L4_M4_32: + b L(cgemm_kernel_L4_M4_44) +L(cgemm_kernel_L4_M4_32): tst counterL, #1 - ble .Lcgemm_kernel_L4_M4_40 + ble L(cgemm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lcgemm_kernel_L4_M4_44 -.Lcgemm_kernel_L4_M4_40: + b L(cgemm_kernel_L4_M4_44) +L(cgemm_kernel_L4_M4_40): INIT4x4 -.Lcgemm_kernel_L4_M4_44: +L(cgemm_kernel_L4_M4_44): ands counterL , origK, #1 - ble .Lcgemm_kernel_L4_M4_100 + ble L(cgemm_kernel_L4_M4_100) -.Lcgemm_kernel_L4_M4_46: +L(cgemm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lcgemm_kernel_L4_M4_100: +L(cgemm_kernel_L4_M4_100): SAVE4x4 -.Lcgemm_kernel_L4_M4_END: +L(cgemm_kernel_L4_M4_END): -.Lcgemm_kernel_L4_M2_BEGIN: +L(cgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lcgemm_kernel_L4_END + ble L(cgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lcgemm_kernel_L4_M1_BEGIN + ble L(cgemm_kernel_L4_M1_BEGIN) -.Lcgemm_kernel_L4_M2_20: +L(cgemm_kernel_L4_M2_20): INIT2x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L4_M2_40 + ble L(cgemm_kernel_L4_M2_40) -.Lcgemm_kernel_L4_M2_22: +L(cgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1615,43 +1615,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M2_22 + bgt L(cgemm_kernel_L4_M2_22) -.Lcgemm_kernel_L4_M2_40: +L(cgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L4_M2_100 + ble L(cgemm_kernel_L4_M2_100) -.Lcgemm_kernel_L4_M2_42: +L(cgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M2_42 + bgt L(cgemm_kernel_L4_M2_42) -.Lcgemm_kernel_L4_M2_100: +L(cgemm_kernel_L4_M2_100): SAVE2x4 -.Lcgemm_kernel_L4_M2_END: +L(cgemm_kernel_L4_M2_END): -.Lcgemm_kernel_L4_M1_BEGIN: +L(cgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lcgemm_kernel_L4_END + ble L(cgemm_kernel_L4_END) -.Lcgemm_kernel_L4_M1_20: +L(cgemm_kernel_L4_M1_20): INIT1x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L4_M1_40 + ble L(cgemm_kernel_L4_M1_40) -.Lcgemm_kernel_L4_M1_22: +L(cgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1663,45 +1663,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M1_22 + bgt L(cgemm_kernel_L4_M1_22) -.Lcgemm_kernel_L4_M1_40: +L(cgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L4_M1_100 + ble L(cgemm_kernel_L4_M1_100) -.Lcgemm_kernel_L4_M1_42: +L(cgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M1_42 + bgt L(cgemm_kernel_L4_M1_42) -.Lcgemm_kernel_L4_M1_100: +L(cgemm_kernel_L4_M1_100): SAVE1x4 -.Lcgemm_kernel_L4_END: +L(cgemm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 subs counterJ, counterJ , #1 // j-- - bgt .Lcgemm_kernel_L4_BEGIN + bgt L(cgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Lcgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(cgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lcgemm_kernel_L999 // error, N was less than 4? + ble L(cgemm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Lcgemm_kernel_L1_BEGIN + ble L(cgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1710,14 +1710,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lcgemm_kernel_L2_M8_BEGIN: +L(cgemm_kernel_L2_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lcgemm_kernel_L2_M4_BEGIN + ble L(cgemm_kernel_L2_M4_BEGIN) -.Lcgemm_kernel_L2_M8_20: +L(cgemm_kernel_L2_M8_20): INIT8x2 @@ -1725,10 +1725,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lcgemm_kernel_L2_M8_40 + ble L(cgemm_kernel_L2_M8_40) .align 5 -.Lcgemm_kernel_L2_M8_22: +L(cgemm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB KERNEL8x2_SUB @@ -1740,50 +1740,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M8_22 + bgt L(cgemm_kernel_L2_M8_22) -.Lcgemm_kernel_L2_M8_40: +L(cgemm_kernel_L2_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_M8_100 + ble L(cgemm_kernel_L2_M8_100) -.Lcgemm_kernel_L2_M8_42: +L(cgemm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M8_42 + bgt L(cgemm_kernel_L2_M8_42) -.Lcgemm_kernel_L2_M8_100: +L(cgemm_kernel_L2_M8_100): SAVE8x2 -.Lcgemm_kernel_L2_M8_END: +L(cgemm_kernel_L2_M8_END): subs counterI, counterI, #1 - bgt .Lcgemm_kernel_L2_M8_20 + bgt L(cgemm_kernel_L2_M8_20) -.Lcgemm_kernel_L2_M4_BEGIN: +L(cgemm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lcgemm_kernel_L2_END + ble L(cgemm_kernel_L2_END) tst counterI, #4 // counterI = counterI / 2 - ble .Lcgemm_kernel_L2_M2_BEGIN + ble L(cgemm_kernel_L2_M2_BEGIN) -.Lcgemm_kernel_L2_M4_20: +L(cgemm_kernel_L2_M4_20): INIT4x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lcgemm_kernel_L2_M4_40 + ble L(cgemm_kernel_L2_M4_40) .align 5 -.Lcgemm_kernel_L2_M4_22: +L(cgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1795,46 +1795,46 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M4_22 + bgt L(cgemm_kernel_L2_M4_22) -.Lcgemm_kernel_L2_M4_40: +L(cgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_M4_100 + ble L(cgemm_kernel_L2_M4_100) -.Lcgemm_kernel_L2_M4_42: +L(cgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M4_42 + bgt L(cgemm_kernel_L2_M4_42) -.Lcgemm_kernel_L2_M4_100: +L(cgemm_kernel_L2_M4_100): SAVE4x2 -.Lcgemm_kernel_L2_M4_END: +L(cgemm_kernel_L2_M4_END): -.Lcgemm_kernel_L2_M2_BEGIN: +L(cgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lcgemm_kernel_L2_END + ble L(cgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lcgemm_kernel_L2_M1_BEGIN + ble L(cgemm_kernel_L2_M1_BEGIN) -.Lcgemm_kernel_L2_M2_20: +L(cgemm_kernel_L2_M2_20): INIT2x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lcgemm_kernel_L2_M2_40 + ble L(cgemm_kernel_L2_M2_40) -.Lcgemm_kernel_L2_M2_22: +L(cgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1847,43 +1847,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M2_22 + bgt L(cgemm_kernel_L2_M2_22) -.Lcgemm_kernel_L2_M2_40: +L(cgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_M2_100 + ble L(cgemm_kernel_L2_M2_100) -.Lcgemm_kernel_L2_M2_42: +L(cgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M2_42 + bgt L(cgemm_kernel_L2_M2_42) -.Lcgemm_kernel_L2_M2_100: +L(cgemm_kernel_L2_M2_100): SAVE2x2 -.Lcgemm_kernel_L2_M2_END: +L(cgemm_kernel_L2_M2_END): -.Lcgemm_kernel_L2_M1_BEGIN: +L(cgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lcgemm_kernel_L2_END + ble L(cgemm_kernel_L2_END) -.Lcgemm_kernel_L2_M1_20: +L(cgemm_kernel_L2_M1_20): INIT1x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lcgemm_kernel_L2_M1_40 + ble L(cgemm_kernel_L2_M1_40) -.Lcgemm_kernel_L2_M1_22: +L(cgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1895,36 +1895,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M1_22 + bgt L(cgemm_kernel_L2_M1_22) -.Lcgemm_kernel_L2_M1_40: +L(cgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_M1_100 + ble L(cgemm_kernel_L2_M1_100) -.Lcgemm_kernel_L2_M1_42: +L(cgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M1_42 + bgt L(cgemm_kernel_L2_M1_42) -.Lcgemm_kernel_L2_M1_100: +L(cgemm_kernel_L2_M1_100): SAVE1x2 -.Lcgemm_kernel_L2_END: +L(cgemm_kernel_L2_END): add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 /******************************************************************************/ -.Lcgemm_kernel_L1_BEGIN: +L(cgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lcgemm_kernel_L999 // done + ble L(cgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1933,24 +1933,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lcgemm_kernel_L1_M8_BEGIN: +L(cgemm_kernel_L1_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lcgemm_kernel_L1_M4_BEGIN + ble L(cgemm_kernel_L1_M4_BEGIN) -.Lcgemm_kernel_L1_M8_20: +L(cgemm_kernel_L1_M8_20): INIT8x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_M8_40 + ble L(cgemm_kernel_L1_M8_40) .align 5 -.Lcgemm_kernel_L1_M8_22: +L(cgemm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -1962,51 +1962,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M8_22 + bgt L(cgemm_kernel_L1_M8_22) -.Lcgemm_kernel_L1_M8_40: +L(cgemm_kernel_L1_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_M8_100 + ble L(cgemm_kernel_L1_M8_100) -.Lcgemm_kernel_L1_M8_42: +L(cgemm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M8_42 + bgt L(cgemm_kernel_L1_M8_42) -.Lcgemm_kernel_L1_M8_100: +L(cgemm_kernel_L1_M8_100): SAVE8x1 -.Lcgemm_kernel_L1_M8_END: +L(cgemm_kernel_L1_M8_END): subs counterI, counterI, #1 - bgt .Lcgemm_kernel_L1_M8_20 + bgt L(cgemm_kernel_L1_M8_20) -.Lcgemm_kernel_L1_M4_BEGIN: +L(cgemm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lcgemm_kernel_L1_END + ble L(cgemm_kernel_L1_END) tst counterI, #4 // counterI = counterI / 2 - ble .Lcgemm_kernel_L1_M2_BEGIN + ble L(cgemm_kernel_L1_M2_BEGIN) -.Lcgemm_kernel_L1_M4_20: +L(cgemm_kernel_L1_M4_20): INIT4x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_M4_40 + ble L(cgemm_kernel_L1_M4_40) .align 5 -.Lcgemm_kernel_L1_M4_22: +L(cgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -2018,47 +2018,47 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M4_22 + bgt L(cgemm_kernel_L1_M4_22) -.Lcgemm_kernel_L1_M4_40: +L(cgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_M4_100 + ble L(cgemm_kernel_L1_M4_100) -.Lcgemm_kernel_L1_M4_42: +L(cgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M4_42 + bgt L(cgemm_kernel_L1_M4_42) -.Lcgemm_kernel_L1_M4_100: +L(cgemm_kernel_L1_M4_100): SAVE4x1 -.Lcgemm_kernel_L1_M4_END: +L(cgemm_kernel_L1_M4_END): -.Lcgemm_kernel_L1_M2_BEGIN: +L(cgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lcgemm_kernel_L1_END + ble L(cgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lcgemm_kernel_L1_M1_BEGIN + ble L(cgemm_kernel_L1_M1_BEGIN) -.Lcgemm_kernel_L1_M2_20: +L(cgemm_kernel_L1_M2_20): INIT2x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_M2_40 + ble L(cgemm_kernel_L1_M2_40) -.Lcgemm_kernel_L1_M2_22: +L(cgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -2071,43 +2071,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M2_22 + bgt L(cgemm_kernel_L1_M2_22) -.Lcgemm_kernel_L1_M2_40: +L(cgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_M2_100 + ble L(cgemm_kernel_L1_M2_100) -.Lcgemm_kernel_L1_M2_42: +L(cgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M2_42 + bgt L(cgemm_kernel_L1_M2_42) -.Lcgemm_kernel_L1_M2_100: +L(cgemm_kernel_L1_M2_100): SAVE2x1 -.Lcgemm_kernel_L1_M2_END: +L(cgemm_kernel_L1_M2_END): -.Lcgemm_kernel_L1_M1_BEGIN: +L(cgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lcgemm_kernel_L1_END + ble L(cgemm_kernel_L1_END) -.Lcgemm_kernel_L1_M1_20: +L(cgemm_kernel_L1_M1_20): INIT1x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_M1_40 + ble L(cgemm_kernel_L1_M1_40) -.Lcgemm_kernel_L1_M1_22: +L(cgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -2119,30 +2119,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M1_22 + bgt L(cgemm_kernel_L1_M1_22) -.Lcgemm_kernel_L1_M1_40: +L(cgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_M1_100 + ble L(cgemm_kernel_L1_M1_100) -.Lcgemm_kernel_L1_M1_42: +L(cgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M1_42 + bgt L(cgemm_kernel_L1_M1_42) -.Lcgemm_kernel_L1_M1_100: +L(cgemm_kernel_L1_M1_100): SAVE1x1 -.Lcgemm_kernel_L1_END: +L(cgemm_kernel_L1_END): -.Lcgemm_kernel_L999: +L(cgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/cgemm_kernel_8x4_thunderx2t99.S b/kernel/arm64/cgemm_kernel_8x4_thunderx2t99.S index 2c63925be2..a6fb58e2fc 100644 --- a/kernel/arm64/cgemm_kernel_8x4_thunderx2t99.S +++ b/kernel/arm64/cgemm_kernel_8x4_thunderx2t99.S @@ -1432,11 +1432,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lcgemm_kernel_L2_BEGIN + ble L(cgemm_kernel_L2_BEGIN) /******************************************************************************/ -.Lcgemm_kernel_L4_BEGIN: +L(cgemm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -1446,21 +1446,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lcgemm_kernel_L4_M8_BEGIN: +L(cgemm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lcgemm_kernel_L4_M4_BEGIN + ble L(cgemm_kernel_L4_M4_BEGIN) .align 5 -.Lcgemm_kernel_L4_M8_20: +L(cgemm_kernel_L4_M8_20): mov pB, origPB asr counterL , origK, #5 // origK / 32 cmp counterL , #2 - blt .Lcgemm_kernel_L4_M8_32 + blt L(cgemm_kernel_L4_M8_32) KERNEL8x4_I KERNEL8x4_M2 @@ -1470,18 +1470,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1_M2_x8 subs counterL, counterL, #2 // subtract 2 - ble .Lcgemm_kernel_L4_M8_22a + ble L(cgemm_kernel_L4_M8_22a) .align 5 -.Lcgemm_kernel_L4_M8_22: +L(cgemm_kernel_L4_M8_22): KERNEL8x4_M1_M2_x16 subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M8_22 + bgt L(cgemm_kernel_L4_M8_22) .align 5 -.Lcgemm_kernel_L4_M8_22a: +L(cgemm_kernel_L4_M8_22a): KERNEL8x4_M1_M2_x8 KERNEL8x4_M1_M2_x4 @@ -1490,13 +1490,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Lcgemm_kernel_L4_M8_44 + b L(cgemm_kernel_L4_M8_44) .align 5 -.Lcgemm_kernel_L4_M8_32: +L(cgemm_kernel_L4_M8_32): tst counterL, #1 - ble .Lcgemm_kernel_L4_M8_40 + ble L(cgemm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_M2 @@ -1506,116 +1506,116 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Lcgemm_kernel_L4_M8_44 + b L(cgemm_kernel_L4_M8_44) -.Lcgemm_kernel_L4_M8_40: +L(cgemm_kernel_L4_M8_40): INIT8x4 -.Lcgemm_kernel_L4_M8_44: +L(cgemm_kernel_L4_M8_44): ands counterL , origK, #31 - ble .Lcgemm_kernel_L4_M8_100 + ble L(cgemm_kernel_L4_M8_100) .align 5 -.Lcgemm_kernel_L4_M8_46: +L(cgemm_kernel_L4_M8_46): KERNEL8x4_SUB subs counterL, counterL, #1 - bne .Lcgemm_kernel_L4_M8_46 + bne L(cgemm_kernel_L4_M8_46) -.Lcgemm_kernel_L4_M8_100: +L(cgemm_kernel_L4_M8_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVE8x4 -.Lcgemm_kernel_L4_M8_END: +L(cgemm_kernel_L4_M8_END): subs counterI, counterI, #1 - bne .Lcgemm_kernel_L4_M8_20 + bne L(cgemm_kernel_L4_M8_20) -.Lcgemm_kernel_L4_M4_BEGIN: +L(cgemm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lcgemm_kernel_L4_END + ble L(cgemm_kernel_L4_END) tst counterI, #4 - ble .Lcgemm_kernel_L4_M2_BEGIN + ble L(cgemm_kernel_L4_M2_BEGIN) -.Lcgemm_kernel_L4_M4_20: +L(cgemm_kernel_L4_M4_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lcgemm_kernel_L4_M4_32 + blt L(cgemm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lcgemm_kernel_L4_M4_22a + ble L(cgemm_kernel_L4_M4_22a) .align 5 -.Lcgemm_kernel_L4_M4_22: +L(cgemm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M4_22 + bgt L(cgemm_kernel_L4_M4_22) -.Lcgemm_kernel_L4_M4_22a: +L(cgemm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lcgemm_kernel_L4_M4_44 -.Lcgemm_kernel_L4_M4_32: + b L(cgemm_kernel_L4_M4_44) +L(cgemm_kernel_L4_M4_32): tst counterL, #1 - ble .Lcgemm_kernel_L4_M4_40 + ble L(cgemm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lcgemm_kernel_L4_M4_44 -.Lcgemm_kernel_L4_M4_40: + b L(cgemm_kernel_L4_M4_44) +L(cgemm_kernel_L4_M4_40): INIT4x4 -.Lcgemm_kernel_L4_M4_44: +L(cgemm_kernel_L4_M4_44): ands counterL , origK, #1 - ble .Lcgemm_kernel_L4_M4_100 + ble L(cgemm_kernel_L4_M4_100) -.Lcgemm_kernel_L4_M4_46: +L(cgemm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lcgemm_kernel_L4_M4_100: +L(cgemm_kernel_L4_M4_100): SAVE4x4 -.Lcgemm_kernel_L4_M4_END: +L(cgemm_kernel_L4_M4_END): -.Lcgemm_kernel_L4_M2_BEGIN: +L(cgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lcgemm_kernel_L4_END + ble L(cgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lcgemm_kernel_L4_M1_BEGIN + ble L(cgemm_kernel_L4_M1_BEGIN) -.Lcgemm_kernel_L4_M2_20: +L(cgemm_kernel_L4_M2_20): INIT2x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L4_M2_40 + ble L(cgemm_kernel_L4_M2_40) -.Lcgemm_kernel_L4_M2_22: +L(cgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1628,43 +1628,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M2_22 + bgt L(cgemm_kernel_L4_M2_22) -.Lcgemm_kernel_L4_M2_40: +L(cgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L4_M2_100 + ble L(cgemm_kernel_L4_M2_100) -.Lcgemm_kernel_L4_M2_42: +L(cgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M2_42 + bgt L(cgemm_kernel_L4_M2_42) -.Lcgemm_kernel_L4_M2_100: +L(cgemm_kernel_L4_M2_100): SAVE2x4 -.Lcgemm_kernel_L4_M2_END: +L(cgemm_kernel_L4_M2_END): -.Lcgemm_kernel_L4_M1_BEGIN: +L(cgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lcgemm_kernel_L4_END + ble L(cgemm_kernel_L4_END) -.Lcgemm_kernel_L4_M1_20: +L(cgemm_kernel_L4_M1_20): INIT1x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L4_M1_40 + ble L(cgemm_kernel_L4_M1_40) -.Lcgemm_kernel_L4_M1_22: +L(cgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1676,45 +1676,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M1_22 + bgt L(cgemm_kernel_L4_M1_22) -.Lcgemm_kernel_L4_M1_40: +L(cgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L4_M1_100 + ble L(cgemm_kernel_L4_M1_100) -.Lcgemm_kernel_L4_M1_42: +L(cgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_M1_42 + bgt L(cgemm_kernel_L4_M1_42) -.Lcgemm_kernel_L4_M1_100: +L(cgemm_kernel_L4_M1_100): SAVE1x4 -.Lcgemm_kernel_L4_END: +L(cgemm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 subs counterJ, counterJ , #1 // j-- - bgt .Lcgemm_kernel_L4_BEGIN + bgt L(cgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Lcgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(cgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lcgemm_kernel_L999 // error, N was less than 4? + ble L(cgemm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Lcgemm_kernel_L1_BEGIN + ble L(cgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1723,14 +1723,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lcgemm_kernel_L2_M8_BEGIN: +L(cgemm_kernel_L2_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lcgemm_kernel_L2_M4_BEGIN + ble L(cgemm_kernel_L2_M4_BEGIN) -.Lcgemm_kernel_L2_M8_20: +L(cgemm_kernel_L2_M8_20): INIT8x2 @@ -1738,10 +1738,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lcgemm_kernel_L2_M8_40 + ble L(cgemm_kernel_L2_M8_40) .align 5 -.Lcgemm_kernel_L2_M8_22: +L(cgemm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB KERNEL8x2_SUB @@ -1753,50 +1753,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M8_22 + bgt L(cgemm_kernel_L2_M8_22) -.Lcgemm_kernel_L2_M8_40: +L(cgemm_kernel_L2_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_M8_100 + ble L(cgemm_kernel_L2_M8_100) -.Lcgemm_kernel_L2_M8_42: +L(cgemm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M8_42 + bgt L(cgemm_kernel_L2_M8_42) -.Lcgemm_kernel_L2_M8_100: +L(cgemm_kernel_L2_M8_100): SAVE8x2 -.Lcgemm_kernel_L2_M8_END: +L(cgemm_kernel_L2_M8_END): subs counterI, counterI, #1 - bgt .Lcgemm_kernel_L2_M8_20 + bgt L(cgemm_kernel_L2_M8_20) -.Lcgemm_kernel_L2_M4_BEGIN: +L(cgemm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lcgemm_kernel_L2_END + ble L(cgemm_kernel_L2_END) tst counterI, #4 // counterI = counterI / 2 - ble .Lcgemm_kernel_L2_M2_BEGIN + ble L(cgemm_kernel_L2_M2_BEGIN) -.Lcgemm_kernel_L2_M4_20: +L(cgemm_kernel_L2_M4_20): INIT4x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lcgemm_kernel_L2_M4_40 + ble L(cgemm_kernel_L2_M4_40) .align 5 -.Lcgemm_kernel_L2_M4_22: +L(cgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1808,46 +1808,46 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M4_22 + bgt L(cgemm_kernel_L2_M4_22) -.Lcgemm_kernel_L2_M4_40: +L(cgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_M4_100 + ble L(cgemm_kernel_L2_M4_100) -.Lcgemm_kernel_L2_M4_42: +L(cgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M4_42 + bgt L(cgemm_kernel_L2_M4_42) -.Lcgemm_kernel_L2_M4_100: +L(cgemm_kernel_L2_M4_100): SAVE4x2 -.Lcgemm_kernel_L2_M4_END: +L(cgemm_kernel_L2_M4_END): -.Lcgemm_kernel_L2_M2_BEGIN: +L(cgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lcgemm_kernel_L2_END + ble L(cgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lcgemm_kernel_L2_M1_BEGIN + ble L(cgemm_kernel_L2_M1_BEGIN) -.Lcgemm_kernel_L2_M2_20: +L(cgemm_kernel_L2_M2_20): INIT2x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lcgemm_kernel_L2_M2_40 + ble L(cgemm_kernel_L2_M2_40) -.Lcgemm_kernel_L2_M2_22: +L(cgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1860,43 +1860,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M2_22 + bgt L(cgemm_kernel_L2_M2_22) -.Lcgemm_kernel_L2_M2_40: +L(cgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_M2_100 + ble L(cgemm_kernel_L2_M2_100) -.Lcgemm_kernel_L2_M2_42: +L(cgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M2_42 + bgt L(cgemm_kernel_L2_M2_42) -.Lcgemm_kernel_L2_M2_100: +L(cgemm_kernel_L2_M2_100): SAVE2x2 -.Lcgemm_kernel_L2_M2_END: +L(cgemm_kernel_L2_M2_END): -.Lcgemm_kernel_L2_M1_BEGIN: +L(cgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lcgemm_kernel_L2_END + ble L(cgemm_kernel_L2_END) -.Lcgemm_kernel_L2_M1_20: +L(cgemm_kernel_L2_M1_20): INIT1x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lcgemm_kernel_L2_M1_40 + ble L(cgemm_kernel_L2_M1_40) -.Lcgemm_kernel_L2_M1_22: +L(cgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1908,36 +1908,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M1_22 + bgt L(cgemm_kernel_L2_M1_22) -.Lcgemm_kernel_L2_M1_40: +L(cgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_M1_100 + ble L(cgemm_kernel_L2_M1_100) -.Lcgemm_kernel_L2_M1_42: +L(cgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_M1_42 + bgt L(cgemm_kernel_L2_M1_42) -.Lcgemm_kernel_L2_M1_100: +L(cgemm_kernel_L2_M1_100): SAVE1x2 -.Lcgemm_kernel_L2_END: +L(cgemm_kernel_L2_END): add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 /******************************************************************************/ -.Lcgemm_kernel_L1_BEGIN: +L(cgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lcgemm_kernel_L999 // done + ble L(cgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1946,24 +1946,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lcgemm_kernel_L1_M8_BEGIN: +L(cgemm_kernel_L1_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lcgemm_kernel_L1_M4_BEGIN + ble L(cgemm_kernel_L1_M4_BEGIN) -.Lcgemm_kernel_L1_M8_20: +L(cgemm_kernel_L1_M8_20): INIT8x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_M8_40 + ble L(cgemm_kernel_L1_M8_40) .align 5 -.Lcgemm_kernel_L1_M8_22: +L(cgemm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -1975,51 +1975,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M8_22 + bgt L(cgemm_kernel_L1_M8_22) -.Lcgemm_kernel_L1_M8_40: +L(cgemm_kernel_L1_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_M8_100 + ble L(cgemm_kernel_L1_M8_100) -.Lcgemm_kernel_L1_M8_42: +L(cgemm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M8_42 + bgt L(cgemm_kernel_L1_M8_42) -.Lcgemm_kernel_L1_M8_100: +L(cgemm_kernel_L1_M8_100): SAVE8x1 -.Lcgemm_kernel_L1_M8_END: +L(cgemm_kernel_L1_M8_END): subs counterI, counterI, #1 - bgt .Lcgemm_kernel_L1_M8_20 + bgt L(cgemm_kernel_L1_M8_20) -.Lcgemm_kernel_L1_M4_BEGIN: +L(cgemm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lcgemm_kernel_L1_END + ble L(cgemm_kernel_L1_END) tst counterI, #4 // counterI = counterI / 2 - ble .Lcgemm_kernel_L1_M2_BEGIN + ble L(cgemm_kernel_L1_M2_BEGIN) -.Lcgemm_kernel_L1_M4_20: +L(cgemm_kernel_L1_M4_20): INIT4x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_M4_40 + ble L(cgemm_kernel_L1_M4_40) .align 5 -.Lcgemm_kernel_L1_M4_22: +L(cgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -2031,47 +2031,47 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M4_22 + bgt L(cgemm_kernel_L1_M4_22) -.Lcgemm_kernel_L1_M4_40: +L(cgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_M4_100 + ble L(cgemm_kernel_L1_M4_100) -.Lcgemm_kernel_L1_M4_42: +L(cgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M4_42 + bgt L(cgemm_kernel_L1_M4_42) -.Lcgemm_kernel_L1_M4_100: +L(cgemm_kernel_L1_M4_100): SAVE4x1 -.Lcgemm_kernel_L1_M4_END: +L(cgemm_kernel_L1_M4_END): -.Lcgemm_kernel_L1_M2_BEGIN: +L(cgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lcgemm_kernel_L1_END + ble L(cgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lcgemm_kernel_L1_M1_BEGIN + ble L(cgemm_kernel_L1_M1_BEGIN) -.Lcgemm_kernel_L1_M2_20: +L(cgemm_kernel_L1_M2_20): INIT2x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_M2_40 + ble L(cgemm_kernel_L1_M2_40) -.Lcgemm_kernel_L1_M2_22: +L(cgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -2084,43 +2084,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M2_22 + bgt L(cgemm_kernel_L1_M2_22) -.Lcgemm_kernel_L1_M2_40: +L(cgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_M2_100 + ble L(cgemm_kernel_L1_M2_100) -.Lcgemm_kernel_L1_M2_42: +L(cgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M2_42 + bgt L(cgemm_kernel_L1_M2_42) -.Lcgemm_kernel_L1_M2_100: +L(cgemm_kernel_L1_M2_100): SAVE2x1 -.Lcgemm_kernel_L1_M2_END: +L(cgemm_kernel_L1_M2_END): -.Lcgemm_kernel_L1_M1_BEGIN: +L(cgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lcgemm_kernel_L1_END + ble L(cgemm_kernel_L1_END) -.Lcgemm_kernel_L1_M1_20: +L(cgemm_kernel_L1_M1_20): INIT1x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_M1_40 + ble L(cgemm_kernel_L1_M1_40) -.Lcgemm_kernel_L1_M1_22: +L(cgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -2132,30 +2132,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M1_22 + bgt L(cgemm_kernel_L1_M1_22) -.Lcgemm_kernel_L1_M1_40: +L(cgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_M1_100 + ble L(cgemm_kernel_L1_M1_100) -.Lcgemm_kernel_L1_M1_42: +L(cgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_M1_42 + bgt L(cgemm_kernel_L1_M1_42) -.Lcgemm_kernel_L1_M1_100: +L(cgemm_kernel_L1_M1_100): SAVE1x1 -.Lcgemm_kernel_L1_END: +L(cgemm_kernel_L1_END): -.Lcgemm_kernel_L999: +L(cgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/cgemm_kernel_sve_v1x4.S b/kernel/arm64/cgemm_kernel_sve_v1x4.S index 2136ebbeee..f8fbf398f6 100644 --- a/kernel/arm64/cgemm_kernel_sve_v1x4.S +++ b/kernel/arm64/cgemm_kernel_sve_v1x4.S @@ -535,10 +535,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lcgemm_kernel_L2_BEGIN + ble L(cgemm_kernel_L2_BEGIN) /******************************************************************************/ -.Lcgemm_kernel_L4_BEGIN: +L(cgemm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -548,7 +548,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lcgemm_kernel_L4_Mv1_BEGIN: +L(cgemm_kernel_L4_Mv1_BEGIN): /* Loop over M is done in an SVE fashion. This has the benefit of the last M%SVE_LEN iterations being done in a single sweep */ mov counterI, #0 @@ -556,14 +556,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension .align 5 -.Lcgemm_kernel_L4_Mv1_20: +L(cgemm_kernel_L4_Mv1_20): mov pB, origPB INITv1x4 // fill with zeros asr counterL , origK, #3 cmp counterL , #2 - blt .Lcgemm_kernel_L4_Mv1_32 + blt L(cgemm_kernel_L4_Mv1_32) KERNELv1x4_I KERNELv1x4_M2 @@ -575,10 +575,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lcgemm_kernel_L4_Mv1_22a + ble L(cgemm_kernel_L4_Mv1_22a) .align 5 -.Lcgemm_kernel_L4_Mv1_22: +L(cgemm_kernel_L4_Mv1_22): KERNELv1x4_M1 KERNELv1x4_M2 @@ -590,10 +590,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M2 subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L4_Mv1_22 + bgt L(cgemm_kernel_L4_Mv1_22) .align 5 -.Lcgemm_kernel_L4_Mv1_22a: +L(cgemm_kernel_L4_Mv1_22a): KERNELv1x4_M1 KERNELv1x4_M2 @@ -604,13 +604,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M1 KERNELv1x4_E - b .Lcgemm_kernel_L4_Mv1_44 + b L(cgemm_kernel_L4_Mv1_44) .align 5 -.Lcgemm_kernel_L4_Mv1_32: +L(cgemm_kernel_L4_Mv1_32): tst counterL, #1 - ble .Lcgemm_kernel_L4_Mv1_40 + ble L(cgemm_kernel_L4_Mv1_40) KERNELv1x4_I KERNELv1x4_M2 @@ -621,56 +621,56 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M1 KERNELv1x4_E - b .Lcgemm_kernel_L4_Mv1_44 + b L(cgemm_kernel_L4_Mv1_44) -.Lcgemm_kernel_L4_Mv1_40: +L(cgemm_kernel_L4_Mv1_40): INITv1x4 -.Lcgemm_kernel_L4_Mv1_44: +L(cgemm_kernel_L4_Mv1_44): ands counterL , origK, #7 - ble .Lcgemm_kernel_L4_Mv1_100 + ble L(cgemm_kernel_L4_Mv1_100) .align 5 -.Lcgemm_kernel_L4_Mv1_46: +L(cgemm_kernel_L4_Mv1_46): KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Lcgemm_kernel_L4_Mv1_46 + bne L(cgemm_kernel_L4_Mv1_46) -.Lcgemm_kernel_L4_Mv1_100: +L(cgemm_kernel_L4_Mv1_100): SAVEv1x4 -.Lcgemm_kernel_L4_Mv1_END: +L(cgemm_kernel_L4_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension - b.any .Lcgemm_kernel_L4_Mv1_20 + b.any L(cgemm_kernel_L4_Mv1_20) -.Lcgemm_kernel_L4_END: +L(cgemm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 4 * 2 subs counterJ, counterJ , #1 // j-- - bgt .Lcgemm_kernel_L4_BEGIN + bgt L(cgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Lcgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(cgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lcgemm_kernel_L999 + ble L(cgemm_kernel_L999) tst counterJ , #2 - ble .Lcgemm_kernel_L1_BEGIN + ble L(cgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC add pCRow1, pCRow0, LDC @@ -681,24 +681,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lcgemm_kernel_L2_Mv1_BEGIN: +L(cgemm_kernel_L2_Mv1_BEGIN): mov counterI, #0 whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s -.Lcgemm_kernel_L2_Mv1_20: +L(cgemm_kernel_L2_Mv1_20): INITv1x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lcgemm_kernel_L2_Mv1_40 + ble L(cgemm_kernel_L2_Mv1_40) .align 5 -.Lcgemm_kernel_L2_Mv1_22: +L(cgemm_kernel_L2_Mv1_22): KERNELv1x2_SUB KERNELv1x2_SUB KERNELv1x2_SUB @@ -710,45 +710,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_Mv1_22 + bgt L(cgemm_kernel_L2_Mv1_22) -.Lcgemm_kernel_L2_Mv1_40: +L(cgemm_kernel_L2_Mv1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L2_Mv1_100 + ble L(cgemm_kernel_L2_Mv1_100) -.Lcgemm_kernel_L2_Mv1_42: +L(cgemm_kernel_L2_Mv1_42): KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L2_Mv1_42 + bgt L(cgemm_kernel_L2_Mv1_42) -.Lcgemm_kernel_L2_Mv1_100: +L(cgemm_kernel_L2_Mv1_100): SAVEv1x2 -.Lcgemm_kernel_L2_Mv1_END: +L(cgemm_kernel_L2_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lcgemm_kernel_L2_Mv1_20 + b.any L(cgemm_kernel_L2_Mv1_20) -.Lcgemm_kernel_L2_END: +L(cgemm_kernel_L2_END): lsl temp, origK, #4 add origPB, origPB, temp // B = B + K * 2 * 4 * 2 /******************************************************************************/ -.Lcgemm_kernel_L1_BEGIN: +L(cgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lcgemm_kernel_L999 // done + ble L(cgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -756,24 +756,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lcgemm_kernel_L1_Mv1_BEGIN: +L(cgemm_kernel_L1_Mv1_BEGIN): mov counterI, #0 whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s -.Lcgemm_kernel_L1_Mv1_20: +L(cgemm_kernel_L1_Mv1_20): INITv1x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lcgemm_kernel_L1_Mv1_40 + ble L(cgemm_kernel_L1_Mv1_40) .align 5 -.Lcgemm_kernel_L1_Mv1_22: +L(cgemm_kernel_L1_Mv1_22): KERNELv1x1_SUB KERNELv1x1_SUB KERNELv1x1_SUB @@ -785,37 +785,37 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_Mv1_22 + bgt L(cgemm_kernel_L1_Mv1_22) -.Lcgemm_kernel_L1_Mv1_40: +L(cgemm_kernel_L1_Mv1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lcgemm_kernel_L1_Mv1_100 + ble L(cgemm_kernel_L1_Mv1_100) -.Lcgemm_kernel_L1_Mv1_42: +L(cgemm_kernel_L1_Mv1_42): KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lcgemm_kernel_L1_Mv1_42 + bgt L(cgemm_kernel_L1_Mv1_42) -.Lcgemm_kernel_L1_Mv1_100: +L(cgemm_kernel_L1_Mv1_100): SAVEv1x1 -.Lcgemm_kernel_L1_Mv1_END: +L(cgemm_kernel_L1_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lcgemm_kernel_L1_Mv1_20 + b.any L(cgemm_kernel_L1_Mv1_20) -.Lcgemm_kernel_L1_END: +L(cgemm_kernel_L1_END): /******************************************************************************/ -.Lcgemm_kernel_L999: +L(cgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/copy.S b/kernel/arm64/copy.S index b8c6bfcd42..c04a95448d 100644 --- a/kernel/arm64/copy.S +++ b/kernel/arm64/copy.S @@ -159,50 +159,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Lcopy_kernel_L999 + ble L(copy_kernel_L999) cmp INC_X, #1 - bne .Lcopy_kernel_S_BEGIN + bne L(copy_kernel_S_BEGIN) cmp INC_Y, #1 - bne .Lcopy_kernel_S_BEGIN + bne L(copy_kernel_S_BEGIN) -.Lcopy_kernel_F_BEGIN: +L(copy_kernel_F_BEGIN): asr I, N, #2 cmp I, xzr - beq .Lcopy_kernel_F1 + beq L(copy_kernel_F1) -.Lcopy_kernel_F4: +L(copy_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lcopy_kernel_F4 + bne L(copy_kernel_F4) -.Lcopy_kernel_F1: +L(copy_kernel_F1): ands I, N, #3 - ble .Lcopy_kernel_L999 + ble L(copy_kernel_L999) -.Lcopy_kernel_F10: +L(copy_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lcopy_kernel_F10 + bne L(copy_kernel_F10) mov w0, wzr ret -.Lcopy_kernel_S_BEGIN: +L(copy_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lcopy_kernel_S1 + ble L(copy_kernel_S1) -.Lcopy_kernel_S4: +L(copy_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -210,21 +210,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lcopy_kernel_S4 + bne L(copy_kernel_S4) -.Lcopy_kernel_S1: +L(copy_kernel_S1): ands I, N, #3 - ble .Lcopy_kernel_L999 + ble L(copy_kernel_L999) -.Lcopy_kernel_S10: +L(copy_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lcopy_kernel_S10 + bne L(copy_kernel_S10) -.Lcopy_kernel_L999: +L(copy_kernel_L999): mov w0, wzr ret diff --git a/kernel/arm64/csum.S b/kernel/arm64/csum.S index 90746bc392..a5aba37fb3 100644 --- a/kernel/arm64/csum.S +++ b/kernel/arm64/csum.S @@ -92,52 +92,52 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. fmov s1, SUMF cmp N, xzr - ble .Lcsum_kernel_L999 + ble L(csum_kernel_L999) cmp INC_X, xzr - ble .Lcsum_kernel_L999 + ble L(csum_kernel_L999) cmp INC_X, #1 - bne .Lcsum_kernel_S_BEGIN + bne L(csum_kernel_S_BEGIN) -.Lcsum_kernel_F_BEGIN: +L(csum_kernel_F_BEGIN): asr I, N, #3 cmp I, xzr - beq .Lcsum_kernel_F1 + beq L(csum_kernel_F1) -.Lcsum_kernel_F8: +L(csum_kernel_F8): KERNEL_F8 subs I, I, #1 - bne .Lcsum_kernel_F8 + bne L(csum_kernel_F8) KERNEL_F8_FINALIZE -.Lcsum_kernel_F1: +L(csum_kernel_F1): ands I, N, #7 - ble .Lcsum_kernel_L999 + ble L(csum_kernel_L999) -.Lcsum_kernel_F10: +L(csum_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lcsum_kernel_F10 + bne L(csum_kernel_F10) -.Lcsum_kernel_L999: +L(csum_kernel_L999): ret -.Lcsum_kernel_S_BEGIN: +L(csum_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lcsum_kernel_S1 + ble L(csum_kernel_S1) -.Lcsum_kernel_S4: +L(csum_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -145,19 +145,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lcsum_kernel_S4 + bne L(csum_kernel_S4) -.Lcsum_kernel_S1: +L(csum_kernel_S1): ands I, N, #3 - ble .Lcsum_kernel_L999 + ble L(csum_kernel_L999) -.Lcsum_kernel_S10: +L(csum_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lcsum_kernel_S10 + bne L(csum_kernel_S10) ret diff --git a/kernel/arm64/ctrmm_kernel_4x4.S b/kernel/arm64/ctrmm_kernel_4x4.S index 79d33e93c0..edca492dc9 100644 --- a/kernel/arm64/ctrmm_kernel_4x4.S +++ b/kernel/arm64/ctrmm_kernel_4x4.S @@ -785,11 +785,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lctrmm_kernel_L2_BEGIN + ble L(ctrmm_kernel_L2_BEGIN) /******************************************************************************/ -.Lctrmm_kernel_L4_BEGIN: +L(ctrmm_kernel_L4_BEGIN): mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #2 @@ -798,14 +798,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = start of A array -.Lctrmm_kernel_L4_M4_BEGIN: +L(ctrmm_kernel_L4_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lctrmm_kernel_L4_M2_BEGIN + ble L(ctrmm_kernel_L4_M2_BEGIN) -.Lctrmm_kernel_L4_M4_20: +L(ctrmm_kernel_L4_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -826,55 +826,55 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lctrmm_kernel_L4_M4_32 + blt L(ctrmm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lctrmm_kernel_L4_M4_22a + ble L(ctrmm_kernel_L4_M4_22a) .align 5 -.Lctrmm_kernel_L4_M4_22: +L(ctrmm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_M4_22 + bgt L(ctrmm_kernel_L4_M4_22) -.Lctrmm_kernel_L4_M4_22a: +L(ctrmm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lctrmm_kernel_L4_M4_44 + b L(ctrmm_kernel_L4_M4_44) -.Lctrmm_kernel_L4_M4_32: +L(ctrmm_kernel_L4_M4_32): tst counterL, #1 - ble .Lctrmm_kernel_L4_M4_40 + ble L(ctrmm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lctrmm_kernel_L4_M4_44 + b L(ctrmm_kernel_L4_M4_44) -.Lctrmm_kernel_L4_M4_40: +L(ctrmm_kernel_L4_M4_40): INIT4x4 -.Lctrmm_kernel_L4_M4_44: +L(ctrmm_kernel_L4_M4_44): ands counterL , tempK, #1 - ble .Lctrmm_kernel_L4_M4_100 + ble L(ctrmm_kernel_L4_M4_100) -.Lctrmm_kernel_L4_M4_46: +L(ctrmm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lctrmm_kernel_L4_M4_100: +L(ctrmm_kernel_L4_M4_100): SAVE4x4 @@ -893,20 +893,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lctrmm_kernel_L4_M4_END: +L(ctrmm_kernel_L4_M4_END): subs counterI, counterI, #1 - bne .Lctrmm_kernel_L4_M4_20 + bne L(ctrmm_kernel_L4_M4_20) -.Lctrmm_kernel_L4_M2_BEGIN: +L(ctrmm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lctrmm_kernel_L4_END + ble L(ctrmm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lctrmm_kernel_L4_M1_BEGIN + ble L(ctrmm_kernel_L4_M1_BEGIN) -.Lctrmm_kernel_L4_M2_20: +L(ctrmm_kernel_L4_M2_20): INIT2x4 @@ -930,9 +930,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L4_M2_40 + ble L(ctrmm_kernel_L4_M2_40) -.Lctrmm_kernel_L4_M2_22: +L(ctrmm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -945,22 +945,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_M2_22 + bgt L(ctrmm_kernel_L4_M2_22) -.Lctrmm_kernel_L4_M2_40: +L(ctrmm_kernel_L4_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L4_M2_100 + ble L(ctrmm_kernel_L4_M2_100) -.Lctrmm_kernel_L4_M2_42: +L(ctrmm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_M2_42 + bgt L(ctrmm_kernel_L4_M2_42) -.Lctrmm_kernel_L4_M2_100: +L(ctrmm_kernel_L4_M2_100): SAVE2x4 @@ -980,15 +980,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lctrmm_kernel_L4_M2_END: +L(ctrmm_kernel_L4_M2_END): -.Lctrmm_kernel_L4_M1_BEGIN: +L(ctrmm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lctrmm_kernel_L4_END + ble L(ctrmm_kernel_L4_END) -.Lctrmm_kernel_L4_M1_20: +L(ctrmm_kernel_L4_M1_20): INIT1x4 @@ -1012,9 +1012,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L4_M1_40 + ble L(ctrmm_kernel_L4_M1_40) -.Lctrmm_kernel_L4_M1_22: +L(ctrmm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1026,22 +1026,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_M1_22 + bgt L(ctrmm_kernel_L4_M1_22) -.Lctrmm_kernel_L4_M1_40: +L(ctrmm_kernel_L4_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L4_M1_100 + ble L(ctrmm_kernel_L4_M1_100) -.Lctrmm_kernel_L4_M1_42: +L(ctrmm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_M1_42 + bgt L(ctrmm_kernel_L4_M1_42) -.Lctrmm_kernel_L4_M1_100: +L(ctrmm_kernel_L4_M1_100): SAVE1x4 @@ -1061,7 +1061,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Lctrmm_kernel_L4_END: +L(ctrmm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 @@ -1071,19 +1071,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Lctrmm_kernel_L4_BEGIN + bgt L(ctrmm_kernel_L4_BEGIN) /******************************************************************************/ -.Lctrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(ctrmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lctrmm_kernel_L999 // error, N was less than 4? + ble L(ctrmm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Lctrmm_kernel_L1_BEGIN + ble L(ctrmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1095,14 +1095,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lctrmm_kernel_L2_M4_BEGIN: +L(ctrmm_kernel_L2_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI,#0 - ble .Lctrmm_kernel_L2_M2_BEGIN + ble L(ctrmm_kernel_L2_M2_BEGIN) -.Lctrmm_kernel_L2_M4_20: +L(ctrmm_kernel_L2_M4_20): INIT4x2 @@ -1126,10 +1126,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lctrmm_kernel_L2_M4_40 + ble L(ctrmm_kernel_L2_M4_40) .align 5 -.Lctrmm_kernel_L2_M4_22: +L(ctrmm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1141,22 +1141,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M4_22 + bgt L(ctrmm_kernel_L2_M4_22) -.Lctrmm_kernel_L2_M4_40: +L(ctrmm_kernel_L2_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L2_M4_100 + ble L(ctrmm_kernel_L2_M4_100) -.Lctrmm_kernel_L2_M4_42: +L(ctrmm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M4_42 + bgt L(ctrmm_kernel_L2_M4_42) -.Lctrmm_kernel_L2_M4_100: +L(ctrmm_kernel_L2_M4_100): SAVE4x2 @@ -1176,22 +1176,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lctrmm_kernel_L2_M4_END: +L(ctrmm_kernel_L2_M4_END): subs counterI, counterI, #1 - bgt .Lctrmm_kernel_L2_M4_20 + bgt L(ctrmm_kernel_L2_M4_20) -.Lctrmm_kernel_L2_M2_BEGIN: +L(ctrmm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lctrmm_kernel_L2_END + ble L(ctrmm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lctrmm_kernel_L2_M1_BEGIN + ble L(ctrmm_kernel_L2_M1_BEGIN) -.Lctrmm_kernel_L2_M2_20: +L(ctrmm_kernel_L2_M2_20): INIT2x2 @@ -1215,9 +1215,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lctrmm_kernel_L2_M2_40 + ble L(ctrmm_kernel_L2_M2_40) -.Lctrmm_kernel_L2_M2_22: +L(ctrmm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1230,22 +1230,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M2_22 + bgt L(ctrmm_kernel_L2_M2_22) -.Lctrmm_kernel_L2_M2_40: +L(ctrmm_kernel_L2_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L2_M2_100 + ble L(ctrmm_kernel_L2_M2_100) -.Lctrmm_kernel_L2_M2_42: +L(ctrmm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M2_42 + bgt L(ctrmm_kernel_L2_M2_42) -.Lctrmm_kernel_L2_M2_100: +L(ctrmm_kernel_L2_M2_100): SAVE2x2 @@ -1265,15 +1265,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lctrmm_kernel_L2_M2_END: +L(ctrmm_kernel_L2_M2_END): -.Lctrmm_kernel_L2_M1_BEGIN: +L(ctrmm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lctrmm_kernel_L2_END + ble L(ctrmm_kernel_L2_END) -.Lctrmm_kernel_L2_M1_20: +L(ctrmm_kernel_L2_M1_20): INIT1x2 @@ -1297,9 +1297,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lctrmm_kernel_L2_M1_40 + ble L(ctrmm_kernel_L2_M1_40) -.Lctrmm_kernel_L2_M1_22: +L(ctrmm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1311,22 +1311,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M1_22 + bgt L(ctrmm_kernel_L2_M1_22) -.Lctrmm_kernel_L2_M1_40: +L(ctrmm_kernel_L2_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L2_M1_100 + ble L(ctrmm_kernel_L2_M1_100) -.Lctrmm_kernel_L2_M1_42: +L(ctrmm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M1_42 + bgt L(ctrmm_kernel_L2_M1_42) -.Lctrmm_kernel_L2_M1_100: +L(ctrmm_kernel_L2_M1_100): SAVE1x2 @@ -1346,7 +1346,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Lctrmm_kernel_L2_END: +L(ctrmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -1354,11 +1354,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lctrmm_kernel_L1_BEGIN: +L(ctrmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lctrmm_kernel_L999 // done + ble L(ctrmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1370,14 +1370,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lctrmm_kernel_L1_M4_BEGIN: +L(ctrmm_kernel_L1_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lctrmm_kernel_L1_M2_BEGIN + ble L(ctrmm_kernel_L1_M2_BEGIN) -.Lctrmm_kernel_L1_M4_20: +L(ctrmm_kernel_L1_M4_20): INIT4x1 @@ -1401,10 +1401,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L1_M4_40 + ble L(ctrmm_kernel_L1_M4_40) .align 5 -.Lctrmm_kernel_L1_M4_22: +L(ctrmm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1416,22 +1416,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M4_22 + bgt L(ctrmm_kernel_L1_M4_22) -.Lctrmm_kernel_L1_M4_40: +L(ctrmm_kernel_L1_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L1_M4_100 + ble L(ctrmm_kernel_L1_M4_100) -.Lctrmm_kernel_L1_M4_42: +L(ctrmm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M4_42 + bgt L(ctrmm_kernel_L1_M4_42) -.Lctrmm_kernel_L1_M4_100: +L(ctrmm_kernel_L1_M4_100): SAVE4x1 @@ -1451,22 +1451,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lctrmm_kernel_L1_M4_END: +L(ctrmm_kernel_L1_M4_END): subs counterI, counterI, #1 - bgt .Lctrmm_kernel_L1_M4_20 + bgt L(ctrmm_kernel_L1_M4_20) -.Lctrmm_kernel_L1_M2_BEGIN: +L(ctrmm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lctrmm_kernel_L1_END + ble L(ctrmm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lctrmm_kernel_L1_M1_BEGIN + ble L(ctrmm_kernel_L1_M1_BEGIN) -.Lctrmm_kernel_L1_M2_20: +L(ctrmm_kernel_L1_M2_20): INIT2x1 @@ -1490,9 +1490,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L1_M2_40 + ble L(ctrmm_kernel_L1_M2_40) -.Lctrmm_kernel_L1_M2_22: +L(ctrmm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1505,22 +1505,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M2_22 + bgt L(ctrmm_kernel_L1_M2_22) -.Lctrmm_kernel_L1_M2_40: +L(ctrmm_kernel_L1_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L1_M2_100 + ble L(ctrmm_kernel_L1_M2_100) -.Lctrmm_kernel_L1_M2_42: +L(ctrmm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M2_42 + bgt L(ctrmm_kernel_L1_M2_42) -.Lctrmm_kernel_L1_M2_100: +L(ctrmm_kernel_L1_M2_100): SAVE2x1 @@ -1540,15 +1540,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lctrmm_kernel_L1_M2_END: +L(ctrmm_kernel_L1_M2_END): -.Lctrmm_kernel_L1_M1_BEGIN: +L(ctrmm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lctrmm_kernel_L1_END + ble L(ctrmm_kernel_L1_END) -.Lctrmm_kernel_L1_M1_20: +L(ctrmm_kernel_L1_M1_20): INIT1x1 @@ -1572,9 +1572,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L1_M1_40 + ble L(ctrmm_kernel_L1_M1_40) -.Lctrmm_kernel_L1_M1_22: +L(ctrmm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1586,30 +1586,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M1_22 + bgt L(ctrmm_kernel_L1_M1_22) -.Lctrmm_kernel_L1_M1_40: +L(ctrmm_kernel_L1_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L1_M1_100 + ble L(ctrmm_kernel_L1_M1_100) -.Lctrmm_kernel_L1_M1_42: +L(ctrmm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M1_42 + bgt L(ctrmm_kernel_L1_M1_42) -.Lctrmm_kernel_L1_M1_100: +L(ctrmm_kernel_L1_M1_100): SAVE1x1 -.Lctrmm_kernel_L1_END: +L(ctrmm_kernel_L1_END): -.Lctrmm_kernel_L999: +L(ctrmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/ctrmm_kernel_8x4.S b/kernel/arm64/ctrmm_kernel_8x4.S index e8f1d8cf30..417273a6e3 100644 --- a/kernel/arm64/ctrmm_kernel_8x4.S +++ b/kernel/arm64/ctrmm_kernel_8x4.S @@ -1405,11 +1405,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lctrmm_kernel_L2_BEGIN + ble L(ctrmm_kernel_L2_BEGIN) /******************************************************************************/ -.Lctrmm_kernel_L4_BEGIN: +L(ctrmm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -1423,14 +1423,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = start of A array -.Lctrmm_kernel_L4_M8_BEGIN: +L(ctrmm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lctrmm_kernel_L4_M4_BEGIN + ble L(ctrmm_kernel_L4_M4_BEGIN) -.Lctrmm_kernel_L4_M8_20: +L(ctrmm_kernel_L4_M8_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1452,7 +1452,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 cmp counterL , #2 - blt .Lctrmm_kernel_L4_M8_32 + blt L(ctrmm_kernel_L4_M8_32) KERNEL8x4_I KERNEL8x4_M2 @@ -1464,10 +1464,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lctrmm_kernel_L4_M8_22a + ble L(ctrmm_kernel_L4_M8_22a) .align 5 -.Lctrmm_kernel_L4_M8_22: +L(ctrmm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 @@ -1479,10 +1479,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_M8_22 + bgt L(ctrmm_kernel_L4_M8_22) .align 5 -.Lctrmm_kernel_L4_M8_22a: +L(ctrmm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_M2 @@ -1493,13 +1493,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Lctrmm_kernel_L4_M8_44 + b L(ctrmm_kernel_L4_M8_44) .align 5 -.Lctrmm_kernel_L4_M8_32: +L(ctrmm_kernel_L4_M8_32): tst counterL, #1 - ble .Lctrmm_kernel_L4_M8_40 + ble L(ctrmm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_M2 @@ -1510,26 +1510,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Lctrmm_kernel_L4_M8_44 + b L(ctrmm_kernel_L4_M8_44) -.Lctrmm_kernel_L4_M8_40: +L(ctrmm_kernel_L4_M8_40): INIT8x4 -.Lctrmm_kernel_L4_M8_44: +L(ctrmm_kernel_L4_M8_44): ands counterL , tempK, #7 - ble .Lctrmm_kernel_L4_M8_100 + ble L(ctrmm_kernel_L4_M8_100) .align 5 -.Lctrmm_kernel_L4_M8_46: +L(ctrmm_kernel_L4_M8_46): KERNEL8x4_SUB subs counterL, counterL, #1 - bne .Lctrmm_kernel_L4_M8_46 + bne L(ctrmm_kernel_L4_M8_46) -.Lctrmm_kernel_L4_M8_100: +L(ctrmm_kernel_L4_M8_100): SAVE8x4 @@ -1552,21 +1552,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] -.Lctrmm_kernel_L4_M8_END: +L(ctrmm_kernel_L4_M8_END): subs counterI, counterI, #1 - bne .Lctrmm_kernel_L4_M8_20 + bne L(ctrmm_kernel_L4_M8_20) -.Lctrmm_kernel_L4_M4_BEGIN: +L(ctrmm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lctrmm_kernel_L4_END + ble L(ctrmm_kernel_L4_END) tst counterI, #4 - ble .Lctrmm_kernel_L4_M2_BEGIN + ble L(ctrmm_kernel_L4_M2_BEGIN) -.Lctrmm_kernel_L4_M4_20: +L(ctrmm_kernel_L4_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1587,46 +1587,46 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lctrmm_kernel_L4_M4_32 + blt L(ctrmm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lctrmm_kernel_L4_M4_22a + ble L(ctrmm_kernel_L4_M4_22a) .align 5 -.Lctrmm_kernel_L4_M4_22: +L(ctrmm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_M4_22 + bgt L(ctrmm_kernel_L4_M4_22) -.Lctrmm_kernel_L4_M4_22a: +L(ctrmm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lctrmm_kernel_L4_M4_44 -.Lctrmm_kernel_L4_M4_32: + b L(ctrmm_kernel_L4_M4_44) +L(ctrmm_kernel_L4_M4_32): tst counterL, #1 - ble .Lctrmm_kernel_L4_M4_40 + ble L(ctrmm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lctrmm_kernel_L4_M4_44 -.Lctrmm_kernel_L4_M4_40: + b L(ctrmm_kernel_L4_M4_44) +L(ctrmm_kernel_L4_M4_40): INIT4x4 -.Lctrmm_kernel_L4_M4_44: +L(ctrmm_kernel_L4_M4_44): ands counterL , tempK, #1 - ble .Lctrmm_kernel_L4_M4_100 + ble L(ctrmm_kernel_L4_M4_100) -.Lctrmm_kernel_L4_M4_46: +L(ctrmm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lctrmm_kernel_L4_M4_100: +L(ctrmm_kernel_L4_M4_100): SAVE4x4 @@ -1645,18 +1645,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lctrmm_kernel_L4_M4_END: +L(ctrmm_kernel_L4_M4_END): -.Lctrmm_kernel_L4_M2_BEGIN: +L(ctrmm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lctrmm_kernel_L4_END + ble L(ctrmm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lctrmm_kernel_L4_M1_BEGIN + ble L(ctrmm_kernel_L4_M1_BEGIN) -.Lctrmm_kernel_L4_M2_20: +L(ctrmm_kernel_L4_M2_20): INIT2x4 @@ -1679,9 +1679,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L4_M2_40 + ble L(ctrmm_kernel_L4_M2_40) -.Lctrmm_kernel_L4_M2_22: +L(ctrmm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1694,22 +1694,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_M2_22 + bgt L(ctrmm_kernel_L4_M2_22) -.Lctrmm_kernel_L4_M2_40: +L(ctrmm_kernel_L4_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L4_M2_100 + ble L(ctrmm_kernel_L4_M2_100) -.Lctrmm_kernel_L4_M2_42: +L(ctrmm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_M2_42 + bgt L(ctrmm_kernel_L4_M2_42) -.Lctrmm_kernel_L4_M2_100: +L(ctrmm_kernel_L4_M2_100): SAVE2x4 @@ -1729,15 +1729,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lctrmm_kernel_L4_M2_END: +L(ctrmm_kernel_L4_M2_END): -.Lctrmm_kernel_L4_M1_BEGIN: +L(ctrmm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lctrmm_kernel_L4_END + ble L(ctrmm_kernel_L4_END) -.Lctrmm_kernel_L4_M1_20: +L(ctrmm_kernel_L4_M1_20): INIT1x4 @@ -1761,9 +1761,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L4_M1_40 + ble L(ctrmm_kernel_L4_M1_40) -.Lctrmm_kernel_L4_M1_22: +L(ctrmm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1775,22 +1775,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_M1_22 + bgt L(ctrmm_kernel_L4_M1_22) -.Lctrmm_kernel_L4_M1_40: +L(ctrmm_kernel_L4_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L4_M1_100 + ble L(ctrmm_kernel_L4_M1_100) -.Lctrmm_kernel_L4_M1_42: +L(ctrmm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_M1_42 + bgt L(ctrmm_kernel_L4_M1_42) -.Lctrmm_kernel_L4_M1_100: +L(ctrmm_kernel_L4_M1_100): SAVE1x4 @@ -1810,7 +1810,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Lctrmm_kernel_L4_END: +L(ctrmm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 @@ -1820,19 +1820,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Lctrmm_kernel_L4_BEGIN + bgt L(ctrmm_kernel_L4_BEGIN) /******************************************************************************/ -.Lctrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(ctrmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lctrmm_kernel_L999 // error, N was less than 4? + ble L(ctrmm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Lctrmm_kernel_L1_BEGIN + ble L(ctrmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1843,14 +1843,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = A -.Lctrmm_kernel_L2_M8_BEGIN: +L(ctrmm_kernel_L2_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lctrmm_kernel_L2_M4_BEGIN + ble L(ctrmm_kernel_L2_M4_BEGIN) -.Lctrmm_kernel_L2_M8_20: +L(ctrmm_kernel_L2_M8_20): INIT8x2 @@ -1874,10 +1874,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lctrmm_kernel_L2_M8_40 + ble L(ctrmm_kernel_L2_M8_40) .align 5 -.Lctrmm_kernel_L2_M8_22: +L(ctrmm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB KERNEL8x2_SUB @@ -1889,22 +1889,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M8_22 + bgt L(ctrmm_kernel_L2_M8_22) -.Lctrmm_kernel_L2_M8_40: +L(ctrmm_kernel_L2_M8_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L2_M8_100 + ble L(ctrmm_kernel_L2_M8_100) -.Lctrmm_kernel_L2_M8_42: +L(ctrmm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M8_42 + bgt L(ctrmm_kernel_L2_M8_42) -.Lctrmm_kernel_L2_M8_100: +L(ctrmm_kernel_L2_M8_100): SAVE8x2 @@ -1924,21 +1924,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #8 #endif -.Lctrmm_kernel_L2_M8_END: +L(ctrmm_kernel_L2_M8_END): subs counterI, counterI, #1 - bgt .Lctrmm_kernel_L2_M8_20 + bgt L(ctrmm_kernel_L2_M8_20) -.Lctrmm_kernel_L2_M4_BEGIN: +L(ctrmm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lctrmm_kernel_L2_END + ble L(ctrmm_kernel_L2_END) tst counterI, #4 // counterI = counterI / 2 - ble .Lctrmm_kernel_L2_M2_BEGIN + ble L(ctrmm_kernel_L2_M2_BEGIN) -.Lctrmm_kernel_L2_M4_20: +L(ctrmm_kernel_L2_M4_20): INIT4x2 @@ -1962,10 +1962,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lctrmm_kernel_L2_M4_40 + ble L(ctrmm_kernel_L2_M4_40) .align 5 -.Lctrmm_kernel_L2_M4_22: +L(ctrmm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1977,22 +1977,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M4_22 + bgt L(ctrmm_kernel_L2_M4_22) -.Lctrmm_kernel_L2_M4_40: +L(ctrmm_kernel_L2_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L2_M4_100 + ble L(ctrmm_kernel_L2_M4_100) -.Lctrmm_kernel_L2_M4_42: +L(ctrmm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M4_42 + bgt L(ctrmm_kernel_L2_M4_42) -.Lctrmm_kernel_L2_M4_100: +L(ctrmm_kernel_L2_M4_100): SAVE4x2 @@ -2012,19 +2012,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lctrmm_kernel_L2_M4_END: +L(ctrmm_kernel_L2_M4_END): -.Lctrmm_kernel_L2_M2_BEGIN: +L(ctrmm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lctrmm_kernel_L2_END + ble L(ctrmm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lctrmm_kernel_L2_M1_BEGIN + ble L(ctrmm_kernel_L2_M1_BEGIN) -.Lctrmm_kernel_L2_M2_20: +L(ctrmm_kernel_L2_M2_20): INIT2x2 @@ -2048,9 +2048,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lctrmm_kernel_L2_M2_40 + ble L(ctrmm_kernel_L2_M2_40) -.Lctrmm_kernel_L2_M2_22: +L(ctrmm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -2063,22 +2063,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M2_22 + bgt L(ctrmm_kernel_L2_M2_22) -.Lctrmm_kernel_L2_M2_40: +L(ctrmm_kernel_L2_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L2_M2_100 + ble L(ctrmm_kernel_L2_M2_100) -.Lctrmm_kernel_L2_M2_42: +L(ctrmm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M2_42 + bgt L(ctrmm_kernel_L2_M2_42) -.Lctrmm_kernel_L2_M2_100: +L(ctrmm_kernel_L2_M2_100): SAVE2x2 @@ -2098,15 +2098,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lctrmm_kernel_L2_M2_END: +L(ctrmm_kernel_L2_M2_END): -.Lctrmm_kernel_L2_M1_BEGIN: +L(ctrmm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lctrmm_kernel_L2_END + ble L(ctrmm_kernel_L2_END) -.Lctrmm_kernel_L2_M1_20: +L(ctrmm_kernel_L2_M1_20): INIT1x2 @@ -2130,9 +2130,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lctrmm_kernel_L2_M1_40 + ble L(ctrmm_kernel_L2_M1_40) -.Lctrmm_kernel_L2_M1_22: +L(ctrmm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -2144,22 +2144,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M1_22 + bgt L(ctrmm_kernel_L2_M1_22) -.Lctrmm_kernel_L2_M1_40: +L(ctrmm_kernel_L2_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L2_M1_100 + ble L(ctrmm_kernel_L2_M1_100) -.Lctrmm_kernel_L2_M1_42: +L(ctrmm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_M1_42 + bgt L(ctrmm_kernel_L2_M1_42) -.Lctrmm_kernel_L2_M1_100: +L(ctrmm_kernel_L2_M1_100): SAVE1x2 @@ -2179,7 +2179,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Lctrmm_kernel_L2_END: +L(ctrmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -2187,11 +2187,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lctrmm_kernel_L1_BEGIN: +L(ctrmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lctrmm_kernel_L999 // done + ble L(ctrmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C add pC , pC , LDC // Update pC to point to next @@ -2201,14 +2201,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = A -.Lctrmm_kernel_L1_M8_BEGIN: +L(ctrmm_kernel_L1_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lctrmm_kernel_L1_M4_BEGIN + ble L(ctrmm_kernel_L1_M4_BEGIN) -.Lctrmm_kernel_L1_M8_20: +L(ctrmm_kernel_L1_M8_20): INIT8x1 @@ -2232,10 +2232,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L1_M8_40 + ble L(ctrmm_kernel_L1_M8_40) .align 5 -.Lctrmm_kernel_L1_M8_22: +L(ctrmm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -2247,22 +2247,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M8_22 + bgt L(ctrmm_kernel_L1_M8_22) -.Lctrmm_kernel_L1_M8_40: +L(ctrmm_kernel_L1_M8_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L1_M8_100 + ble L(ctrmm_kernel_L1_M8_100) -.Lctrmm_kernel_L1_M8_42: +L(ctrmm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M8_42 + bgt L(ctrmm_kernel_L1_M8_42) -.Lctrmm_kernel_L1_M8_100: +L(ctrmm_kernel_L1_M8_100): SAVE8x1 @@ -2282,21 +2282,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #8 #endif -.Lctrmm_kernel_L1_M8_END: +L(ctrmm_kernel_L1_M8_END): subs counterI, counterI, #1 - bgt .Lctrmm_kernel_L1_M8_20 + bgt L(ctrmm_kernel_L1_M8_20) -.Lctrmm_kernel_L1_M4_BEGIN: +L(ctrmm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lctrmm_kernel_L1_END + ble L(ctrmm_kernel_L1_END) tst counterI, #4 // counterI = counterI / 2 - ble .Lctrmm_kernel_L1_M2_BEGIN + ble L(ctrmm_kernel_L1_M2_BEGIN) -.Lctrmm_kernel_L1_M4_20: +L(ctrmm_kernel_L1_M4_20): INIT4x1 @@ -2319,10 +2319,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L1_M4_40 + ble L(ctrmm_kernel_L1_M4_40) .align 5 -.Lctrmm_kernel_L1_M4_22: +L(ctrmm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -2334,22 +2334,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M4_22 + bgt L(ctrmm_kernel_L1_M4_22) -.Lctrmm_kernel_L1_M4_40: +L(ctrmm_kernel_L1_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L1_M4_100 + ble L(ctrmm_kernel_L1_M4_100) -.Lctrmm_kernel_L1_M4_42: +L(ctrmm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M4_42 + bgt L(ctrmm_kernel_L1_M4_42) -.Lctrmm_kernel_L1_M4_100: +L(ctrmm_kernel_L1_M4_100): SAVE4x1 @@ -2369,18 +2369,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lctrmm_kernel_L1_M4_END: +L(ctrmm_kernel_L1_M4_END): -.Lctrmm_kernel_L1_M2_BEGIN: +L(ctrmm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lctrmm_kernel_L1_END + ble L(ctrmm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lctrmm_kernel_L1_M1_BEGIN + ble L(ctrmm_kernel_L1_M1_BEGIN) -.Lctrmm_kernel_L1_M2_20: +L(ctrmm_kernel_L1_M2_20): INIT2x1 @@ -2404,9 +2404,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L1_M2_40 + ble L(ctrmm_kernel_L1_M2_40) -.Lctrmm_kernel_L1_M2_22: +L(ctrmm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -2419,22 +2419,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M2_22 + bgt L(ctrmm_kernel_L1_M2_22) -.Lctrmm_kernel_L1_M2_40: +L(ctrmm_kernel_L1_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L1_M2_100 + ble L(ctrmm_kernel_L1_M2_100) -.Lctrmm_kernel_L1_M2_42: +L(ctrmm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M2_42 + bgt L(ctrmm_kernel_L1_M2_42) -.Lctrmm_kernel_L1_M2_100: +L(ctrmm_kernel_L1_M2_100): SAVE2x1 @@ -2454,15 +2454,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lctrmm_kernel_L1_M2_END: +L(ctrmm_kernel_L1_M2_END): -.Lctrmm_kernel_L1_M1_BEGIN: +L(ctrmm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lctrmm_kernel_L1_END + ble L(ctrmm_kernel_L1_END) -.Lctrmm_kernel_L1_M1_20: +L(ctrmm_kernel_L1_M1_20): INIT1x1 @@ -2486,9 +2486,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L1_M1_40 + ble L(ctrmm_kernel_L1_M1_40) -.Lctrmm_kernel_L1_M1_22: +L(ctrmm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -2500,30 +2500,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M1_22 + bgt L(ctrmm_kernel_L1_M1_22) -.Lctrmm_kernel_L1_M1_40: +L(ctrmm_kernel_L1_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L1_M1_100 + ble L(ctrmm_kernel_L1_M1_100) -.Lctrmm_kernel_L1_M1_42: +L(ctrmm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_M1_42 + bgt L(ctrmm_kernel_L1_M1_42) -.Lctrmm_kernel_L1_M1_100: +L(ctrmm_kernel_L1_M1_100): SAVE1x1 -.Lctrmm_kernel_L1_END: +L(ctrmm_kernel_L1_END): -.Lctrmm_kernel_L999: +L(ctrmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/ctrmm_kernel_sve_v1x4.S b/kernel/arm64/ctrmm_kernel_sve_v1x4.S index 242968f636..1b7f23a96e 100644 --- a/kernel/arm64/ctrmm_kernel_sve_v1x4.S +++ b/kernel/arm64/ctrmm_kernel_sve_v1x4.S @@ -584,10 +584,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lctrmm_kernel_L2_BEGIN + ble L(ctrmm_kernel_L2_BEGIN) /******************************************************************************/ -.Lctrmm_kernel_L4_BEGIN: +L(ctrmm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -600,7 +600,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = start of A array -.Lctrmm_kernel_L4_Mv1_BEGIN: +L(ctrmm_kernel_L4_Mv1_BEGIN): /* Loop over M is done in an SVE fashion. This has the benefit of the last M%SVE_LEN iterations being done in a single sweep */ mov counterI, #0 @@ -608,7 +608,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension .align 5 -.Lctrmm_kernel_L4_Mv1_20: +L(ctrmm_kernel_L4_Mv1_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -631,7 +631,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 cmp counterL , #2 - blt .Lctrmm_kernel_L4_Mv1_32 + blt L(ctrmm_kernel_L4_Mv1_32) KERNELv1x4_I KERNELv1x4_M2 @@ -643,10 +643,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lctrmm_kernel_L4_Mv1_22a + ble L(ctrmm_kernel_L4_Mv1_22a) .align 5 -.Lctrmm_kernel_L4_Mv1_22: +L(ctrmm_kernel_L4_Mv1_22): KERNELv1x4_M1 KERNELv1x4_M2 @@ -658,10 +658,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M2 subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L4_Mv1_22 + bgt L(ctrmm_kernel_L4_Mv1_22) .align 5 -.Lctrmm_kernel_L4_Mv1_22a: +L(ctrmm_kernel_L4_Mv1_22a): KERNELv1x4_M1 KERNELv1x4_M2 @@ -672,13 +672,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M1 KERNELv1x4_E - b .Lctrmm_kernel_L4_Mv1_44 + b L(ctrmm_kernel_L4_Mv1_44) .align 5 -.Lctrmm_kernel_L4_Mv1_32: +L(ctrmm_kernel_L4_Mv1_32): tst counterL, #1 - ble .Lctrmm_kernel_L4_Mv1_40 + ble L(ctrmm_kernel_L4_Mv1_40) KERNELv1x4_I KERNELv1x4_M2 @@ -689,26 +689,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M1 KERNELv1x4_E - b .Lctrmm_kernel_L4_Mv1_44 + b L(ctrmm_kernel_L4_Mv1_44) -.Lctrmm_kernel_L4_Mv1_40: +L(ctrmm_kernel_L4_Mv1_40): INITv1x4 -.Lctrmm_kernel_L4_Mv1_44: +L(ctrmm_kernel_L4_Mv1_44): ands counterL , tempK, #7 - ble .Lctrmm_kernel_L4_Mv1_100 + ble L(ctrmm_kernel_L4_Mv1_100) .align 5 -.Lctrmm_kernel_L4_Mv1_46: +L(ctrmm_kernel_L4_Mv1_46): KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Lctrmm_kernel_L4_Mv1_46 + bne L(ctrmm_kernel_L4_Mv1_46) -.Lctrmm_kernel_L4_Mv1_100: +L(ctrmm_kernel_L4_Mv1_100): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -732,16 +732,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SAVEv1x4 -.Lctrmm_kernel_L4_Mv1_END: +L(ctrmm_kernel_L4_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension - b.any .Lctrmm_kernel_L4_Mv1_20 + b.any L(ctrmm_kernel_L4_Mv1_20) -.Lctrmm_kernel_L4_END: +L(ctrmm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 * 2 @@ -751,19 +751,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Lctrmm_kernel_L4_BEGIN + bgt L(ctrmm_kernel_L4_BEGIN) /******************************************************************************/ -.Lctrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(ctrmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lctrmm_kernel_L999 + ble L(ctrmm_kernel_L999) tst counterJ , #2 - ble .Lctrmm_kernel_L1_BEGIN + ble L(ctrmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC add pCRow1, pCRow0, LDC @@ -778,14 +778,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lctrmm_kernel_L2_Mv1_BEGIN: +L(ctrmm_kernel_L2_Mv1_BEGIN): mov counterI, #0 whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s -.Lctrmm_kernel_L2_Mv1_20: +L(ctrmm_kernel_L2_Mv1_20): INITv1x2 @@ -809,10 +809,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lctrmm_kernel_L2_Mv1_40 + ble L(ctrmm_kernel_L2_Mv1_40) .align 5 -.Lctrmm_kernel_L2_Mv1_22: +L(ctrmm_kernel_L2_Mv1_22): KERNELv1x2_SUB KERNELv1x2_SUB KERNELv1x2_SUB @@ -824,22 +824,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_Mv1_22 + bgt L(ctrmm_kernel_L2_Mv1_22) -.Lctrmm_kernel_L2_Mv1_40: +L(ctrmm_kernel_L2_Mv1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L2_Mv1_100 + ble L(ctrmm_kernel_L2_Mv1_100) -.Lctrmm_kernel_L2_Mv1_42: +L(ctrmm_kernel_L2_Mv1_42): KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L2_Mv1_42 + bgt L(ctrmm_kernel_L2_Mv1_42) -.Lctrmm_kernel_L2_Mv1_100: +L(ctrmm_kernel_L2_Mv1_100): SAVEv1x2 @@ -859,16 +859,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, lanes #endif -.Lctrmm_kernel_L2_Mv1_END: +L(ctrmm_kernel_L2_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lctrmm_kernel_L2_Mv1_20 + b.any L(ctrmm_kernel_L2_Mv1_20) -.Lctrmm_kernel_L2_END: +L(ctrmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -878,11 +878,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lctrmm_kernel_L1_BEGIN: +L(ctrmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lctrmm_kernel_L999 // done + ble L(ctrmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -894,14 +894,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lctrmm_kernel_L1_Mv1_BEGIN: +L(ctrmm_kernel_L1_Mv1_BEGIN): mov counterI, #0 whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s -.Lctrmm_kernel_L1_Mv1_20: +L(ctrmm_kernel_L1_Mv1_20): INITv1x1 @@ -925,10 +925,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lctrmm_kernel_L1_Mv1_40 + ble L(ctrmm_kernel_L1_Mv1_40) .align 5 -.Lctrmm_kernel_L1_Mv1_22: +L(ctrmm_kernel_L1_Mv1_22): KERNELv1x1_SUB KERNELv1x1_SUB KERNELv1x1_SUB @@ -940,22 +940,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_Mv1_22 + bgt L(ctrmm_kernel_L1_Mv1_22) -.Lctrmm_kernel_L1_Mv1_40: +L(ctrmm_kernel_L1_Mv1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lctrmm_kernel_L1_Mv1_100 + ble L(ctrmm_kernel_L1_Mv1_100) -.Lctrmm_kernel_L1_Mv1_42: +L(ctrmm_kernel_L1_Mv1_42): KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lctrmm_kernel_L1_Mv1_42 + bgt L(ctrmm_kernel_L1_Mv1_42) -.Lctrmm_kernel_L1_Mv1_100: +L(ctrmm_kernel_L1_Mv1_100): SAVEv1x1 @@ -975,18 +975,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, lanes #endif -.Lctrmm_kernel_L1_Mv1_END: +L(ctrmm_kernel_L1_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lctrmm_kernel_L1_Mv1_20 + b.any L(ctrmm_kernel_L1_Mv1_20) -.Lctrmm_kernel_L1_END: +L(ctrmm_kernel_L1_END): /******************************************************************************/ -.Lctrmm_kernel_L999: +L(ctrmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/daxpy_thunderx2t99.S b/kernel/arm64/daxpy_thunderx2t99.S index baf39150fe..5205032ccc 100644 --- a/kernel/arm64/daxpy_thunderx2t99.S +++ b/kernel/arm64/daxpy_thunderx2t99.S @@ -169,65 +169,65 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Ldaxpy_kernel_L999 + ble L(daxpy_kernel_L999) fcmp DA, #0.0 - beq .Ldaxpy_kernel_L999 + beq L(daxpy_kernel_L999) cmp INC_X, #1 - bne .Ldaxpy_kernel_S_BEGIN + bne L(daxpy_kernel_S_BEGIN) cmp INC_Y, #1 - bne .Ldaxpy_kernel_S_BEGIN + bne L(daxpy_kernel_S_BEGIN) -.Ldaxpy_kernel_F_BEGIN: +L(daxpy_kernel_F_BEGIN): asr I, N, #5 cmp I, xzr - beq .Ldaxpy_kernel_F1 + beq L(daxpy_kernel_F1) cmp N, #2048 - ble .Ldaxpy_kernel_F32_L1CACHE + ble L(daxpy_kernel_F32_L1CACHE) .align 5 -.Ldaxpy_kernel_F32: +L(daxpy_kernel_F32): KERNEL_F32 subs I, I, #1 - bne .Ldaxpy_kernel_F32 - b .Ldaxpy_kernel_F1 + bne L(daxpy_kernel_F32) + b L(daxpy_kernel_F1) .align 5 -.Ldaxpy_kernel_F32_L1CACHE: +L(daxpy_kernel_F32_L1CACHE): KERNEL_F32_L1CACHE subs I, I, #1 - bne .Ldaxpy_kernel_F32_L1CACHE + bne L(daxpy_kernel_F32_L1CACHE) -.Ldaxpy_kernel_F1: +L(daxpy_kernel_F1): ands I, N, #31 - ble .Ldaxpy_kernel_L999 + ble L(daxpy_kernel_L999) -.Ldaxpy_kernel_F10: +L(daxpy_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Ldaxpy_kernel_F10 + bne L(daxpy_kernel_F10) - b .Ldaxpy_kernel_L999 + b L(daxpy_kernel_L999) -.Ldaxpy_kernel_S_BEGIN: +L(daxpy_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Ldaxpy_kernel_S1 + ble L(daxpy_kernel_S1) -.Ldaxpy_kernel_S4: +L(daxpy_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -235,21 +235,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Ldaxpy_kernel_S4 + bne L(daxpy_kernel_S4) -.Ldaxpy_kernel_S1: +L(daxpy_kernel_S1): ands I, N, #3 - ble .Ldaxpy_kernel_L999 + ble L(daxpy_kernel_L999) -.Ldaxpy_kernel_S10: +L(daxpy_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Ldaxpy_kernel_S10 + bne L(daxpy_kernel_S10) -.Ldaxpy_kernel_L999: +L(daxpy_kernel_L999): mov w0, wzr ret diff --git a/kernel/arm64/dgemm_beta.S b/kernel/arm64/dgemm_beta.S index 7d21525c2e..26fa5d7770 100644 --- a/kernel/arm64/dgemm_beta.S +++ b/kernel/arm64/dgemm_beta.S @@ -102,33 +102,33 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ldr LDC, [sp] SAVE_REGS -.Lgemm_beta_BEGIN: +L(gemm_beta_BEGIN): fmov beta0, BETA cmp N, #0 - ble .Lgemm_beta_L999 + ble L(gemm_beta_L999) fcmp BETA, #0.0 - beq .Lgemm_beta_zero_01 + beq L(gemm_beta_zero_01) -.Lgemm_beta_01: +L(gemm_beta_01): lsl LDC, LDC, #3 .align 5 -.Lgemm_beta_02: +L(gemm_beta_02): mov A01, C00 add C00, C00, LDC asr I, M, #4 cmp I, #0 - ble .Lgemm_beta_04 + ble L(gemm_beta_04) add A02, A01, #32 add A03, A02, #32 add A04, A03, #32 .align 5 -.Lgemm_beta_03: +L(gemm_beta_03): ldp q0, q1, [A01] ldp q2, q3, [A02] @@ -161,17 +161,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add A04, A04, calc_size subs I , I , #1 - bne .Lgemm_beta_03 + bne L(gemm_beta_03) .align 5 -.Lgemm_beta_04: +L(gemm_beta_04): and I, M , #15 // M%16 cmp I, #0 - ble .Lgemm_beta_06 + ble L(gemm_beta_06) .align 5 -.Lgemm_beta_05: +L(gemm_beta_05): ldr d12, [A01] fmul d12, d12, beta0 @@ -179,38 +179,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add A01, A01, #8 subs I , I , #1 - bne .Lgemm_beta_05 + bne L(gemm_beta_05) .align 5 -.Lgemm_beta_06: +L(gemm_beta_06): subs N , N, #1 // N-- - bne .Lgemm_beta_02 + bne L(gemm_beta_02) .align 5 -.Lgemm_beta_L999: +L(gemm_beta_L999): mov x0, #0 RESTORE_REGS ret -.Lgemm_beta_zero_01: +L(gemm_beta_zero_01): INIT_ZERO lsl LDC, LDC, #3 .align 5 -.Lgemm_beta_zero_02: +L(gemm_beta_zero_02): mov A01, C00 add C00, C00, LDC asr I, M, #4 cmp I, #0 - ble .Lgemm_beta_zero_04 + ble L(gemm_beta_zero_04) add A02, A01, #64 .align 5 -.Lgemm_beta_zero_03: +L(gemm_beta_zero_03): st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [A01] add A01, A01, calc_size @@ -218,32 +218,32 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add A02, A02, calc_size subs I, I, #1 - bne .Lgemm_beta_zero_03 + bne L(gemm_beta_zero_03) .align 5 -.Lgemm_beta_zero_04: +L(gemm_beta_zero_04): and I, M, #15 cmp I, #0 - ble .Lgemm_beta_zero_06 + ble L(gemm_beta_zero_06) .align 5 -.Lgemm_beta_zero_05: +L(gemm_beta_zero_05): str beta0, [A01] add A01, A01, #8 subs I, I, #1 - bne .Lgemm_beta_zero_05 + bne L(gemm_beta_zero_05) .align 5 -.Lgemm_beta_zero_06: +L(gemm_beta_zero_06): subs N, N, #1 - bne .Lgemm_beta_zero_02 + bne L(gemm_beta_zero_02) .align 5 -.Lgemm_beta_zero_L999: +L(gemm_beta_zero_L999): mov x0, #0 RESTORE_REGS diff --git a/kernel/arm64/dgemm_kernel_4x4.S b/kernel/arm64/dgemm_kernel_4x4.S index 3491670628..5c2ec3a0ed 100644 --- a/kernel/arm64/dgemm_kernel_4x4.S +++ b/kernel/arm64/dgemm_kernel_4x4.S @@ -775,9 +775,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Ldgemm_kernel_L2_BEGIN + ble L(dgemm_kernel_L2_BEGIN) -.Ldgemm_kernel_L4_BEGIN: +L(dgemm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -791,20 +791,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //------------------------------------------------------------------------------ -.Ldgemm_kernel_L4_M8_BEGIN: +L(dgemm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Ldgemm_kernel_L4_M4_BEGIN + ble L(dgemm_kernel_L4_M4_BEGIN) .align 5 -.Ldgemm_kernel_L4_M8_20: +L(dgemm_kernel_L4_M8_20): mov pB, origPB asr counterL , origK, #2 // L = K / 4 cmp counterL , #2 - blt .Ldgemm_kernel_L4_M8_32 + blt L(dgemm_kernel_L4_M8_32) KERNEL8x4_I KERNEL8x4_M2 @@ -812,60 +812,60 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M2 subs counterL, counterL, #2 // subtract 2 - ble .Ldgemm_kernel_L4_M8_22a + ble L(dgemm_kernel_L4_M8_22a) .align 5 -.Ldgemm_kernel_L4_M8_22: +L(dgemm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 KERNEL8x4_M1 KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M8_22 + bgt L(dgemm_kernel_L4_M8_22) .align 5 -.Ldgemm_kernel_L4_M8_22a: +L(dgemm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_M2 KERNEL8x4_M1 KERNEL8x4_E - b .Ldgemm_kernel_L4_M8_44 + b L(dgemm_kernel_L4_M8_44) .align 5 -.Ldgemm_kernel_L4_M8_32: +L(dgemm_kernel_L4_M8_32): tst counterL, #1 - ble .Ldgemm_kernel_L4_M8_40 + ble L(dgemm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_M2 KERNEL8x4_M1 KERNEL8x4_E - b .Ldgemm_kernel_L4_M8_44 + b L(dgemm_kernel_L4_M8_44) -.Ldgemm_kernel_L4_M8_40: +L(dgemm_kernel_L4_M8_40): INIT8x4 -.Ldgemm_kernel_L4_M8_44: +L(dgemm_kernel_L4_M8_44): ands counterL , origK, #3 - ble .Ldgemm_kernel_L4_M8_100 + ble L(dgemm_kernel_L4_M8_100) .align 5 -.Ldgemm_kernel_L4_M8_46: +L(dgemm_kernel_L4_M8_46): KERNEL8x4_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L4_M8_46 + bne L(dgemm_kernel_L4_M8_46) -.Ldgemm_kernel_L4_M8_100: +L(dgemm_kernel_L4_M8_100): lsl temp, origK, #5 prfm PLDL1KEEP, [pA, temp] prfm PLDL1KEEP, [ppA, temp] @@ -873,31 +873,31 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SAVE8x4 -.Ldgemm_kernel_L4_M8_END: +L(dgemm_kernel_L4_M8_END): lsl temp, origK, #5 // k * 4 * 8 add pA, pA, temp add ppA, ppA, temp subs counterI, counterI, #1 - bne .Ldgemm_kernel_L4_M8_20 + bne L(dgemm_kernel_L4_M8_20) -.Ldgemm_kernel_L4_M4_BEGIN: +L(dgemm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Ldgemm_kernel_L4_END + ble L(dgemm_kernel_L4_END) tst counterI, #4 - ble .Ldgemm_kernel_L4_M2_BEGIN + ble L(dgemm_kernel_L4_M2_BEGIN) -.Ldgemm_kernel_L4_M4_20: +L(dgemm_kernel_L4_M4_20): INIT4x4 mov pB, origPB asr counterL, origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Ldgemm_kernel_L4_M4_40 + ble L(dgemm_kernel_L4_M4_40) -.Ldgemm_kernel_L4_M4_22: +L(dgemm_kernel_L4_M4_22): KERNEL4x4_SUB KERNEL4x4_SUB @@ -910,47 +910,47 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M4_22 + bgt L(dgemm_kernel_L4_M4_22) -.Ldgemm_kernel_L4_M4_40: +L(dgemm_kernel_L4_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L4_M4_100 + ble L(dgemm_kernel_L4_M4_100) -.Ldgemm_kernel_L4_M4_42: +L(dgemm_kernel_L4_M4_42): KERNEL4x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M4_42 + bgt L(dgemm_kernel_L4_M4_42) -.Ldgemm_kernel_L4_M4_100: +L(dgemm_kernel_L4_M4_100): SAVE4x4 -.Ldgemm_kernel_L4_M4_END: +L(dgemm_kernel_L4_M4_END): -.Ldgemm_kernel_L4_M2_BEGIN: +L(dgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L4_END + ble L(dgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L4_M1_BEGIN + ble L(dgemm_kernel_L4_M1_BEGIN) -.Ldgemm_kernel_L4_M2_20: +L(dgemm_kernel_L4_M2_20): INIT2x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L4_M2_40 + ble L(dgemm_kernel_L4_M2_40) -.Ldgemm_kernel_L4_M2_22: +L(dgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -963,43 +963,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M2_22 + bgt L(dgemm_kernel_L4_M2_22) -.Ldgemm_kernel_L4_M2_40: +L(dgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L4_M2_100 + ble L(dgemm_kernel_L4_M2_100) -.Ldgemm_kernel_L4_M2_42: +L(dgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M2_42 + bgt L(dgemm_kernel_L4_M2_42) -.Ldgemm_kernel_L4_M2_100: +L(dgemm_kernel_L4_M2_100): SAVE2x4 -.Ldgemm_kernel_L4_M2_END: +L(dgemm_kernel_L4_M2_END): -.Ldgemm_kernel_L4_M1_BEGIN: +L(dgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L4_END + ble L(dgemm_kernel_L4_END) -.Ldgemm_kernel_L4_M1_20: +L(dgemm_kernel_L4_M1_20): INIT1x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L4_M1_40 + ble L(dgemm_kernel_L4_M1_40) -.Ldgemm_kernel_L4_M1_22: +L(dgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1011,45 +1011,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M1_22 + bgt L(dgemm_kernel_L4_M1_22) -.Ldgemm_kernel_L4_M1_40: +L(dgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L4_M1_100 + ble L(dgemm_kernel_L4_M1_100) -.Ldgemm_kernel_L4_M1_42: +L(dgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M1_42 + bgt L(dgemm_kernel_L4_M1_42) -.Ldgemm_kernel_L4_M1_100: +L(dgemm_kernel_L4_M1_100): SAVE1x4 -.Ldgemm_kernel_L4_END: +L(dgemm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 subs counterJ, counterJ , #1 // j-- - bgt .Ldgemm_kernel_L4_BEGIN + bgt L(dgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Ldgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(dgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Ldgemm_kernel_L999 // error, N was less than 4? + ble L(dgemm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Ldgemm_kernel_L1_BEGIN + ble L(dgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1059,24 +1059,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Ldgemm_kernel_L2_M4_BEGIN: +L(dgemm_kernel_L2_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI,#0 - ble .Ldgemm_kernel_L2_M2_BEGIN + ble L(dgemm_kernel_L2_M2_BEGIN) -.Ldgemm_kernel_L2_M4_20: +L(dgemm_kernel_L2_M4_20): INIT4x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldgemm_kernel_L2_M4_40 + ble L(dgemm_kernel_L2_M4_40) .align 5 -.Ldgemm_kernel_L2_M4_22: +L(dgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1088,50 +1088,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M4_22 + bgt L(dgemm_kernel_L2_M4_22) -.Ldgemm_kernel_L2_M4_40: +L(dgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M4_100 + ble L(dgemm_kernel_L2_M4_100) -.Ldgemm_kernel_L2_M4_42: +L(dgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M4_42 + bgt L(dgemm_kernel_L2_M4_42) -.Ldgemm_kernel_L2_M4_100: +L(dgemm_kernel_L2_M4_100): SAVE4x2 -.Ldgemm_kernel_L2_M4_END: +L(dgemm_kernel_L2_M4_END): subs counterI, counterI, #1 - bgt .Ldgemm_kernel_L2_M4_20 + bgt L(dgemm_kernel_L2_M4_20) -.Ldgemm_kernel_L2_M2_BEGIN: +L(dgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L2_END + ble L(dgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L2_M1_BEGIN + ble L(dgemm_kernel_L2_M1_BEGIN) -.Ldgemm_kernel_L2_M2_20: +L(dgemm_kernel_L2_M2_20): INIT2x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldgemm_kernel_L2_M2_40 + ble L(dgemm_kernel_L2_M2_40) -.Ldgemm_kernel_L2_M2_22: +L(dgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1144,43 +1144,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M2_22 + bgt L(dgemm_kernel_L2_M2_22) -.Ldgemm_kernel_L2_M2_40: +L(dgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M2_100 + ble L(dgemm_kernel_L2_M2_100) -.Ldgemm_kernel_L2_M2_42: +L(dgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M2_42 + bgt L(dgemm_kernel_L2_M2_42) -.Ldgemm_kernel_L2_M2_100: +L(dgemm_kernel_L2_M2_100): SAVE2x2 -.Ldgemm_kernel_L2_M2_END: +L(dgemm_kernel_L2_M2_END): -.Ldgemm_kernel_L2_M1_BEGIN: +L(dgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L2_END + ble L(dgemm_kernel_L2_END) -.Ldgemm_kernel_L2_M1_20: +L(dgemm_kernel_L2_M1_20): INIT1x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Ldgemm_kernel_L2_M1_40 + ble L(dgemm_kernel_L2_M1_40) -.Ldgemm_kernel_L2_M1_22: +L(dgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1192,36 +1192,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M1_22 + bgt L(dgemm_kernel_L2_M1_22) -.Ldgemm_kernel_L2_M1_40: +L(dgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M1_100 + ble L(dgemm_kernel_L2_M1_100) -.Ldgemm_kernel_L2_M1_42: +L(dgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M1_42 + bgt L(dgemm_kernel_L2_M1_42) -.Ldgemm_kernel_L2_M1_100: +L(dgemm_kernel_L2_M1_100): SAVE1x2 -.Ldgemm_kernel_L2_END: +L(dgemm_kernel_L2_END): add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 /******************************************************************************/ -.Ldgemm_kernel_L1_BEGIN: +L(dgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldgemm_kernel_L999 // done + ble L(dgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1231,24 +1231,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Ldgemm_kernel_L1_M4_BEGIN: +L(dgemm_kernel_L1_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Ldgemm_kernel_L1_M2_BEGIN + ble L(dgemm_kernel_L1_M2_BEGIN) -.Ldgemm_kernel_L1_M4_20: +L(dgemm_kernel_L1_M4_20): INIT4x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M4_40 + ble L(dgemm_kernel_L1_M4_40) .align 5 -.Ldgemm_kernel_L1_M4_22: +L(dgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1260,50 +1260,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M4_22 + bgt L(dgemm_kernel_L1_M4_22) -.Ldgemm_kernel_L1_M4_40: +L(dgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M4_100 + ble L(dgemm_kernel_L1_M4_100) -.Ldgemm_kernel_L1_M4_42: +L(dgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M4_42 + bgt L(dgemm_kernel_L1_M4_42) -.Ldgemm_kernel_L1_M4_100: +L(dgemm_kernel_L1_M4_100): SAVE4x1 -.Ldgemm_kernel_L1_M4_END: +L(dgemm_kernel_L1_M4_END): subs counterI, counterI, #1 - bgt .Ldgemm_kernel_L1_M4_20 + bgt L(dgemm_kernel_L1_M4_20) -.Ldgemm_kernel_L1_M2_BEGIN: +L(dgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L1_END + ble L(dgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L1_M1_BEGIN + ble L(dgemm_kernel_L1_M1_BEGIN) -.Ldgemm_kernel_L1_M2_20: +L(dgemm_kernel_L1_M2_20): INIT2x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M2_40 + ble L(dgemm_kernel_L1_M2_40) -.Ldgemm_kernel_L1_M2_22: +L(dgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1316,43 +1316,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M2_22 + bgt L(dgemm_kernel_L1_M2_22) -.Ldgemm_kernel_L1_M2_40: +L(dgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M2_100 + ble L(dgemm_kernel_L1_M2_100) -.Ldgemm_kernel_L1_M2_42: +L(dgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M2_42 + bgt L(dgemm_kernel_L1_M2_42) -.Ldgemm_kernel_L1_M2_100: +L(dgemm_kernel_L1_M2_100): SAVE2x1 -.Ldgemm_kernel_L1_M2_END: +L(dgemm_kernel_L1_M2_END): -.Ldgemm_kernel_L1_M1_BEGIN: +L(dgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L1_END + ble L(dgemm_kernel_L1_END) -.Ldgemm_kernel_L1_M1_20: +L(dgemm_kernel_L1_M1_20): INIT1x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M1_40 + ble L(dgemm_kernel_L1_M1_40) -.Ldgemm_kernel_L1_M1_22: +L(dgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1364,30 +1364,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M1_22 + bgt L(dgemm_kernel_L1_M1_22) -.Ldgemm_kernel_L1_M1_40: +L(dgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M1_100 + ble L(dgemm_kernel_L1_M1_100) -.Ldgemm_kernel_L1_M1_42: +L(dgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M1_42 + bgt L(dgemm_kernel_L1_M1_42) -.Ldgemm_kernel_L1_M1_100: +L(dgemm_kernel_L1_M1_100): SAVE1x1 -.Ldgemm_kernel_L1_END: +L(dgemm_kernel_L1_END): -.Ldgemm_kernel_L999: +L(dgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dgemm_kernel_4x8.S b/kernel/arm64/dgemm_kernel_4x8.S index ced26b49ce..f4af8cc25d 100644 --- a/kernel/arm64/dgemm_kernel_4x8.S +++ b/kernel/arm64/dgemm_kernel_4x8.S @@ -938,98 +938,98 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Ldgemm_kernel_L4_BEGIN + ble L(dgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Ldgemm_kernel_L8_BEGIN: +L(dgemm_kernel_L8_BEGIN): mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #3 mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L8_M4_BEGIN: +L(dgemm_kernel_L8_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Ldgemm_kernel_L8_M2_BEGIN + ble L(dgemm_kernel_L8_M2_BEGIN) -.Ldgemm_kernel_L8_M4_20: +L(dgemm_kernel_L8_M4_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Ldgemm_kernel_L8_M4_32 + blt L(dgemm_kernel_L8_M4_32) KERNEL4x8_I // do one in the K KERNEL4x8_M2 // do another in the K subs counterL, counterL, #2 - ble .Ldgemm_kernel_L8_M4_22a + ble L(dgemm_kernel_L8_M4_22a) .align 5 -.Ldgemm_kernel_L8_M4_22: +L(dgemm_kernel_L8_M4_22): KERNEL4x8_M1 KERNEL4x8_M2 subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L8_M4_22 + bgt L(dgemm_kernel_L8_M4_22) -.Ldgemm_kernel_L8_M4_22a: +L(dgemm_kernel_L8_M4_22a): KERNEL4x8_M1 KERNEL4x8_E - b .Ldgemm_kernel_L8_M4_44 + b L(dgemm_kernel_L8_M4_44) -.Ldgemm_kernel_L8_M4_32: +L(dgemm_kernel_L8_M4_32): tst counterL, #1 - ble .Ldgemm_kernel_L8_M4_40 + ble L(dgemm_kernel_L8_M4_40) KERNEL4x8_I KERNEL4x8_E - b .Ldgemm_kernel_L8_M4_44 + b L(dgemm_kernel_L8_M4_44) -.Ldgemm_kernel_L8_M4_40: +L(dgemm_kernel_L8_M4_40): INIT4x8 -.Ldgemm_kernel_L8_M4_44: +L(dgemm_kernel_L8_M4_44): ands counterL , origK, #1 - ble .Ldgemm_kernel_L8_M4_100 + ble L(dgemm_kernel_L8_M4_100) -.Ldgemm_kernel_L8_M4_46: +L(dgemm_kernel_L8_M4_46): KERNEL4x8_SUB -.Ldgemm_kernel_L8_M4_100: +L(dgemm_kernel_L8_M4_100): SAVE4x8 -.Ldgemm_kernel_L8_M4_END: +L(dgemm_kernel_L8_M4_END): subs counterI, counterI, #1 - bne .Ldgemm_kernel_L8_M4_20 + bne L(dgemm_kernel_L8_M4_20) -.Ldgemm_kernel_L8_M2_BEGIN: +L(dgemm_kernel_L8_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L8_END + ble L(dgemm_kernel_L8_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L8_M1_BEGIN + ble L(dgemm_kernel_L8_M1_BEGIN) -.Ldgemm_kernel_L8_M2_20: +L(dgemm_kernel_L8_M2_20): INIT2x8 @@ -1037,9 +1037,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L8_M2_40 + ble L(dgemm_kernel_L8_M2_40) -.Ldgemm_kernel_L8_M2_22: +L(dgemm_kernel_L8_M2_22): KERNEL2x8_SUB KERNEL2x8_SUB @@ -1052,34 +1052,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L8_M2_22 + bgt L(dgemm_kernel_L8_M2_22) -.Ldgemm_kernel_L8_M2_40: +L(dgemm_kernel_L8_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L8_M2_100 + ble L(dgemm_kernel_L8_M2_100) -.Ldgemm_kernel_L8_M2_42: +L(dgemm_kernel_L8_M2_42): KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L8_M2_42 + bgt L(dgemm_kernel_L8_M2_42) -.Ldgemm_kernel_L8_M2_100: +L(dgemm_kernel_L8_M2_100): SAVE2x8 -.Ldgemm_kernel_L8_M2_END: +L(dgemm_kernel_L8_M2_END): -.Ldgemm_kernel_L8_M1_BEGIN: +L(dgemm_kernel_L8_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L8_END + ble L(dgemm_kernel_L8_END) -.Ldgemm_kernel_L8_M1_20: +L(dgemm_kernel_L8_M1_20): INIT1x8 @@ -1087,9 +1087,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L8_M1_40 + ble L(dgemm_kernel_L8_M1_40) -.Ldgemm_kernel_L8_M1_22: +L(dgemm_kernel_L8_M1_22): KERNEL1x8_SUB KERNEL1x8_SUB KERNEL1x8_SUB @@ -1101,131 +1101,131 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L8_M1_22 + bgt L(dgemm_kernel_L8_M1_22) -.Ldgemm_kernel_L8_M1_40: +L(dgemm_kernel_L8_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L8_M1_100 + ble L(dgemm_kernel_L8_M1_100) -.Ldgemm_kernel_L8_M1_42: +L(dgemm_kernel_L8_M1_42): KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L8_M1_42 + bgt L(dgemm_kernel_L8_M1_42) -.Ldgemm_kernel_L8_M1_100: +L(dgemm_kernel_L8_M1_100): SAVE1x8 -.Ldgemm_kernel_L8_END: +L(dgemm_kernel_L8_END): lsl temp, origK, #6 add origPB, origPB, temp // B = B + K * 8 * 8 subs counterJ, counterJ , #1 // j-- - bgt .Ldgemm_kernel_L8_BEGIN + bgt L(dgemm_kernel_L8_BEGIN) /******************************************************************************/ -.Ldgemm_kernel_L4_BEGIN: +L(dgemm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #7 - ble .Ldgemm_kernel_L999 + ble L(dgemm_kernel_L999) tst counterJ , #4 - ble .Ldgemm_kernel_L2_BEGIN + ble L(dgemm_kernel_L2_BEGIN) mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #2 mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L4_M4_BEGIN: +L(dgemm_kernel_L4_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Ldgemm_kernel_L4_M2_BEGIN + ble L(dgemm_kernel_L4_M2_BEGIN) -.Ldgemm_kernel_L4_M4_20: +L(dgemm_kernel_L4_M4_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Ldgemm_kernel_L4_M4_32 + blt L(dgemm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Ldgemm_kernel_L4_M4_22a + ble L(dgemm_kernel_L4_M4_22a) .align 5 -.Ldgemm_kernel_L4_M4_22: +L(dgemm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M4_22 + bgt L(dgemm_kernel_L4_M4_22) -.Ldgemm_kernel_L4_M4_22a: +L(dgemm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Ldgemm_kernel_L4_M4_44 + b L(dgemm_kernel_L4_M4_44) -.Ldgemm_kernel_L4_M4_32: +L(dgemm_kernel_L4_M4_32): tst counterL, #1 - ble .Ldgemm_kernel_L4_M4_40 + ble L(dgemm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Ldgemm_kernel_L4_M4_44 + b L(dgemm_kernel_L4_M4_44) -.Ldgemm_kernel_L4_M4_40: +L(dgemm_kernel_L4_M4_40): INIT4x4 -.Ldgemm_kernel_L4_M4_44: +L(dgemm_kernel_L4_M4_44): ands counterL , origK, #1 - ble .Ldgemm_kernel_L4_M4_100 + ble L(dgemm_kernel_L4_M4_100) -.Ldgemm_kernel_L4_M4_46: +L(dgemm_kernel_L4_M4_46): KERNEL4x4_SUB -.Ldgemm_kernel_L4_M4_100: +L(dgemm_kernel_L4_M4_100): SAVE4x4 -.Ldgemm_kernel_L4_M4_END: +L(dgemm_kernel_L4_M4_END): subs counterI, counterI, #1 - bne .Ldgemm_kernel_L4_M4_20 + bne L(dgemm_kernel_L4_M4_20) -.Ldgemm_kernel_L4_M2_BEGIN: +L(dgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L4_END + ble L(dgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L4_M1_BEGIN + ble L(dgemm_kernel_L4_M1_BEGIN) -.Ldgemm_kernel_L4_M2_20: +L(dgemm_kernel_L4_M2_20): INIT2x4 @@ -1233,9 +1233,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L4_M2_40 + ble L(dgemm_kernel_L4_M2_40) -.Ldgemm_kernel_L4_M2_22: +L(dgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1248,34 +1248,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M2_22 + bgt L(dgemm_kernel_L4_M2_22) -.Ldgemm_kernel_L4_M2_40: +L(dgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L4_M2_100 + ble L(dgemm_kernel_L4_M2_100) -.Ldgemm_kernel_L4_M2_42: +L(dgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M2_42 + bgt L(dgemm_kernel_L4_M2_42) -.Ldgemm_kernel_L4_M2_100: +L(dgemm_kernel_L4_M2_100): SAVE2x4 -.Ldgemm_kernel_L4_M2_END: +L(dgemm_kernel_L4_M2_END): -.Ldgemm_kernel_L4_M1_BEGIN: +L(dgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L4_END + ble L(dgemm_kernel_L4_END) -.Ldgemm_kernel_L4_M1_20: +L(dgemm_kernel_L4_M1_20): INIT1x4 @@ -1283,9 +1283,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L4_M1_40 + ble L(dgemm_kernel_L4_M1_40) -.Ldgemm_kernel_L4_M1_22: +L(dgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1297,40 +1297,40 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M1_22 + bgt L(dgemm_kernel_L4_M1_22) -.Ldgemm_kernel_L4_M1_40: +L(dgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L4_M1_100 + ble L(dgemm_kernel_L4_M1_100) -.Ldgemm_kernel_L4_M1_42: +L(dgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M1_42 + bgt L(dgemm_kernel_L4_M1_42) -.Ldgemm_kernel_L4_M1_100: +L(dgemm_kernel_L4_M1_100): SAVE1x4 -.Ldgemm_kernel_L4_END: +L(dgemm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 /******************************************************************************/ -.Ldgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(dgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Ldgemm_kernel_L999 // error, N was less than 4? + ble L(dgemm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Ldgemm_kernel_L1_BEGIN + ble L(dgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1339,14 +1339,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Ldgemm_kernel_L2_M4_BEGIN: +L(dgemm_kernel_L2_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI,#0 - ble .Ldgemm_kernel_L2_M2_BEGIN + ble L(dgemm_kernel_L2_M2_BEGIN) -.Ldgemm_kernel_L2_M4_20: +L(dgemm_kernel_L2_M4_20): INIT4x2 @@ -1354,10 +1354,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldgemm_kernel_L2_M4_40 + ble L(dgemm_kernel_L2_M4_40) .align 5 -.Ldgemm_kernel_L2_M4_22: +L(dgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1369,41 +1369,41 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M4_22 + bgt L(dgemm_kernel_L2_M4_22) -.Ldgemm_kernel_L2_M4_40: +L(dgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M4_100 + ble L(dgemm_kernel_L2_M4_100) -.Ldgemm_kernel_L2_M4_42: +L(dgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M4_42 + bgt L(dgemm_kernel_L2_M4_42) -.Ldgemm_kernel_L2_M4_100: +L(dgemm_kernel_L2_M4_100): SAVE4x2 -.Ldgemm_kernel_L2_M4_END: +L(dgemm_kernel_L2_M4_END): subs counterI, counterI, #1 - bgt .Ldgemm_kernel_L2_M4_20 + bgt L(dgemm_kernel_L2_M4_20) -.Ldgemm_kernel_L2_M2_BEGIN: +L(dgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L2_END + ble L(dgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L2_M1_BEGIN + ble L(dgemm_kernel_L2_M1_BEGIN) -.Ldgemm_kernel_L2_M2_20: +L(dgemm_kernel_L2_M2_20): INIT2x2 @@ -1411,9 +1411,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldgemm_kernel_L2_M2_40 + ble L(dgemm_kernel_L2_M2_40) -.Ldgemm_kernel_L2_M2_22: +L(dgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1426,34 +1426,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M2_22 + bgt L(dgemm_kernel_L2_M2_22) -.Ldgemm_kernel_L2_M2_40: +L(dgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M2_100 + ble L(dgemm_kernel_L2_M2_100) -.Ldgemm_kernel_L2_M2_42: +L(dgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M2_42 + bgt L(dgemm_kernel_L2_M2_42) -.Ldgemm_kernel_L2_M2_100: +L(dgemm_kernel_L2_M2_100): SAVE2x2 -.Ldgemm_kernel_L2_M2_END: +L(dgemm_kernel_L2_M2_END): -.Ldgemm_kernel_L2_M1_BEGIN: +L(dgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L2_END + ble L(dgemm_kernel_L2_END) -.Ldgemm_kernel_L2_M1_20: +L(dgemm_kernel_L2_M1_20): INIT1x2 @@ -1461,9 +1461,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Ldgemm_kernel_L2_M1_40 + ble L(dgemm_kernel_L2_M1_40) -.Ldgemm_kernel_L2_M1_22: +L(dgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1475,35 +1475,35 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M1_22 + bgt L(dgemm_kernel_L2_M1_22) -.Ldgemm_kernel_L2_M1_40: +L(dgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M1_100 + ble L(dgemm_kernel_L2_M1_100) -.Ldgemm_kernel_L2_M1_42: +L(dgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M1_42 + bgt L(dgemm_kernel_L2_M1_42) -.Ldgemm_kernel_L2_M1_100: +L(dgemm_kernel_L2_M1_100): SAVE1x2 -.Ldgemm_kernel_L2_END: +L(dgemm_kernel_L2_END): add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 /******************************************************************************/ -.Ldgemm_kernel_L1_BEGIN: +L(dgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldgemm_kernel_L999 // done + ble L(dgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1511,24 +1511,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Ldgemm_kernel_L1_M4_BEGIN: +L(dgemm_kernel_L1_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Ldgemm_kernel_L1_M2_BEGIN + ble L(dgemm_kernel_L1_M2_BEGIN) -.Ldgemm_kernel_L1_M4_20: +L(dgemm_kernel_L1_M4_20): INIT4x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M4_40 + ble L(dgemm_kernel_L1_M4_40) .align 5 -.Ldgemm_kernel_L1_M4_22: +L(dgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1540,41 +1540,41 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M4_22 + bgt L(dgemm_kernel_L1_M4_22) -.Ldgemm_kernel_L1_M4_40: +L(dgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M4_100 + ble L(dgemm_kernel_L1_M4_100) -.Ldgemm_kernel_L1_M4_42: +L(dgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M4_42 + bgt L(dgemm_kernel_L1_M4_42) -.Ldgemm_kernel_L1_M4_100: +L(dgemm_kernel_L1_M4_100): SAVE4x1 -.Ldgemm_kernel_L1_M4_END: +L(dgemm_kernel_L1_M4_END): subs counterI, counterI, #1 - bgt .Ldgemm_kernel_L1_M4_20 + bgt L(dgemm_kernel_L1_M4_20) -.Ldgemm_kernel_L1_M2_BEGIN: +L(dgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L1_END + ble L(dgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L1_M1_BEGIN + ble L(dgemm_kernel_L1_M1_BEGIN) -.Ldgemm_kernel_L1_M2_20: +L(dgemm_kernel_L1_M2_20): INIT2x1 @@ -1582,9 +1582,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M2_40 + ble L(dgemm_kernel_L1_M2_40) -.Ldgemm_kernel_L1_M2_22: +L(dgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1597,34 +1597,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M2_22 + bgt L(dgemm_kernel_L1_M2_22) -.Ldgemm_kernel_L1_M2_40: +L(dgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M2_100 + ble L(dgemm_kernel_L1_M2_100) -.Ldgemm_kernel_L1_M2_42: +L(dgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M2_42 + bgt L(dgemm_kernel_L1_M2_42) -.Ldgemm_kernel_L1_M2_100: +L(dgemm_kernel_L1_M2_100): SAVE2x1 -.Ldgemm_kernel_L1_M2_END: +L(dgemm_kernel_L1_M2_END): -.Ldgemm_kernel_L1_M1_BEGIN: +L(dgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L1_END + ble L(dgemm_kernel_L1_END) -.Ldgemm_kernel_L1_M1_20: +L(dgemm_kernel_L1_M1_20): INIT1x1 @@ -1632,9 +1632,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M1_40 + ble L(dgemm_kernel_L1_M1_40) -.Ldgemm_kernel_L1_M1_22: +L(dgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1646,30 +1646,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M1_22 + bgt L(dgemm_kernel_L1_M1_22) -.Ldgemm_kernel_L1_M1_40: +L(dgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M1_100 + ble L(dgemm_kernel_L1_M1_100) -.Ldgemm_kernel_L1_M1_42: +L(dgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M1_42 + bgt L(dgemm_kernel_L1_M1_42) -.Ldgemm_kernel_L1_M1_100: +L(dgemm_kernel_L1_M1_100): SAVE1x1 -.Ldgemm_kernel_L1_END: +L(dgemm_kernel_L1_END): -.Ldgemm_kernel_L999: +L(dgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dgemm_kernel_6x8_cortexa72.S b/kernel/arm64/dgemm_kernel_6x8_cortexa72.S index 54478af0b3..5a2a7f0999 100644 --- a/kernel/arm64/dgemm_kernel_6x8_cortexa72.S +++ b/kernel/arm64/dgemm_kernel_6x8_cortexa72.S @@ -774,86 +774,86 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 - cbz counterJ, .L6x8_L4_BEGIN + cbz counterJ, L(6x8_L4_BEGIN) -.L6x8_L8_BEGIN: +L(6x8_L8_BEGIN): mov pCRow0, pC add pC, pC, LDC, lsl #3 mov pA, origPA mov temp, #6 udiv counterI, origM, temp - cbz counterI, .L6x8_L8_Mrem + cbz counterI, L(6x8_L8_Mrem) -.L6x8_L8_M6: +L(6x8_L8_M6): INIT6x8 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L8_M6s -.L6x8_L8_M6l: + cbz counterL, L(6x8_L8_M6s) +L(6x8_L8_M6l): KERNEL6x8_SUB subs counterL, counterL, #1 - bgt .L6x8_L8_M6l -.L6x8_L8_M6s: + bgt L(6x8_L8_M6l) +L(6x8_L8_M6s): SAVE6x8 subs counterI, counterI, #1 - bgt .L6x8_L8_M6 + bgt L(6x8_L8_M6) -.L6x8_L8_Mrem: +L(6x8_L8_Mrem): mov temp, #6 udiv counterI, origM, temp msub counterI, counterI, temp, origM cmp counterI, #4 - blt .L6x8_L8_M2c + blt L(6x8_L8_M2c) INIT4x8 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L8_M4s -.L6x8_L8_M4l: + cbz counterL, L(6x8_L8_M4s) +L(6x8_L8_M4l): KERNEL4x8_SUB subs counterL, counterL, #1 - bgt .L6x8_L8_M4l -.L6x8_L8_M4s: + bgt L(6x8_L8_M4l) +L(6x8_L8_M4s): SAVE4x8 sub counterI, counterI, #4 -.L6x8_L8_M2c: +L(6x8_L8_M2c): cmp counterI, #2 - blt .L6x8_L8_M1c + blt L(6x8_L8_M1c) INIT2x8 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L8_M2s -.L6x8_L8_M2l: + cbz counterL, L(6x8_L8_M2s) +L(6x8_L8_M2l): KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .L6x8_L8_M2l -.L6x8_L8_M2s: + bgt L(6x8_L8_M2l) +L(6x8_L8_M2s): SAVE2x8 sub counterI, counterI, #2 -.L6x8_L8_M1c: - cbz counterI, .L6x8_L8_END +L(6x8_L8_M1c): + cbz counterI, L(6x8_L8_END) INIT1x8 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L8_M1s -.L6x8_L8_M1l: + cbz counterL, L(6x8_L8_M1s) +L(6x8_L8_M1l): KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .L6x8_L8_M1l -.L6x8_L8_M1s: + bgt L(6x8_L8_M1l) +L(6x8_L8_M1s): SAVE1x8 -.L6x8_L8_END: +L(6x8_L8_END): lsl temp, origK, #6 /* + K*8*8 bytes */ add origPB, origPB, temp subs counterJ, counterJ, #1 - bgt .L6x8_L8_BEGIN + bgt L(6x8_L8_BEGIN) -.L6x8_L4_BEGIN: +L(6x8_L4_BEGIN): tst origN, #7 - beq .L6x8_L999 + beq L(6x8_L999) tst origN, #4 - beq .L6x8_L2_BEGIN + beq L(6x8_L2_BEGIN) mov pCRow0, pC add pC, pC, LDC, lsl #2 @@ -861,200 +861,200 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov temp, #6 udiv counterI, origM, temp - cbz counterI, .L6x8_L4_Mrem -.L6x8_L4_M6: + cbz counterI, L(6x8_L4_Mrem) +L(6x8_L4_M6): INIT6x4 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L4_M6s -.L6x8_L4_M6l: + cbz counterL, L(6x8_L4_M6s) +L(6x8_L4_M6l): KERNEL6x4_SUB subs counterL, counterL, #1 - bgt .L6x8_L4_M6l -.L6x8_L4_M6s: + bgt L(6x8_L4_M6l) +L(6x8_L4_M6s): SAVE6x4 subs counterI, counterI, #1 - bgt .L6x8_L4_M6 -.L6x8_L4_Mrem: + bgt L(6x8_L4_M6) +L(6x8_L4_Mrem): mov temp, #6 udiv counterI, origM, temp msub counterI, counterI, temp, origM cmp counterI, #4 - blt .L6x8_L4_M2c + blt L(6x8_L4_M2c) INIT4x4 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L4_M4s -.L6x8_L4_M4l: + cbz counterL, L(6x8_L4_M4s) +L(6x8_L4_M4l): KERNEL4x4_SUB subs counterL, counterL, #1 - bgt .L6x8_L4_M4l -.L6x8_L4_M4s: + bgt L(6x8_L4_M4l) +L(6x8_L4_M4s): SAVE4x4 sub counterI, counterI, #4 -.L6x8_L4_M2c: +L(6x8_L4_M2c): cmp counterI, #2 - blt .L6x8_L4_M1c + blt L(6x8_L4_M1c) INIT2x4 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L4_M2s -.L6x8_L4_M2l: + cbz counterL, L(6x8_L4_M2s) +L(6x8_L4_M2l): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .L6x8_L4_M2l -.L6x8_L4_M2s: + bgt L(6x8_L4_M2l) +L(6x8_L4_M2s): SAVE2x4 sub counterI, counterI, #2 -.L6x8_L4_M1c: - cbz counterI, .L6x8_L4_END +L(6x8_L4_M1c): + cbz counterI, L(6x8_L4_END) INIT1x4 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L4_M1s -.L6x8_L4_M1l: + cbz counterL, L(6x8_L4_M1s) +L(6x8_L4_M1l): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .L6x8_L4_M1l -.L6x8_L4_M1s: + bgt L(6x8_L4_M1l) +L(6x8_L4_M1s): SAVE1x4 -.L6x8_L4_END: +L(6x8_L4_END): add origPB, origPB, origK, lsl #5 /* +K*4*8 */ -.L6x8_L2_BEGIN: +L(6x8_L2_BEGIN): tst origN, #2 - beq .L6x8_L1_BEGIN + beq L(6x8_L1_BEGIN) mov pCRow0, pC add pC, pC, LDC, lsl #1 mov pA, origPA mov temp, #6 udiv counterI, origM, temp - cbz counterI, .L6x8_L2_Mrem -.L6x8_L2_M6: + cbz counterI, L(6x8_L2_Mrem) +L(6x8_L2_M6): INIT6x2 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L2_M6s -.L6x8_L2_M6l: + cbz counterL, L(6x8_L2_M6s) +L(6x8_L2_M6l): KERNEL6x2_SUB subs counterL, counterL, #1 - bgt .L6x8_L2_M6l -.L6x8_L2_M6s: + bgt L(6x8_L2_M6l) +L(6x8_L2_M6s): SAVE6x2 subs counterI, counterI, #1 - bgt .L6x8_L2_M6 -.L6x8_L2_Mrem: + bgt L(6x8_L2_M6) +L(6x8_L2_Mrem): mov temp, #6 udiv counterI, origM, temp msub counterI, counterI, temp, origM cmp counterI, #4 - blt .L6x8_L2_M2c + blt L(6x8_L2_M2c) INIT4x2 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L2_M4s -.L6x8_L2_M4l: + cbz counterL, L(6x8_L2_M4s) +L(6x8_L2_M4l): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .L6x8_L2_M4l -.L6x8_L2_M4s: + bgt L(6x8_L2_M4l) +L(6x8_L2_M4s): SAVE4x2 sub counterI, counterI, #4 -.L6x8_L2_M2c: +L(6x8_L2_M2c): cmp counterI, #2 - blt .L6x8_L2_M1c + blt L(6x8_L2_M1c) INIT2x2 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L2_M2s -.L6x8_L2_M2l: + cbz counterL, L(6x8_L2_M2s) +L(6x8_L2_M2l): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .L6x8_L2_M2l -.L6x8_L2_M2s: + bgt L(6x8_L2_M2l) +L(6x8_L2_M2s): SAVE2x2 sub counterI, counterI, #2 -.L6x8_L2_M1c: - cbz counterI, .L6x8_L2_END +L(6x8_L2_M1c): + cbz counterI, L(6x8_L2_END) INIT1x2 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L2_M1s -.L6x8_L2_M1l: + cbz counterL, L(6x8_L2_M1s) +L(6x8_L2_M1l): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .L6x8_L2_M1l -.L6x8_L2_M1s: + bgt L(6x8_L2_M1l) +L(6x8_L2_M1s): SAVE1x2 -.L6x8_L2_END: +L(6x8_L2_END): add origPB, origPB, origK, lsl #4 -.L6x8_L1_BEGIN: +L(6x8_L1_BEGIN): tst origN, #1 - beq .L6x8_L999 + beq L(6x8_L999) mov pCRow0, pC mov pA, origPA mov temp, #6 udiv counterI, origM, temp - cbz counterI, .L6x8_L1_Mrem -.L6x8_L1_M6: + cbz counterI, L(6x8_L1_Mrem) +L(6x8_L1_M6): INIT6x1 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L1_M6s -.L6x8_L1_M6l: + cbz counterL, L(6x8_L1_M6s) +L(6x8_L1_M6l): KERNEL6x1_SUB subs counterL, counterL, #1 - bgt .L6x8_L1_M6l -.L6x8_L1_M6s: + bgt L(6x8_L1_M6l) +L(6x8_L1_M6s): SAVE6x1 subs counterI, counterI, #1 - bgt .L6x8_L1_M6 -.L6x8_L1_Mrem: + bgt L(6x8_L1_M6) +L(6x8_L1_Mrem): mov temp, #6 udiv counterI, origM, temp msub counterI, counterI, temp, origM cmp counterI, #4 - blt .L6x8_L1_M2c + blt L(6x8_L1_M2c) INIT4x1 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L1_M4s -.L6x8_L1_M4l: + cbz counterL, L(6x8_L1_M4s) +L(6x8_L1_M4l): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .L6x8_L1_M4l -.L6x8_L1_M4s: + bgt L(6x8_L1_M4l) +L(6x8_L1_M4s): SAVE4x1 sub counterI, counterI, #4 -.L6x8_L1_M2c: +L(6x8_L1_M2c): cmp counterI, #2 - blt .L6x8_L1_M1c + blt L(6x8_L1_M1c) INIT2x1 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L1_M2s -.L6x8_L1_M2l: + cbz counterL, L(6x8_L1_M2s) +L(6x8_L1_M2l): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .L6x8_L1_M2l -.L6x8_L1_M2s: + bgt L(6x8_L1_M2l) +L(6x8_L1_M2s): SAVE2x1 sub counterI, counterI, #2 -.L6x8_L1_M1c: - cbz counterI, .L6x8_L999 +L(6x8_L1_M1c): + cbz counterI, L(6x8_L999) INIT1x1 mov pB, origPB mov counterL, origK - cbz counterL, .L6x8_L1_M1s -.L6x8_L1_M1l: + cbz counterL, L(6x8_L1_M1s) +L(6x8_L1_M1l): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .L6x8_L1_M1l -.L6x8_L1_M1s: + bgt L(6x8_L1_M1l) +L(6x8_L1_M1s): SAVE1x1 -.L6x8_L999: +L(6x8_L999): mov x0, #0 ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dgemm_kernel_8x4.S b/kernel/arm64/dgemm_kernel_8x4.S index af3aa0217d..02fd16570b 100644 --- a/kernel/arm64/dgemm_kernel_8x4.S +++ b/kernel/arm64/dgemm_kernel_8x4.S @@ -885,12 +885,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Ldgemm_kernel_L2_BEGIN + ble L(dgemm_kernel_L2_BEGIN) /******************************************************************************/ .align 5 -.Ldgemm_kernel_L4_BEGIN: +L(dgemm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -900,21 +900,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L4_M8_BEGIN: +L(dgemm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Ldgemm_kernel_L4_M4_BEGIN + ble L(dgemm_kernel_L4_M4_BEGIN) .align 5 -.Ldgemm_kernel_L4_M8_20: +L(dgemm_kernel_L4_M8_20): mov pB, origPB asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Ldgemm_kernel_L4_M8_32 + blt L(dgemm_kernel_L4_M8_32) KERNEL8x4_I KERNEL8x4_M2 @@ -926,10 +926,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M2 subs counterL, counterL, #2 // subtract 2 - ble .Ldgemm_kernel_L4_M8_22a + ble L(dgemm_kernel_L4_M8_22a) .align 5 -.Ldgemm_kernel_L4_M8_22: +L(dgemm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 @@ -941,10 +941,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M8_22 + bgt L(dgemm_kernel_L4_M8_22) .align 5 -.Ldgemm_kernel_L4_M8_22a: +L(dgemm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_M2 @@ -955,13 +955,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Ldgemm_kernel_L4_M8_44 + b L(dgemm_kernel_L4_M8_44) .align 5 -.Ldgemm_kernel_L4_M8_32: +L(dgemm_kernel_L4_M8_32): tst counterL, #1 - ble .Ldgemm_kernel_L4_M8_40 + ble L(dgemm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_M2 @@ -972,46 +972,46 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Ldgemm_kernel_L4_M8_44 + b L(dgemm_kernel_L4_M8_44) -.Ldgemm_kernel_L4_M8_40: +L(dgemm_kernel_L4_M8_40): INIT8x4 -.Ldgemm_kernel_L4_M8_44: +L(dgemm_kernel_L4_M8_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L4_M8_100 + ble L(dgemm_kernel_L4_M8_100) .align 5 -.Ldgemm_kernel_L4_M8_46: +L(dgemm_kernel_L4_M8_46): KERNEL8x4_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L4_M8_46 + bne L(dgemm_kernel_L4_M8_46) -.Ldgemm_kernel_L4_M8_100: +L(dgemm_kernel_L4_M8_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVE8x4 -.Ldgemm_kernel_L4_M8_END: +L(dgemm_kernel_L4_M8_END): subs counterI, counterI, #1 - bne .Ldgemm_kernel_L4_M8_20 + bne L(dgemm_kernel_L4_M8_20) -.Ldgemm_kernel_L4_M4_BEGIN: +L(dgemm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Ldgemm_kernel_L4_END + ble L(dgemm_kernel_L4_END) tst counterI, #4 - ble .Ldgemm_kernel_L4_M2_BEGIN + ble L(dgemm_kernel_L4_M2_BEGIN) -.Ldgemm_kernel_L4_M4_20: +L(dgemm_kernel_L4_M4_20): INIT4x4 @@ -1019,10 +1019,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L4_M4_40 + ble L(dgemm_kernel_L4_M4_40) .align 5 -.Ldgemm_kernel_L4_M4_22: +L(dgemm_kernel_L4_M4_22): KERNEL4x4_SUB prfm PLDL1KEEP, [pB, #B_PRE_SIZE] @@ -1043,38 +1043,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prfm PLDL1KEEP, [pA, #A_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M4_22 + bgt L(dgemm_kernel_L4_M4_22) -.Ldgemm_kernel_L4_M4_40: +L(dgemm_kernel_L4_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L4_M4_100 + ble L(dgemm_kernel_L4_M4_100) -.Ldgemm_kernel_L4_M4_42: +L(dgemm_kernel_L4_M4_42): KERNEL4x4_SUB prfm PLDL1KEEP, [pB, #B_PRE_SIZE] prfm PLDL1KEEP, [pA, #A_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M4_42 + bgt L(dgemm_kernel_L4_M4_42) -.Ldgemm_kernel_L4_M4_100: +L(dgemm_kernel_L4_M4_100): SAVE4x4 -.Ldgemm_kernel_L4_M4_END: +L(dgemm_kernel_L4_M4_END): -.Ldgemm_kernel_L4_M2_BEGIN: +L(dgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L4_END + ble L(dgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L4_M1_BEGIN + ble L(dgemm_kernel_L4_M1_BEGIN) -.Ldgemm_kernel_L4_M2_20: +L(dgemm_kernel_L4_M2_20): INIT2x4 @@ -1082,10 +1082,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L4_M2_40 + ble L(dgemm_kernel_L4_M2_40) .align 5 -.Ldgemm_kernel_L4_M2_22: +L(dgemm_kernel_L4_M2_22): KERNEL2x4_SUB prfm PLDL1KEEP, [pB, #B_PRE_SIZE] @@ -1104,37 +1104,37 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M2_22 + bgt L(dgemm_kernel_L4_M2_22) -.Ldgemm_kernel_L4_M2_40: +L(dgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L4_M2_100 + ble L(dgemm_kernel_L4_M2_100) prfm PLDL1KEEP, [pA, #A_PRE_SIZE] prfm PLDL1KEEP, [pA, #A_PRE_SIZE+64] -.Ldgemm_kernel_L4_M2_42: +L(dgemm_kernel_L4_M2_42): KERNEL2x4_SUB prfm PLDL1KEEP, [pB, #B_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M2_42 + bgt L(dgemm_kernel_L4_M2_42) -.Ldgemm_kernel_L4_M2_100: +L(dgemm_kernel_L4_M2_100): SAVE2x4 -.Ldgemm_kernel_L4_M2_END: +L(dgemm_kernel_L4_M2_END): -.Ldgemm_kernel_L4_M1_BEGIN: +L(dgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L4_END + ble L(dgemm_kernel_L4_END) -.Ldgemm_kernel_L4_M1_20: +L(dgemm_kernel_L4_M1_20): INIT1x4 @@ -1142,10 +1142,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L4_M1_40 + ble L(dgemm_kernel_L4_M1_40) .align 5 -.Ldgemm_kernel_L4_M1_22: +L(dgemm_kernel_L4_M1_22): KERNEL1x4_SUB prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNEL1x4_SUB @@ -1163,46 +1163,46 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M1_22 + bgt L(dgemm_kernel_L4_M1_22) -.Ldgemm_kernel_L4_M1_40: +L(dgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L4_M1_100 + ble L(dgemm_kernel_L4_M1_100) prfm PLDL1KEEP, [pA, #A_PRE_SIZE] -.Ldgemm_kernel_L4_M1_42: +L(dgemm_kernel_L4_M1_42): KERNEL1x4_SUB prfm PLDL1KEEP, [pB, #B_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M1_42 + bgt L(dgemm_kernel_L4_M1_42) -.Ldgemm_kernel_L4_M1_100: +L(dgemm_kernel_L4_M1_100): SAVE1x4 -.Ldgemm_kernel_L4_END: +L(dgemm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 subs counterJ, counterJ , #1 // j-- - bgt .Ldgemm_kernel_L4_BEGIN + bgt L(dgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Ldgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(dgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Ldgemm_kernel_L999 // error, N was less than 4? + ble L(dgemm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Ldgemm_kernel_L1_BEGIN + ble L(dgemm_kernel_L1_BEGIN) mov pCRow0, pC add pCRow1, pCRow0, LDC @@ -1211,15 +1211,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Ldgemm_kernel_L2_M8_BEGIN: +L(dgemm_kernel_L2_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Ldgemm_kernel_L2_M4_BEGIN + ble L(dgemm_kernel_L2_M4_BEGIN) .align 5 -.Ldgemm_kernel_L2_M8_20: +L(dgemm_kernel_L2_M8_20): INIT8x2 @@ -1227,10 +1227,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldgemm_kernel_L2_M8_40 + ble L(dgemm_kernel_L2_M8_40) .align 5 -.Ldgemm_kernel_L2_M8_22: +L(dgemm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB prfm PLDL1KEEP, [pB, #B_PRE_SIZE] @@ -1244,41 +1244,41 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M8_22 + bgt L(dgemm_kernel_L2_M8_22) -.Ldgemm_kernel_L2_M8_40: +L(dgemm_kernel_L2_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M8_100 + ble L(dgemm_kernel_L2_M8_100) prfm PLDL1KEEP, [pB, #B_PRE_SIZE] prfm PLDL1KEEP, [pB, #B_PRE_SIZE+64] -.Ldgemm_kernel_L2_M8_42: +L(dgemm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M8_42 + bgt L(dgemm_kernel_L2_M8_42) -.Ldgemm_kernel_L2_M8_100: +L(dgemm_kernel_L2_M8_100): SAVE8x2 -.Ldgemm_kernel_L2_M8_END: +L(dgemm_kernel_L2_M8_END): subs counterI, counterI, #1 - bgt .Ldgemm_kernel_L2_M8_20 + bgt L(dgemm_kernel_L2_M8_20) -.Ldgemm_kernel_L2_M4_BEGIN: +L(dgemm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Ldgemm_kernel_L2_END + ble L(dgemm_kernel_L2_END) tst counterI, #4 // counterI = counterI / 2 - ble .Ldgemm_kernel_L2_M2_BEGIN + ble L(dgemm_kernel_L2_M2_BEGIN) -.Ldgemm_kernel_L2_M4_20: +L(dgemm_kernel_L2_M4_20): INIT4x2 @@ -1286,10 +1286,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldgemm_kernel_L2_M4_40 + ble L(dgemm_kernel_L2_M4_40) .align 5 -.Ldgemm_kernel_L2_M4_22: +L(dgemm_kernel_L2_M4_22): KERNEL4x2_SUB prfm PLDL1KEEP, [pA, #A_PRE_SIZE] KERNEL4x2_SUB @@ -1307,41 +1307,41 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M4_22 + bgt L(dgemm_kernel_L2_M4_22) -.Ldgemm_kernel_L2_M4_40: +L(dgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M4_100 + ble L(dgemm_kernel_L2_M4_100) prfm PLDL1KEEP, [pB, #B_PRE_SIZE] prfm PLDL1KEEP, [pB, #B_PRE_SIZE+64] -.Ldgemm_kernel_L2_M4_42: +L(dgemm_kernel_L2_M4_42): KERNEL4x2_SUB prfm PLDL1KEEP, [pA, #A_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M4_42 + bgt L(dgemm_kernel_L2_M4_42) -.Ldgemm_kernel_L2_M4_100: +L(dgemm_kernel_L2_M4_100): SAVE4x2 -.Ldgemm_kernel_L2_M4_END: +L(dgemm_kernel_L2_M4_END): -.Ldgemm_kernel_L2_M2_BEGIN: +L(dgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L2_END + ble L(dgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L2_M1_BEGIN + ble L(dgemm_kernel_L2_M1_BEGIN) -.Ldgemm_kernel_L2_M2_20: +L(dgemm_kernel_L2_M2_20): INIT2x2 @@ -1349,9 +1349,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldgemm_kernel_L2_M2_40 + ble L(dgemm_kernel_L2_M2_40) -.Ldgemm_kernel_L2_M2_22: +L(dgemm_kernel_L2_M2_22): KERNEL2x2_SUB prfm PLDL1KEEP, [pB, #B_PRE_SIZE] @@ -1368,37 +1368,37 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M2_22 + bgt L(dgemm_kernel_L2_M2_22) prfm PLDL1KEEP, [pA, #A_PRE_SIZE] prfm PLDL1KEEP, [pA, #A_PRE_SIZE+64] prfm PLDL1KEEP, [pB, #B_PRE_SIZE] prfm PLDL1KEEP, [pB, #B_PRE_SIZE+64] -.Ldgemm_kernel_L2_M2_40: +L(dgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M2_100 + ble L(dgemm_kernel_L2_M2_100) -.Ldgemm_kernel_L2_M2_42: +L(dgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M2_42 + bgt L(dgemm_kernel_L2_M2_42) -.Ldgemm_kernel_L2_M2_100: +L(dgemm_kernel_L2_M2_100): SAVE2x2 -.Ldgemm_kernel_L2_M2_END: +L(dgemm_kernel_L2_M2_END): -.Ldgemm_kernel_L2_M1_BEGIN: +L(dgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L2_END + ble L(dgemm_kernel_L2_END) -.Ldgemm_kernel_L2_M1_20: +L(dgemm_kernel_L2_M1_20): INIT1x2 @@ -1406,9 +1406,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Ldgemm_kernel_L2_M1_40 + ble L(dgemm_kernel_L2_M1_40) -.Ldgemm_kernel_L2_M1_22: +L(dgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB prfm PLDL1KEEP, [pB, #B_PRE_SIZE] @@ -1424,62 +1424,62 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M1_22 + bgt L(dgemm_kernel_L2_M1_22) prfm PLDL1KEEP, [pA, #A_PRE_SIZE] prfm PLDL1KEEP, [pB, #B_PRE_SIZE] prfm PLDL1KEEP, [pB, #B_PRE_SIZE+64] -.Ldgemm_kernel_L2_M1_40: +L(dgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M1_100 + ble L(dgemm_kernel_L2_M1_100) -.Ldgemm_kernel_L2_M1_42: +L(dgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M1_42 + bgt L(dgemm_kernel_L2_M1_42) -.Ldgemm_kernel_L2_M1_100: +L(dgemm_kernel_L2_M1_100): SAVE1x2 -.Ldgemm_kernel_L2_END: +L(dgemm_kernel_L2_END): add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 /******************************************************************************/ -.Ldgemm_kernel_L1_BEGIN: +L(dgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldgemm_kernel_L999 // done + ble L(dgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C add pC , pC , LDC // Update pC to point to next mov pA, origPA // pA = A -.Ldgemm_kernel_L1_M8_BEGIN: +L(dgemm_kernel_L1_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Ldgemm_kernel_L1_M4_BEGIN + ble L(dgemm_kernel_L1_M4_BEGIN) .align 5 -.Ldgemm_kernel_L1_M8_20: +L(dgemm_kernel_L1_M8_20): INIT8x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M8_40 + ble L(dgemm_kernel_L1_M8_40) .align 5 -.Ldgemm_kernel_L1_M8_22: +L(dgemm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -1493,51 +1493,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M8_22 + bgt L(dgemm_kernel_L1_M8_22) -.Ldgemm_kernel_L1_M8_40: +L(dgemm_kernel_L1_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M8_100 + ble L(dgemm_kernel_L1_M8_100) prfm PLDL1KEEP, [pB, #B_PRE_SIZE] -.Ldgemm_kernel_L1_M8_42: +L(dgemm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M8_42 + bgt L(dgemm_kernel_L1_M8_42) -.Ldgemm_kernel_L1_M8_100: +L(dgemm_kernel_L1_M8_100): SAVE8x1 -.Ldgemm_kernel_L1_M8_END: +L(dgemm_kernel_L1_M8_END): subs counterI, counterI, #1 - bgt .Ldgemm_kernel_L1_M8_20 + bgt L(dgemm_kernel_L1_M8_20) -.Ldgemm_kernel_L1_M4_BEGIN: +L(dgemm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Ldgemm_kernel_L1_END + ble L(dgemm_kernel_L1_END) tst counterI, #4 // counterI = counterI / 2 - ble .Ldgemm_kernel_L1_M2_BEGIN + ble L(dgemm_kernel_L1_M2_BEGIN) -.Ldgemm_kernel_L1_M4_20: +L(dgemm_kernel_L1_M4_20): INIT4x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M4_40 + ble L(dgemm_kernel_L1_M4_40) .align 5 -.Ldgemm_kernel_L1_M4_22: +L(dgemm_kernel_L1_M4_22): KERNEL4x1_SUB prfm PLDL1KEEP, [pA, #A_PRE_SIZE] KERNEL4x1_SUB @@ -1555,39 +1555,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M4_22 + bgt L(dgemm_kernel_L1_M4_22) -.Ldgemm_kernel_L1_M4_40: +L(dgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M4_100 + ble L(dgemm_kernel_L1_M4_100) prfm PLDL1KEEP, [pB, #B_PRE_SIZE] -.Ldgemm_kernel_L1_M4_42: +L(dgemm_kernel_L1_M4_42): KERNEL4x1_SUB prfm PLDL1KEEP, [pA, #A_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M4_42 + bgt L(dgemm_kernel_L1_M4_42) -.Ldgemm_kernel_L1_M4_100: +L(dgemm_kernel_L1_M4_100): SAVE4x1 -.Ldgemm_kernel_L1_M4_END: +L(dgemm_kernel_L1_M4_END): -.Ldgemm_kernel_L1_M2_BEGIN: +L(dgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L1_END + ble L(dgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L1_M1_BEGIN + ble L(dgemm_kernel_L1_M1_BEGIN) -.Ldgemm_kernel_L1_M2_20: +L(dgemm_kernel_L1_M2_20): INIT2x1 @@ -1595,9 +1595,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M2_40 + ble L(dgemm_kernel_L1_M2_40) -.Ldgemm_kernel_L1_M2_22: +L(dgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1614,36 +1614,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M2_22 + bgt L(dgemm_kernel_L1_M2_22) prfm PLDL1KEEP, [pA, #A_PRE_SIZE] prfm PLDL1KEEP, [pA, #A_PRE_SIZE+64] prfm PLDL1KEEP, [pB, #B_PRE_SIZE] -.Ldgemm_kernel_L1_M2_40: +L(dgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M2_100 + ble L(dgemm_kernel_L1_M2_100) -.Ldgemm_kernel_L1_M2_42: +L(dgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M2_42 + bgt L(dgemm_kernel_L1_M2_42) -.Ldgemm_kernel_L1_M2_100: +L(dgemm_kernel_L1_M2_100): SAVE2x1 -.Ldgemm_kernel_L1_M2_END: +L(dgemm_kernel_L1_M2_END): -.Ldgemm_kernel_L1_M1_BEGIN: +L(dgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L1_END + ble L(dgemm_kernel_L1_END) -.Ldgemm_kernel_L1_M1_20: +L(dgemm_kernel_L1_M1_20): INIT1x1 @@ -1651,10 +1651,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M1_40 + ble L(dgemm_kernel_L1_M1_40) -.Ldgemm_kernel_L1_M1_22: +L(dgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB prfm PLDL1KEEP, [pA, #A_PRE_SIZE] @@ -1668,32 +1668,32 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M1_22 + bgt L(dgemm_kernel_L1_M1_22) -.Ldgemm_kernel_L1_M1_40: +L(dgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M1_100 + ble L(dgemm_kernel_L1_M1_100) prfm PLDL1KEEP, [pA, #A_PRE_SIZE] prfm PLDL1KEEP, [pB, #B_PRE_SIZE] -.Ldgemm_kernel_L1_M1_42: +L(dgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M1_42 + bgt L(dgemm_kernel_L1_M1_42) -.Ldgemm_kernel_L1_M1_100: +L(dgemm_kernel_L1_M1_100): SAVE1x1 -.Ldgemm_kernel_L1_END: +L(dgemm_kernel_L1_END): -.Ldgemm_kernel_L999: +L(dgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dgemm_kernel_8x4_thunderx2t99.S b/kernel/arm64/dgemm_kernel_8x4_thunderx2t99.S index d1551ffeab..7525ee1078 100644 --- a/kernel/arm64/dgemm_kernel_8x4_thunderx2t99.S +++ b/kernel/arm64/dgemm_kernel_8x4_thunderx2t99.S @@ -958,12 +958,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Ldgemm_kernel_L2_BEGIN + ble L(dgemm_kernel_L2_BEGIN) /******************************************************************************/ .align 5 -.Ldgemm_kernel_L4_BEGIN: +L(dgemm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -973,21 +973,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L4_M8_BEGIN: +L(dgemm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Ldgemm_kernel_L4_M4_BEGIN + ble L(dgemm_kernel_L4_M4_BEGIN) .align 5 -.Ldgemm_kernel_L4_M8_20: +L(dgemm_kernel_L4_M8_20): mov pB, origPB asr counterL , origK, #7 // L = K / 128 cmp counterL , #2 // is there at least 4 to do? - blt .Ldgemm_kernel_L4_M8_32 + blt L(dgemm_kernel_L4_M8_32) KERNEL8x4_I KERNEL8x4_M2 @@ -999,18 +999,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1_M2_x1 subs counterL, counterL, #2 // subtract 2 - ble .Ldgemm_kernel_L4_M8_22a + ble L(dgemm_kernel_L4_M8_22a) .align 5 -.Ldgemm_kernel_L4_M8_22: +L(dgemm_kernel_L4_M8_22): KERNEL8x4_M1_M2_x64 subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M8_22 + bgt L(dgemm_kernel_L4_M8_22) .align 5 -.Ldgemm_kernel_L4_M8_22a: +L(dgemm_kernel_L4_M8_22a): KERNEL8x4_M1_M2_x32 KERNEL8x4_M1_M2_x16 @@ -1021,13 +1021,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Ldgemm_kernel_L4_M8_44 + b L(dgemm_kernel_L4_M8_44) .align 5 -.Ldgemm_kernel_L4_M8_32: +L(dgemm_kernel_L4_M8_32): tst counterL, #1 - ble .Ldgemm_kernel_L4_M8_40 + ble L(dgemm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_M2 @@ -1039,26 +1039,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Ldgemm_kernel_L4_M8_44 + b L(dgemm_kernel_L4_M8_44) -.Ldgemm_kernel_L4_M8_40: +L(dgemm_kernel_L4_M8_40): INIT8x4 -.Ldgemm_kernel_L4_M8_44: +L(dgemm_kernel_L4_M8_44): ands counterL , origK, #127 - ble .Ldgemm_kernel_L4_M8_100 + ble L(dgemm_kernel_L4_M8_100) .align 5 -.Ldgemm_kernel_L4_M8_46: +L(dgemm_kernel_L4_M8_46): KERNEL8x4_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L4_M8_46 + bne L(dgemm_kernel_L4_M8_46) -.Ldgemm_kernel_L4_M8_100: +L(dgemm_kernel_L4_M8_100): prfm PLDL2KEEP, [pCRow0, C_PRE_SIZE] prfm PLDL2KEEP, [pCRow1, C_PRE_SIZE] prfm PLDL2KEEP, [pCRow2, C_PRE_SIZE] @@ -1069,20 +1069,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SAVE8x4 -.Ldgemm_kernel_L4_M8_END: +L(dgemm_kernel_L4_M8_END): subs counterI, counterI, #1 - bne .Ldgemm_kernel_L4_M8_20 + bne L(dgemm_kernel_L4_M8_20) -.Ldgemm_kernel_L4_M4_BEGIN: +L(dgemm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Ldgemm_kernel_L4_END + ble L(dgemm_kernel_L4_END) tst counterI, #4 - ble .Ldgemm_kernel_L4_M2_BEGIN + ble L(dgemm_kernel_L4_M2_BEGIN) -.Ldgemm_kernel_L4_M4_20: +L(dgemm_kernel_L4_M4_20): INIT4x4 @@ -1090,10 +1090,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L4_M4_40 + ble L(dgemm_kernel_L4_M4_40) .align 5 -.Ldgemm_kernel_L4_M4_22: +L(dgemm_kernel_L4_M4_22): KERNEL4x4_SUB prfm PLDL1KEEP, [pB, B_PRE_SIZE] @@ -1114,38 +1114,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prfm PLDL1KEEP, [pA, A_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M4_22 + bgt L(dgemm_kernel_L4_M4_22) -.Ldgemm_kernel_L4_M4_40: +L(dgemm_kernel_L4_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L4_M4_100 + ble L(dgemm_kernel_L4_M4_100) -.Ldgemm_kernel_L4_M4_42: +L(dgemm_kernel_L4_M4_42): KERNEL4x4_SUB prfm PLDL1KEEP, [pB, B_PRE_SIZE] prfm PLDL1KEEP, [pA, A_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M4_42 + bgt L(dgemm_kernel_L4_M4_42) -.Ldgemm_kernel_L4_M4_100: +L(dgemm_kernel_L4_M4_100): SAVE4x4 -.Ldgemm_kernel_L4_M4_END: +L(dgemm_kernel_L4_M4_END): -.Ldgemm_kernel_L4_M2_BEGIN: +L(dgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L4_END + ble L(dgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L4_M1_BEGIN + ble L(dgemm_kernel_L4_M1_BEGIN) -.Ldgemm_kernel_L4_M2_20: +L(dgemm_kernel_L4_M2_20): INIT2x4 @@ -1153,10 +1153,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L4_M2_40 + ble L(dgemm_kernel_L4_M2_40) .align 5 -.Ldgemm_kernel_L4_M2_22: +L(dgemm_kernel_L4_M2_22): KERNEL2x4_SUB prfm PLDL1KEEP, [pB, B_PRE_SIZE] @@ -1175,37 +1175,37 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M2_22 + bgt L(dgemm_kernel_L4_M2_22) -.Ldgemm_kernel_L4_M2_40: +L(dgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L4_M2_100 + ble L(dgemm_kernel_L4_M2_100) prfm PLDL1KEEP, [pA, A_PRE_SIZE] prfm PLDL1KEEP, [pA, A_PRE_SIZE_64] -.Ldgemm_kernel_L4_M2_42: +L(dgemm_kernel_L4_M2_42): KERNEL2x4_SUB prfm PLDL1KEEP, [pB, B_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M2_42 + bgt L(dgemm_kernel_L4_M2_42) -.Ldgemm_kernel_L4_M2_100: +L(dgemm_kernel_L4_M2_100): SAVE2x4 -.Ldgemm_kernel_L4_M2_END: +L(dgemm_kernel_L4_M2_END): -.Ldgemm_kernel_L4_M1_BEGIN: +L(dgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L4_END + ble L(dgemm_kernel_L4_END) -.Ldgemm_kernel_L4_M1_20: +L(dgemm_kernel_L4_M1_20): INIT1x4 @@ -1213,10 +1213,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L4_M1_40 + ble L(dgemm_kernel_L4_M1_40) .align 5 -.Ldgemm_kernel_L4_M1_22: +L(dgemm_kernel_L4_M1_22): KERNEL1x4_SUB prfm PLDL1KEEP, [pB, B_PRE_SIZE] KERNEL1x4_SUB @@ -1234,46 +1234,46 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M1_22 + bgt L(dgemm_kernel_L4_M1_22) -.Ldgemm_kernel_L4_M1_40: +L(dgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L4_M1_100 + ble L(dgemm_kernel_L4_M1_100) prfm PLDL1KEEP, [pA, A_PRE_SIZE] -.Ldgemm_kernel_L4_M1_42: +L(dgemm_kernel_L4_M1_42): KERNEL1x4_SUB prfm PLDL1KEEP, [pB, B_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_M1_42 + bgt L(dgemm_kernel_L4_M1_42) -.Ldgemm_kernel_L4_M1_100: +L(dgemm_kernel_L4_M1_100): SAVE1x4 -.Ldgemm_kernel_L4_END: +L(dgemm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 subs counterJ, counterJ , #1 // j-- - bgt .Ldgemm_kernel_L4_BEGIN + bgt L(dgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Ldgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(dgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Ldgemm_kernel_L999 // error, N was less than 4? + ble L(dgemm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Ldgemm_kernel_L1_BEGIN + ble L(dgemm_kernel_L1_BEGIN) mov pCRow0, pC add pCRow1, pCRow0, LDC @@ -1282,15 +1282,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Ldgemm_kernel_L2_M8_BEGIN: +L(dgemm_kernel_L2_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Ldgemm_kernel_L2_M4_BEGIN + ble L(dgemm_kernel_L2_M4_BEGIN) .align 5 -.Ldgemm_kernel_L2_M8_20: +L(dgemm_kernel_L2_M8_20): INIT8x2 @@ -1298,10 +1298,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldgemm_kernel_L2_M8_40 + ble L(dgemm_kernel_L2_M8_40) .align 5 -.Ldgemm_kernel_L2_M8_22: +L(dgemm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB prfm PLDL1KEEP, [pB, B_PRE_SIZE] @@ -1315,41 +1315,41 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M8_22 + bgt L(dgemm_kernel_L2_M8_22) -.Ldgemm_kernel_L2_M8_40: +L(dgemm_kernel_L2_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M8_100 + ble L(dgemm_kernel_L2_M8_100) prfm PLDL1KEEP, [pB, B_PRE_SIZE] prfm PLDL1KEEP, [pB, B_PRE_SIZE_64] -.Ldgemm_kernel_L2_M8_42: +L(dgemm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M8_42 + bgt L(dgemm_kernel_L2_M8_42) -.Ldgemm_kernel_L2_M8_100: +L(dgemm_kernel_L2_M8_100): SAVE8x2 -.Ldgemm_kernel_L2_M8_END: +L(dgemm_kernel_L2_M8_END): subs counterI, counterI, #1 - bgt .Ldgemm_kernel_L2_M8_20 + bgt L(dgemm_kernel_L2_M8_20) -.Ldgemm_kernel_L2_M4_BEGIN: +L(dgemm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Ldgemm_kernel_L2_END + ble L(dgemm_kernel_L2_END) tst counterI, #4 // counterI = counterI / 2 - ble .Ldgemm_kernel_L2_M2_BEGIN + ble L(dgemm_kernel_L2_M2_BEGIN) -.Ldgemm_kernel_L2_M4_20: +L(dgemm_kernel_L2_M4_20): INIT4x2 @@ -1357,10 +1357,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldgemm_kernel_L2_M4_40 + ble L(dgemm_kernel_L2_M4_40) .align 5 -.Ldgemm_kernel_L2_M4_22: +L(dgemm_kernel_L2_M4_22): KERNEL4x2_SUB prfm PLDL1KEEP, [pA, A_PRE_SIZE] KERNEL4x2_SUB @@ -1378,41 +1378,41 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M4_22 + bgt L(dgemm_kernel_L2_M4_22) -.Ldgemm_kernel_L2_M4_40: +L(dgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M4_100 + ble L(dgemm_kernel_L2_M4_100) prfm PLDL1KEEP, [pB, B_PRE_SIZE] prfm PLDL1KEEP, [pB, B_PRE_SIZE_64] -.Ldgemm_kernel_L2_M4_42: +L(dgemm_kernel_L2_M4_42): KERNEL4x2_SUB prfm PLDL1KEEP, [pA, A_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M4_42 + bgt L(dgemm_kernel_L2_M4_42) -.Ldgemm_kernel_L2_M4_100: +L(dgemm_kernel_L2_M4_100): SAVE4x2 -.Ldgemm_kernel_L2_M4_END: +L(dgemm_kernel_L2_M4_END): -.Ldgemm_kernel_L2_M2_BEGIN: +L(dgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L2_END + ble L(dgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L2_M1_BEGIN + ble L(dgemm_kernel_L2_M1_BEGIN) -.Ldgemm_kernel_L2_M2_20: +L(dgemm_kernel_L2_M2_20): INIT2x2 @@ -1420,9 +1420,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldgemm_kernel_L2_M2_40 + ble L(dgemm_kernel_L2_M2_40) -.Ldgemm_kernel_L2_M2_22: +L(dgemm_kernel_L2_M2_22): KERNEL2x2_SUB prfm PLDL1KEEP, [pB, B_PRE_SIZE] @@ -1439,37 +1439,37 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M2_22 + bgt L(dgemm_kernel_L2_M2_22) prfm PLDL1KEEP, [pA, A_PRE_SIZE] prfm PLDL1KEEP, [pA, A_PRE_SIZE_64] prfm PLDL1KEEP, [pB, B_PRE_SIZE] prfm PLDL1KEEP, [pB, B_PRE_SIZE_64] -.Ldgemm_kernel_L2_M2_40: +L(dgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M2_100 + ble L(dgemm_kernel_L2_M2_100) -.Ldgemm_kernel_L2_M2_42: +L(dgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M2_42 + bgt L(dgemm_kernel_L2_M2_42) -.Ldgemm_kernel_L2_M2_100: +L(dgemm_kernel_L2_M2_100): SAVE2x2 -.Ldgemm_kernel_L2_M2_END: +L(dgemm_kernel_L2_M2_END): -.Ldgemm_kernel_L2_M1_BEGIN: +L(dgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L2_END + ble L(dgemm_kernel_L2_END) -.Ldgemm_kernel_L2_M1_20: +L(dgemm_kernel_L2_M1_20): INIT1x2 @@ -1477,9 +1477,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Ldgemm_kernel_L2_M1_40 + ble L(dgemm_kernel_L2_M1_40) -.Ldgemm_kernel_L2_M1_22: +L(dgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB prfm PLDL1KEEP, [pB, B_PRE_SIZE] @@ -1495,62 +1495,62 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M1_22 + bgt L(dgemm_kernel_L2_M1_22) prfm PLDL1KEEP, [pA, A_PRE_SIZE] prfm PLDL1KEEP, [pB, B_PRE_SIZE] prfm PLDL1KEEP, [pB, B_PRE_SIZE_64] -.Ldgemm_kernel_L2_M1_40: +L(dgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L2_M1_100 + ble L(dgemm_kernel_L2_M1_100) -.Ldgemm_kernel_L2_M1_42: +L(dgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_M1_42 + bgt L(dgemm_kernel_L2_M1_42) -.Ldgemm_kernel_L2_M1_100: +L(dgemm_kernel_L2_M1_100): SAVE1x2 -.Ldgemm_kernel_L2_END: +L(dgemm_kernel_L2_END): add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 /******************************************************************************/ -.Ldgemm_kernel_L1_BEGIN: +L(dgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldgemm_kernel_L999 // done + ble L(dgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C add pC , pC , LDC // Update pC to point to next mov pA, origPA // pA = A -.Ldgemm_kernel_L1_M8_BEGIN: +L(dgemm_kernel_L1_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Ldgemm_kernel_L1_M4_BEGIN + ble L(dgemm_kernel_L1_M4_BEGIN) .align 5 -.Ldgemm_kernel_L1_M8_20: +L(dgemm_kernel_L1_M8_20): INIT8x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M8_40 + ble L(dgemm_kernel_L1_M8_40) .align 5 -.Ldgemm_kernel_L1_M8_22: +L(dgemm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -1564,51 +1564,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M8_22 + bgt L(dgemm_kernel_L1_M8_22) -.Ldgemm_kernel_L1_M8_40: +L(dgemm_kernel_L1_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M8_100 + ble L(dgemm_kernel_L1_M8_100) prfm PLDL1KEEP, [pB, B_PRE_SIZE] -.Ldgemm_kernel_L1_M8_42: +L(dgemm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M8_42 + bgt L(dgemm_kernel_L1_M8_42) -.Ldgemm_kernel_L1_M8_100: +L(dgemm_kernel_L1_M8_100): SAVE8x1 -.Ldgemm_kernel_L1_M8_END: +L(dgemm_kernel_L1_M8_END): subs counterI, counterI, #1 - bgt .Ldgemm_kernel_L1_M8_20 + bgt L(dgemm_kernel_L1_M8_20) -.Ldgemm_kernel_L1_M4_BEGIN: +L(dgemm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Ldgemm_kernel_L1_END + ble L(dgemm_kernel_L1_END) tst counterI, #4 // counterI = counterI / 2 - ble .Ldgemm_kernel_L1_M2_BEGIN + ble L(dgemm_kernel_L1_M2_BEGIN) -.Ldgemm_kernel_L1_M4_20: +L(dgemm_kernel_L1_M4_20): INIT4x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M4_40 + ble L(dgemm_kernel_L1_M4_40) .align 5 -.Ldgemm_kernel_L1_M4_22: +L(dgemm_kernel_L1_M4_22): KERNEL4x1_SUB prfm PLDL1KEEP, [pA, A_PRE_SIZE] KERNEL4x1_SUB @@ -1626,39 +1626,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M4_22 + bgt L(dgemm_kernel_L1_M4_22) -.Ldgemm_kernel_L1_M4_40: +L(dgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M4_100 + ble L(dgemm_kernel_L1_M4_100) prfm PLDL1KEEP, [pB, B_PRE_SIZE] -.Ldgemm_kernel_L1_M4_42: +L(dgemm_kernel_L1_M4_42): KERNEL4x1_SUB prfm PLDL1KEEP, [pA, A_PRE_SIZE] subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M4_42 + bgt L(dgemm_kernel_L1_M4_42) -.Ldgemm_kernel_L1_M4_100: +L(dgemm_kernel_L1_M4_100): SAVE4x1 -.Ldgemm_kernel_L1_M4_END: +L(dgemm_kernel_L1_M4_END): -.Ldgemm_kernel_L1_M2_BEGIN: +L(dgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldgemm_kernel_L1_END + ble L(dgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldgemm_kernel_L1_M1_BEGIN + ble L(dgemm_kernel_L1_M1_BEGIN) -.Ldgemm_kernel_L1_M2_20: +L(dgemm_kernel_L1_M2_20): INIT2x1 @@ -1666,9 +1666,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M2_40 + ble L(dgemm_kernel_L1_M2_40) -.Ldgemm_kernel_L1_M2_22: +L(dgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1685,36 +1685,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M2_22 + bgt L(dgemm_kernel_L1_M2_22) prfm PLDL1KEEP, [pA, A_PRE_SIZE] prfm PLDL1KEEP, [pA, A_PRE_SIZE_64] prfm PLDL1KEEP, [pB, B_PRE_SIZE] -.Ldgemm_kernel_L1_M2_40: +L(dgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M2_100 + ble L(dgemm_kernel_L1_M2_100) -.Ldgemm_kernel_L1_M2_42: +L(dgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M2_42 + bgt L(dgemm_kernel_L1_M2_42) -.Ldgemm_kernel_L1_M2_100: +L(dgemm_kernel_L1_M2_100): SAVE2x1 -.Ldgemm_kernel_L1_M2_END: +L(dgemm_kernel_L1_M2_END): -.Ldgemm_kernel_L1_M1_BEGIN: +L(dgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldgemm_kernel_L1_END + ble L(dgemm_kernel_L1_END) -.Ldgemm_kernel_L1_M1_20: +L(dgemm_kernel_L1_M1_20): INIT1x1 @@ -1722,10 +1722,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldgemm_kernel_L1_M1_40 + ble L(dgemm_kernel_L1_M1_40) -.Ldgemm_kernel_L1_M1_22: +L(dgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB prfm PLDL1KEEP, [pA, A_PRE_SIZE] @@ -1739,32 +1739,32 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M1_22 + bgt L(dgemm_kernel_L1_M1_22) -.Ldgemm_kernel_L1_M1_40: +L(dgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Ldgemm_kernel_L1_M1_100 + ble L(dgemm_kernel_L1_M1_100) prfm PLDL1KEEP, [pA, A_PRE_SIZE] prfm PLDL1KEEP, [pB, B_PRE_SIZE] -.Ldgemm_kernel_L1_M1_42: +L(dgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_M1_42 + bgt L(dgemm_kernel_L1_M1_42) -.Ldgemm_kernel_L1_M1_100: +L(dgemm_kernel_L1_M1_100): SAVE1x1 -.Ldgemm_kernel_L1_END: +L(dgemm_kernel_L1_END): -.Ldgemm_kernel_L999: +L(dgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dgemm_kernel_inter_sve_v2x8.S b/kernel/arm64/dgemm_kernel_inter_sve_v2x8.S index dfad8dbfea..20838d13ea 100644 --- a/kernel/arm64/dgemm_kernel_inter_sve_v2x8.S +++ b/kernel/arm64/dgemm_kernel_inter_sve_v2x8.S @@ -857,38 +857,38 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Ldgemm_kernel_L4_BEGIN + ble L(dgemm_kernel_L4_BEGIN) /******************************************************************************/ /* Repeat this as long as there are 8 left in N */ .align 5 -.Ldgemm_kernel_L8_BEGIN: +L(dgemm_kernel_L8_BEGIN): mov pCRow0, pC add pC, pC, LDC, lsl #3 // add 8 x LDC mov pA1, origPA // pA1 = start of A array -.Ldgemm_kernel_L8_Mv2_BEGIN: +L(dgemm_kernel_L8_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 // Check if M < 2*SVE_LEN - blt .Ldgemm_kernel_L8_Mv1_BEGIN + blt L(dgemm_kernel_L8_Mv1_BEGIN) mov counterI, origM /* Until we have at least 2*SVE_LEN iters left in M, we do them with V2*8 kernel */ .align 5 -.Ldgemm_kernel_L8_Mv2_20: +L(dgemm_kernel_L8_Mv2_20): mov pB, origPB INITv2x8 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Ldgemm_kernel_L8_Mv2_32 + blt L(dgemm_kernel_L8_Mv2_32) KERNELv2x8_I KERNELv2x8_M2 @@ -900,10 +900,10 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Ldgemm_kernel_L8_Mv2_22a + ble L(dgemm_kernel_L8_Mv2_22a) .align 5 -.Ldgemm_kernel_L8_Mv2_22: +L(dgemm_kernel_L8_Mv2_22): KERNELv2x8_M1 KERNELv2x8_M2 @@ -915,10 +915,10 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M2 subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L8_Mv2_22 + bgt L(dgemm_kernel_L8_Mv2_22) .align 5 -.Ldgemm_kernel_L8_Mv2_22a: +L(dgemm_kernel_L8_Mv2_22a): KERNELv2x8_M1 KERNELv2x8_M2 @@ -929,13 +929,13 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M1 KERNELv2x8_E - b .Ldgemm_kernel_L8_Mv2_44 + b L(dgemm_kernel_L8_Mv2_44) .align 5 -.Ldgemm_kernel_L8_Mv2_32: +L(dgemm_kernel_L8_Mv2_32): tst counterL, #1 - ble .Ldgemm_kernel_L8_Mv2_40 + ble L(dgemm_kernel_L8_Mv2_40) KERNELv2x8_I KERNELv2x8_M2 @@ -947,54 +947,54 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_E - b .Ldgemm_kernel_L8_Mv2_44 + b L(dgemm_kernel_L8_Mv2_44) -.Ldgemm_kernel_L8_Mv2_40: +L(dgemm_kernel_L8_Mv2_40): INITv2x8 -.Ldgemm_kernel_L8_Mv2_44: +L(dgemm_kernel_L8_Mv2_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L8_Mv2_100 + ble L(dgemm_kernel_L8_Mv2_100) .align 5 -.Ldgemm_kernel_L8_Mv2_46: +L(dgemm_kernel_L8_Mv2_46): KERNELv2x8_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L8_Mv2_46 + bne L(dgemm_kernel_L8_Mv2_46) -.Ldgemm_kernel_L8_Mv2_100: +L(dgemm_kernel_L8_Mv2_100): SAVEv2x8 -.Ldgemm_kernel_L8_Mv2_END: +L(dgemm_kernel_L8_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Ldgemm_kernel_L8_Mv2_20 + bge L(dgemm_kernel_L8_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Ldgemm_kernel_L8_END + beq L(dgemm_kernel_L8_END) ////////////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x8 kernel. -.Ldgemm_kernel_L8_Mv1_BEGIN: +L(dgemm_kernel_L8_Mv1_BEGIN): whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension .align 5 -.Ldgemm_kernel_L8_Mv1_20: +L(dgemm_kernel_L8_Mv1_20): mov pB, origPB INITv1x8 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Ldgemm_kernel_L8_Mv1_32 + blt L(dgemm_kernel_L8_Mv1_32) KERNELv1x8_I KERNELv1x8_M2 @@ -1006,10 +1006,10 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Ldgemm_kernel_L8_Mv1_22a + ble L(dgemm_kernel_L8_Mv1_22a) .align 5 -.Ldgemm_kernel_L8_Mv1_22: +L(dgemm_kernel_L8_Mv1_22): KERNELv1x8_M1 KERNELv1x8_M2 @@ -1021,10 +1021,10 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M2 subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L8_Mv1_22 + bgt L(dgemm_kernel_L8_Mv1_22) .align 5 -.Ldgemm_kernel_L8_Mv1_22a: +L(dgemm_kernel_L8_Mv1_22a): KERNELv1x8_M1 KERNELv1x8_M2 @@ -1035,13 +1035,13 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M1 KERNELv1x8_E - b .Ldgemm_kernel_L8_Mv1_44 + b L(dgemm_kernel_L8_Mv1_44) .align 5 -.Ldgemm_kernel_L8_Mv1_32: +L(dgemm_kernel_L8_Mv1_32): tst counterL, #1 - ble .Ldgemm_kernel_L8_Mv1_40 + ble L(dgemm_kernel_L8_Mv1_40) KERNELv1x8_I KERNELv1x8_M2 @@ -1053,53 +1053,53 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_E - b .Ldgemm_kernel_L8_Mv1_44 + b L(dgemm_kernel_L8_Mv1_44) -.Ldgemm_kernel_L8_Mv1_40: +L(dgemm_kernel_L8_Mv1_40): INITv1x8 -.Ldgemm_kernel_L8_Mv1_44: +L(dgemm_kernel_L8_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L8_Mv1_100 + ble L(dgemm_kernel_L8_Mv1_100) .align 5 -.Ldgemm_kernel_L8_Mv1_46: +L(dgemm_kernel_L8_Mv1_46): KERNELv1x8_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L8_Mv1_46 + bne L(dgemm_kernel_L8_Mv1_46) -.Ldgemm_kernel_L8_Mv1_100: +L(dgemm_kernel_L8_Mv1_100): SAVEv1x8 -.Ldgemm_kernel_L8_Mv1_END: +L(dgemm_kernel_L8_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension - b.any .Ldgemm_kernel_L8_Mv1_20 + b.any L(dgemm_kernel_L8_Mv1_20) -.Ldgemm_kernel_L8_END: +L(dgemm_kernel_L8_END): lsl temp, origK, #6 add origPB, origPB, temp // B = B + K * 8 * 8 subs counterJ, counterJ , #1 // j-- - bgt .Ldgemm_kernel_L8_BEGIN + bgt L(dgemm_kernel_L8_BEGIN) /******************************************************************************/ /* Repeat the same thing if 4 left in N */ .align 5 -.Ldgemm_kernel_L4_BEGIN: +L(dgemm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #4 - ble .Ldgemm_kernel_L2_BEGIN + ble L(dgemm_kernel_L2_BEGIN) mov pCRow0, pC @@ -1108,26 +1108,26 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Ldgemm_kernel_L4_Mv2_BEGIN: +L(dgemm_kernel_L4_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Ldgemm_kernel_L4_Mv1_BEGIN + blt L(dgemm_kernel_L4_Mv1_BEGIN) mov counterI, origM .align 5 -.Ldgemm_kernel_L4_Mv2_20: +L(dgemm_kernel_L4_Mv2_20): mov pB, origPB INITv2x4 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L4_Mv2_44 + ble L(dgemm_kernel_L4_Mv2_44) .align 5 -.Ldgemm_kernel_L4_Mv2_22: +L(dgemm_kernel_L4_Mv2_22): KERNELv2x4_SUB KERNELv2x4_SUB @@ -1139,53 +1139,53 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_Mv2_22 + bgt L(dgemm_kernel_L4_Mv2_22) -.Ldgemm_kernel_L4_Mv2_44: +L(dgemm_kernel_L4_Mv2_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L4_Mv2_100 + ble L(dgemm_kernel_L4_Mv2_100) .align 5 -.Ldgemm_kernel_L4_Mv2_46: +L(dgemm_kernel_L4_Mv2_46): KERNELv2x4_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L4_Mv2_46 + bne L(dgemm_kernel_L4_Mv2_46) -.Ldgemm_kernel_L4_Mv2_100: +L(dgemm_kernel_L4_Mv2_100): SAVEv2x4 -.Ldgemm_kernel_L4_Mv2_END: +L(dgemm_kernel_L4_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Ldgemm_kernel_L4_Mv2_20 + bge L(dgemm_kernel_L4_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Ldgemm_kernel_L4_END + beq L(dgemm_kernel_L4_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x4 kernel. -.Ldgemm_kernel_L4_Mv1_BEGIN: +L(dgemm_kernel_L4_Mv1_BEGIN): whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension .align 5 -.Ldgemm_kernel_L4_Mv1_20: +L(dgemm_kernel_L4_Mv1_20): mov pB, origPB INITv1x4 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L4_Mv1_44 + ble L(dgemm_kernel_L4_Mv1_44) .align 5 -.Ldgemm_kernel_L4_Mv1_22: +L(dgemm_kernel_L4_Mv1_22): KERNELv1x4_SUB KERNELv1x4_SUB @@ -1197,34 +1197,34 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_Mv1_22 + bgt L(dgemm_kernel_L4_Mv1_22) -.Ldgemm_kernel_L4_Mv1_44: +L(dgemm_kernel_L4_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L4_Mv1_100 + ble L(dgemm_kernel_L4_Mv1_100) .align 5 -.Ldgemm_kernel_L4_Mv1_46: +L(dgemm_kernel_L4_Mv1_46): KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L4_Mv1_46 + bne L(dgemm_kernel_L4_Mv1_46) -.Ldgemm_kernel_L4_Mv1_100: +L(dgemm_kernel_L4_Mv1_100): SAVEv1x4 -.Ldgemm_kernel_L4_Mv1_END: +L(dgemm_kernel_L4_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldgemm_kernel_L4_Mv1_20 + b.any L(dgemm_kernel_L4_Mv1_20) -.Ldgemm_kernel_L4_END: +L(dgemm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 @@ -1232,11 +1232,11 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ /* Repeat the same thing if 2 left in N */ .align 5 -.Ldgemm_kernel_L2_BEGIN: +L(dgemm_kernel_L2_BEGIN): mov counterJ , origN tst counterJ , #2 - ble .Ldgemm_kernel_L1_BEGIN + ble L(dgemm_kernel_L1_BEGIN) mov pCRow0, pC @@ -1244,26 +1244,26 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Ldgemm_kernel_L2_Mv2_BEGIN: +L(dgemm_kernel_L2_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Ldgemm_kernel_L2_Mv1_BEGIN + blt L(dgemm_kernel_L2_Mv1_BEGIN) mov counterI, origM .align 5 -.Ldgemm_kernel_L2_Mv2_20: +L(dgemm_kernel_L2_Mv2_20): mov pB, origPB INITv2x2 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L2_Mv2_44 + ble L(dgemm_kernel_L2_Mv2_44) .align 5 -.Ldgemm_kernel_L2_Mv2_22: +L(dgemm_kernel_L2_Mv2_22): KERNELv2x2_SUB KERNELv2x2_SUB @@ -1275,54 +1275,54 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_Mv2_22 + bgt L(dgemm_kernel_L2_Mv2_22) -.Ldgemm_kernel_L2_Mv2_44: +L(dgemm_kernel_L2_Mv2_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L2_Mv2_100 + ble L(dgemm_kernel_L2_Mv2_100) .align 5 -.Ldgemm_kernel_L2_Mv2_46: +L(dgemm_kernel_L2_Mv2_46): KERNELv2x2_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L2_Mv2_46 + bne L(dgemm_kernel_L2_Mv2_46) -.Ldgemm_kernel_L2_Mv2_100: +L(dgemm_kernel_L2_Mv2_100): SAVEv2x2 -.Ldgemm_kernel_L2_Mv2_END: +L(dgemm_kernel_L2_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Ldgemm_kernel_L2_Mv2_20 + bge L(dgemm_kernel_L2_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Ldgemm_kernel_L2_END + beq L(dgemm_kernel_L2_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x2 kernel. -.Ldgemm_kernel_L2_Mv1_BEGIN: +L(dgemm_kernel_L2_Mv1_BEGIN): whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d .align 5 -.Ldgemm_kernel_L2_Mv1_20: +L(dgemm_kernel_L2_Mv1_20): mov pB, origPB INITv1x2 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L2_Mv1_44 + ble L(dgemm_kernel_L2_Mv1_44) .align 5 -.Ldgemm_kernel_L2_Mv1_22: +L(dgemm_kernel_L2_Mv1_22): KERNELv1x2_SUB KERNELv1x2_SUB @@ -1334,45 +1334,45 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_Mv1_22 + bgt L(dgemm_kernel_L2_Mv1_22) -.Ldgemm_kernel_L2_Mv1_44: +L(dgemm_kernel_L2_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L2_Mv1_100 + ble L(dgemm_kernel_L2_Mv1_100) .align 5 -.Ldgemm_kernel_L2_Mv1_46: +L(dgemm_kernel_L2_Mv1_46): KERNELv1x2_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L2_Mv1_46 + bne L(dgemm_kernel_L2_Mv1_46) -.Ldgemm_kernel_L2_Mv1_100: +L(dgemm_kernel_L2_Mv1_100): SAVEv1x2 -.Ldgemm_kernel_L2_Mv1_END: +L(dgemm_kernel_L2_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldgemm_kernel_L2_Mv1_20 + b.any L(dgemm_kernel_L2_Mv1_20) -.Ldgemm_kernel_L2_END: +L(dgemm_kernel_L2_END): add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 /******************************************************************************/ /* Repeat the same thing if 1 left in N */ .align 5 -.Ldgemm_kernel_L1_BEGIN: +L(dgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldgemm_kernel_L999 // done + ble L(dgemm_kernel_L999) // done mov pCRow0, pC @@ -1380,26 +1380,26 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Ldgemm_kernel_L1_Mv2_BEGIN: +L(dgemm_kernel_L1_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Ldgemm_kernel_L1_Mv1_BEGIN + blt L(dgemm_kernel_L1_Mv1_BEGIN) mov counterI, origM .align 5 -.Ldgemm_kernel_L1_Mv2_20: +L(dgemm_kernel_L1_Mv2_20): mov pB, origPB INITv2x1 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Ldgemm_kernel_L1_Mv2_44 + ble L(dgemm_kernel_L1_Mv2_44) .align 5 -.Ldgemm_kernel_L1_Mv2_22: +L(dgemm_kernel_L1_Mv2_22): KERNELv2x1_SUB KERNELv2x1_SUB @@ -1411,54 +1411,54 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv2_22 + bgt L(dgemm_kernel_L1_Mv2_22) -.Ldgemm_kernel_L1_Mv2_44: +L(dgemm_kernel_L1_Mv2_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L1_Mv2_100 + ble L(dgemm_kernel_L1_Mv2_100) .align 5 -.Ldgemm_kernel_L1_Mv2_46: +L(dgemm_kernel_L1_Mv2_46): KERNELv2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv2_46 + bgt L(dgemm_kernel_L1_Mv2_46) -.Ldgemm_kernel_L1_Mv2_100: +L(dgemm_kernel_L1_Mv2_100): SAVEv2x1 -.Ldgemm_kernel_L1_Mv2_END: +L(dgemm_kernel_L1_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Ldgemm_kernel_L1_Mv2_20 + bge L(dgemm_kernel_L1_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Ldgemm_kernel_L1_END + beq L(dgemm_kernel_L1_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x1 kernel. -.Ldgemm_kernel_L1_Mv1_BEGIN: +L(dgemm_kernel_L1_Mv1_BEGIN): whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d .align 5 -.Ldgemm_kernel_L1_Mv1_20: +L(dgemm_kernel_L1_Mv1_20): mov pB, origPB INITv1x1 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Ldgemm_kernel_L1_Mv1_44 + ble L(dgemm_kernel_L1_Mv1_44) .align 5 -.Ldgemm_kernel_L1_Mv1_22: +L(dgemm_kernel_L1_Mv1_22): KERNELv1x1_SUB KERNELv1x1_SUB @@ -1470,38 +1470,38 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv1_22 + bgt L(dgemm_kernel_L1_Mv1_22) -.Ldgemm_kernel_L1_Mv1_44: +L(dgemm_kernel_L1_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L1_Mv1_100 + ble L(dgemm_kernel_L1_Mv1_100) .align 5 -.Ldgemm_kernel_L1_Mv1_46: +L(dgemm_kernel_L1_Mv1_46): KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv1_46 + bgt L(dgemm_kernel_L1_Mv1_46) -.Ldgemm_kernel_L1_Mv1_100: +L(dgemm_kernel_L1_Mv1_100): SAVEv1x1 -.Ldgemm_kernel_L1_Mv1_END: +L(dgemm_kernel_L1_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldgemm_kernel_L1_Mv1_20 + b.any L(dgemm_kernel_L1_Mv1_20) -.Ldgemm_kernel_L1_END: +L(dgemm_kernel_L1_END): /******************************************************************************/ -.Ldgemm_kernel_L999: +L(dgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dgemm_kernel_sve_v1x8.S b/kernel/arm64/dgemm_kernel_sve_v1x8.S index bbbd0fd953..fd7b33b4d8 100644 --- a/kernel/arm64/dgemm_kernel_sve_v1x8.S +++ b/kernel/arm64/dgemm_kernel_sve_v1x8.S @@ -479,20 +479,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Ldgemm_kernel_L4_BEGIN + ble L(dgemm_kernel_L4_BEGIN) /******************************************************************************/ /* Repeat this as long as there are 8 left in N */ .align 5 -.Ldgemm_kernel_L8_BEGIN: +L(dgemm_kernel_L8_BEGIN): mov pCRow0, pC add pC, pC, LDC, lsl #3 // add 8 x LDC mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L8_Mv1_BEGIN: +L(dgemm_kernel_L8_Mv1_BEGIN): /* Loop over M is done in an SVE fashion. This has the benefit of the last M%SVE_LEN iterations being done in a single sweep */ mov counterI, #0 @@ -500,14 +500,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension .align 5 -.Ldgemm_kernel_L8_Mv1_20: +L(dgemm_kernel_L8_Mv1_20): mov pB, origPB INITv1x8 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Ldgemm_kernel_L8_Mv1_32 + blt L(dgemm_kernel_L8_Mv1_32) KERNELv1x8_I KERNELv1x8_M2 @@ -519,10 +519,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Ldgemm_kernel_L8_Mv1_22a + ble L(dgemm_kernel_L8_Mv1_22a) .align 5 -.Ldgemm_kernel_L8_Mv1_22: +L(dgemm_kernel_L8_Mv1_22): KERNELv1x8_M1 KERNELv1x8_M2 @@ -534,10 +534,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M2 subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L8_Mv1_22 + bgt L(dgemm_kernel_L8_Mv1_22) .align 5 -.Ldgemm_kernel_L8_Mv1_22a: +L(dgemm_kernel_L8_Mv1_22a): KERNELv1x8_M1 KERNELv1x8_M2 @@ -548,13 +548,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M1 KERNELv1x8_E - b .Ldgemm_kernel_L8_Mv1_44 + b L(dgemm_kernel_L8_Mv1_44) .align 5 -.Ldgemm_kernel_L8_Mv1_32: +L(dgemm_kernel_L8_Mv1_32): tst counterL, #1 - ble .Ldgemm_kernel_L8_Mv1_40 + ble L(dgemm_kernel_L8_Mv1_40) KERNELv1x8_I KERNELv1x8_M2 @@ -566,56 +566,56 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_E - b .Ldgemm_kernel_L8_Mv1_44 + b L(dgemm_kernel_L8_Mv1_44) -.Ldgemm_kernel_L8_Mv1_40: +L(dgemm_kernel_L8_Mv1_40): INITv1x8 -.Ldgemm_kernel_L8_Mv1_44: +L(dgemm_kernel_L8_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L8_Mv1_100 + ble L(dgemm_kernel_L8_Mv1_100) .align 5 -.Ldgemm_kernel_L8_Mv1_46: +L(dgemm_kernel_L8_Mv1_46): KERNELv1x8_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L8_Mv1_46 + bne L(dgemm_kernel_L8_Mv1_46) -.Ldgemm_kernel_L8_Mv1_100: +L(dgemm_kernel_L8_Mv1_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVEv1x8 -.Ldgemm_kernel_L8_Mv1_END: +L(dgemm_kernel_L8_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension - b.any .Ldgemm_kernel_L8_Mv1_20 + b.any L(dgemm_kernel_L8_Mv1_20) -.Ldgemm_kernel_L8_END: +L(dgemm_kernel_L8_END): lsl temp, origK, #6 add origPB, origPB, temp // B = B + K * 8 * 8 subs counterJ, counterJ , #1 // j-- - bgt .Ldgemm_kernel_L8_BEGIN + bgt L(dgemm_kernel_L8_BEGIN) /******************************************************************************/ /* Repeat the same thing if 4 left in N */ .align 5 -.Ldgemm_kernel_L4_BEGIN: +L(dgemm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #4 - ble .Ldgemm_kernel_L2_BEGIN + ble L(dgemm_kernel_L2_BEGIN) mov pCRow0, pC @@ -624,24 +624,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L4_Mv1_BEGIN: +L(dgemm_kernel_L4_Mv1_BEGIN): mov counterI, #0 whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d .align 5 -.Ldgemm_kernel_L4_Mv1_20: +L(dgemm_kernel_L4_Mv1_20): mov pB, origPB INITv1x4 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L4_Mv1_44 + ble L(dgemm_kernel_L4_Mv1_44) .align 5 -.Ldgemm_kernel_L4_Mv1_22: +L(dgemm_kernel_L4_Mv1_22): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x4_SUB @@ -657,38 +657,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_Mv1_22 + bgt L(dgemm_kernel_L4_Mv1_22) -.Ldgemm_kernel_L4_Mv1_44: +L(dgemm_kernel_L4_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L4_Mv1_100 + ble L(dgemm_kernel_L4_Mv1_100) .align 5 -.Ldgemm_kernel_L4_Mv1_46: +L(dgemm_kernel_L4_Mv1_46): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L4_Mv1_46 + bne L(dgemm_kernel_L4_Mv1_46) -.Ldgemm_kernel_L4_Mv1_100: +L(dgemm_kernel_L4_Mv1_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVEv1x4 -.Ldgemm_kernel_L4_Mv1_END: +L(dgemm_kernel_L4_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldgemm_kernel_L4_Mv1_20 + b.any L(dgemm_kernel_L4_Mv1_20) -.Ldgemm_kernel_L4_END: +L(dgemm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 @@ -696,11 +696,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /* Repeat the same thing if 2 left in N */ .align 5 -.Ldgemm_kernel_L2_BEGIN: +L(dgemm_kernel_L2_BEGIN): mov counterJ , origN tst counterJ , #2 - ble .Ldgemm_kernel_L1_BEGIN + ble L(dgemm_kernel_L1_BEGIN) mov pCRow0, pC @@ -708,24 +708,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L2_Mv1_BEGIN: +L(dgemm_kernel_L2_Mv1_BEGIN): mov counterI, #0 whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d .align 5 -.Ldgemm_kernel_L2_Mv1_20: +L(dgemm_kernel_L2_Mv1_20): mov pB, origPB INITv1x2 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L2_Mv1_44 + ble L(dgemm_kernel_L2_Mv1_44) .align 5 -.Ldgemm_kernel_L2_Mv1_22: +L(dgemm_kernel_L2_Mv1_22): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x2_SUB @@ -739,49 +739,49 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_Mv1_22 + bgt L(dgemm_kernel_L2_Mv1_22) -.Ldgemm_kernel_L2_Mv1_44: +L(dgemm_kernel_L2_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L2_Mv1_100 + ble L(dgemm_kernel_L2_Mv1_100) .align 5 -.Ldgemm_kernel_L2_Mv1_46: +L(dgemm_kernel_L2_Mv1_46): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x2_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L2_Mv1_46 + bne L(dgemm_kernel_L2_Mv1_46) -.Ldgemm_kernel_L2_Mv1_100: +L(dgemm_kernel_L2_Mv1_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVEv1x2 -.Ldgemm_kernel_L2_Mv1_END: +L(dgemm_kernel_L2_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldgemm_kernel_L2_Mv1_20 + b.any L(dgemm_kernel_L2_Mv1_20) -.Ldgemm_kernel_L2_END: +L(dgemm_kernel_L2_END): add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 /******************************************************************************/ /* Repeat the same thing if 1 left in N */ .align 5 -.Ldgemm_kernel_L1_BEGIN: +L(dgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldgemm_kernel_L999 // done + ble L(dgemm_kernel_L999) // done mov pCRow0, pC @@ -789,24 +789,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L1_Mv1_BEGIN: +L(dgemm_kernel_L1_Mv1_BEGIN): mov counterI, #0 whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d .align 5 -.Ldgemm_kernel_L1_Mv1_20: +L(dgemm_kernel_L1_Mv1_20): mov pB, origPB INITv1x1 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Ldgemm_kernel_L1_Mv1_44 + ble L(dgemm_kernel_L1_Mv1_44) .align 5 -.Ldgemm_kernel_L1_Mv1_22: +L(dgemm_kernel_L1_Mv1_22): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x1_SUB @@ -819,42 +819,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv1_22 + bgt L(dgemm_kernel_L1_Mv1_22) -.Ldgemm_kernel_L1_Mv1_44: +L(dgemm_kernel_L1_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L1_Mv1_100 + ble L(dgemm_kernel_L1_Mv1_100) .align 5 -.Ldgemm_kernel_L1_Mv1_46: +L(dgemm_kernel_L1_Mv1_46): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv1_46 + bgt L(dgemm_kernel_L1_Mv1_46) -.Ldgemm_kernel_L1_Mv1_100: +L(dgemm_kernel_L1_Mv1_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVEv1x1 -.Ldgemm_kernel_L1_Mv1_END: +L(dgemm_kernel_L1_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldgemm_kernel_L1_Mv1_20 + b.any L(dgemm_kernel_L1_Mv1_20) -.Ldgemm_kernel_L1_END: +L(dgemm_kernel_L1_END): /******************************************************************************/ -.Ldgemm_kernel_L999: +L(dgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dgemm_kernel_sve_v2x8.S b/kernel/arm64/dgemm_kernel_sve_v2x8.S index d978a3315b..7296a5c578 100644 --- a/kernel/arm64/dgemm_kernel_sve_v2x8.S +++ b/kernel/arm64/dgemm_kernel_sve_v2x8.S @@ -882,24 +882,24 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Ldgemm_kernel_L4_BEGIN + ble L(dgemm_kernel_L4_BEGIN) /******************************************************************************/ /* Repeat this as long as there are 8 left in N */ .align 5 -.Ldgemm_kernel_L8_BEGIN: +L(dgemm_kernel_L8_BEGIN): mov pCRow0, pC add pC, pC, LDC, lsl #3 // add 8 x LDC mov pA1, origPA // pA1 = start of A array -.Ldgemm_kernel_L8_Mv2_BEGIN: +L(dgemm_kernel_L8_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 // Check if M < 2*SVE_LEN - blt .Ldgemm_kernel_L8_Mv1_BEGIN + blt L(dgemm_kernel_L8_Mv1_BEGIN) mov counterI, origM @@ -908,14 +908,14 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ add pA2, pA1, temp, lsl #3 // pA1 = start of A array .align 5 -.Ldgemm_kernel_L8_Mv2_20: +L(dgemm_kernel_L8_Mv2_20): mov pB, origPB INITv2x8 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Ldgemm_kernel_L8_Mv2_32 + blt L(dgemm_kernel_L8_Mv2_32) KERNELv2x8_I KERNELv2x8_M2 @@ -927,10 +927,10 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Ldgemm_kernel_L8_Mv2_22a + ble L(dgemm_kernel_L8_Mv2_22a) .align 5 -.Ldgemm_kernel_L8_Mv2_22: +L(dgemm_kernel_L8_Mv2_22): KERNELv2x8_M1 KERNELv2x8_M2 @@ -942,10 +942,10 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M2 subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L8_Mv2_22 + bgt L(dgemm_kernel_L8_Mv2_22) .align 5 -.Ldgemm_kernel_L8_Mv2_22a: +L(dgemm_kernel_L8_Mv2_22a): KERNELv2x8_M1 KERNELv2x8_M2 @@ -956,13 +956,13 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M1 KERNELv2x8_E - b .Ldgemm_kernel_L8_Mv2_44 + b L(dgemm_kernel_L8_Mv2_44) .align 5 -.Ldgemm_kernel_L8_Mv2_32: +L(dgemm_kernel_L8_Mv2_32): tst counterL, #1 - ble .Ldgemm_kernel_L8_Mv2_40 + ble L(dgemm_kernel_L8_Mv2_40) KERNELv2x8_I KERNELv2x8_M2 @@ -974,57 +974,57 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_E - b .Ldgemm_kernel_L8_Mv2_44 + b L(dgemm_kernel_L8_Mv2_44) -.Ldgemm_kernel_L8_Mv2_40: +L(dgemm_kernel_L8_Mv2_40): INITv2x8 -.Ldgemm_kernel_L8_Mv2_44: +L(dgemm_kernel_L8_Mv2_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L8_Mv2_100 + ble L(dgemm_kernel_L8_Mv2_100) .align 5 -.Ldgemm_kernel_L8_Mv2_46: +L(dgemm_kernel_L8_Mv2_46): KERNELv2x8_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L8_Mv2_46 + bne L(dgemm_kernel_L8_Mv2_46) -.Ldgemm_kernel_L8_Mv2_100: +L(dgemm_kernel_L8_Mv2_100): SAVEv2x8 mov pA1, pA2 // pA1 = pA2 mul temp, vec_len, origK // generate address of pA2 add pA2, pA1, temp, lsl #3 // -.Ldgemm_kernel_L8_Mv2_END: +L(dgemm_kernel_L8_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Ldgemm_kernel_L8_Mv2_20 + bge L(dgemm_kernel_L8_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Ldgemm_kernel_L8_END + beq L(dgemm_kernel_L8_END) ////////////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x8 kernel. -.Ldgemm_kernel_L8_Mv1_BEGIN: +L(dgemm_kernel_L8_Mv1_BEGIN): whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension .align 5 -.Ldgemm_kernel_L8_Mv1_20: +L(dgemm_kernel_L8_Mv1_20): mov pB, origPB INITv1x8 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Ldgemm_kernel_L8_Mv1_32 + blt L(dgemm_kernel_L8_Mv1_32) KERNELv1x8_I KERNELv1x8_M2 @@ -1036,10 +1036,10 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Ldgemm_kernel_L8_Mv1_22a + ble L(dgemm_kernel_L8_Mv1_22a) .align 5 -.Ldgemm_kernel_L8_Mv1_22: +L(dgemm_kernel_L8_Mv1_22): KERNELv1x8_M1 KERNELv1x8_M2 @@ -1051,10 +1051,10 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M2 subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L8_Mv1_22 + bgt L(dgemm_kernel_L8_Mv1_22) .align 5 -.Ldgemm_kernel_L8_Mv1_22a: +L(dgemm_kernel_L8_Mv1_22a): KERNELv1x8_M1 KERNELv1x8_M2 @@ -1065,13 +1065,13 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M1 KERNELv1x8_E - b .Ldgemm_kernel_L8_Mv1_44 + b L(dgemm_kernel_L8_Mv1_44) .align 5 -.Ldgemm_kernel_L8_Mv1_32: +L(dgemm_kernel_L8_Mv1_32): tst counterL, #1 - ble .Ldgemm_kernel_L8_Mv1_40 + ble L(dgemm_kernel_L8_Mv1_40) KERNELv1x8_I KERNELv1x8_M2 @@ -1083,53 +1083,53 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_E - b .Ldgemm_kernel_L8_Mv1_44 + b L(dgemm_kernel_L8_Mv1_44) -.Ldgemm_kernel_L8_Mv1_40: +L(dgemm_kernel_L8_Mv1_40): INITv1x8 -.Ldgemm_kernel_L8_Mv1_44: +L(dgemm_kernel_L8_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L8_Mv1_100 + ble L(dgemm_kernel_L8_Mv1_100) .align 5 -.Ldgemm_kernel_L8_Mv1_46: +L(dgemm_kernel_L8_Mv1_46): KERNELv1x8_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L8_Mv1_46 + bne L(dgemm_kernel_L8_Mv1_46) -.Ldgemm_kernel_L8_Mv1_100: +L(dgemm_kernel_L8_Mv1_100): SAVEv1x8 -.Ldgemm_kernel_L8_Mv1_END: +L(dgemm_kernel_L8_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension - b.any .Ldgemm_kernel_L8_Mv1_20 + b.any L(dgemm_kernel_L8_Mv1_20) -.Ldgemm_kernel_L8_END: +L(dgemm_kernel_L8_END): lsl temp, origK, #6 add origPB, origPB, temp // B = B + K * 8 * 8 subs counterJ, counterJ , #1 // j-- - bgt .Ldgemm_kernel_L8_BEGIN + bgt L(dgemm_kernel_L8_BEGIN) /******************************************************************************/ /* Repeat the same thing if 4 left in N */ .align 5 -.Ldgemm_kernel_L4_BEGIN: +L(dgemm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #4 - ble .Ldgemm_kernel_L2_BEGIN + ble L(dgemm_kernel_L2_BEGIN) mov pCRow0, pC @@ -1138,11 +1138,11 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Ldgemm_kernel_L4_Mv2_BEGIN: +L(dgemm_kernel_L4_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Ldgemm_kernel_L4_Mv1_BEGIN + blt L(dgemm_kernel_L4_Mv1_BEGIN) mov counterI, origM @@ -1150,17 +1150,17 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ add pA2, pA1, temp, lsl #3 // pA1 = start of A array .align 5 -.Ldgemm_kernel_L4_Mv2_20: +L(dgemm_kernel_L4_Mv2_20): mov pB, origPB INITv2x4 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L4_Mv2_44 + ble L(dgemm_kernel_L4_Mv2_44) .align 5 -.Ldgemm_kernel_L4_Mv2_22: +L(dgemm_kernel_L4_Mv2_22): KERNELv2x4_SUB KERNELv2x4_SUB @@ -1172,56 +1172,56 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_Mv2_22 + bgt L(dgemm_kernel_L4_Mv2_22) -.Ldgemm_kernel_L4_Mv2_44: +L(dgemm_kernel_L4_Mv2_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L4_Mv2_100 + ble L(dgemm_kernel_L4_Mv2_100) .align 5 -.Ldgemm_kernel_L4_Mv2_46: +L(dgemm_kernel_L4_Mv2_46): KERNELv2x4_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L4_Mv2_46 + bne L(dgemm_kernel_L4_Mv2_46) -.Ldgemm_kernel_L4_Mv2_100: +L(dgemm_kernel_L4_Mv2_100): SAVEv2x4 mov pA1, pA2 // pA1 = pA2 mul temp, vec_len, origK // generate address of pA2 add pA2, pA1, temp, lsl #3 // -.Ldgemm_kernel_L4_Mv2_END: +L(dgemm_kernel_L4_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Ldgemm_kernel_L4_Mv2_20 + bge L(dgemm_kernel_L4_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Ldgemm_kernel_L4_END + beq L(dgemm_kernel_L4_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x4 kernel. -.Ldgemm_kernel_L4_Mv1_BEGIN: +L(dgemm_kernel_L4_Mv1_BEGIN): whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension .align 5 -.Ldgemm_kernel_L4_Mv1_20: +L(dgemm_kernel_L4_Mv1_20): mov pB, origPB INITv1x4 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L4_Mv1_44 + ble L(dgemm_kernel_L4_Mv1_44) .align 5 -.Ldgemm_kernel_L4_Mv1_22: +L(dgemm_kernel_L4_Mv1_22): KERNELv1x4_SUB KERNELv1x4_SUB @@ -1233,34 +1233,34 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_Mv1_22 + bgt L(dgemm_kernel_L4_Mv1_22) -.Ldgemm_kernel_L4_Mv1_44: +L(dgemm_kernel_L4_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L4_Mv1_100 + ble L(dgemm_kernel_L4_Mv1_100) .align 5 -.Ldgemm_kernel_L4_Mv1_46: +L(dgemm_kernel_L4_Mv1_46): KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L4_Mv1_46 + bne L(dgemm_kernel_L4_Mv1_46) -.Ldgemm_kernel_L4_Mv1_100: +L(dgemm_kernel_L4_Mv1_100): SAVEv1x4 -.Ldgemm_kernel_L4_Mv1_END: +L(dgemm_kernel_L4_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldgemm_kernel_L4_Mv1_20 + b.any L(dgemm_kernel_L4_Mv1_20) -.Ldgemm_kernel_L4_END: +L(dgemm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 @@ -1268,11 +1268,11 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ /* Repeat the same thing if 2 left in N */ .align 5 -.Ldgemm_kernel_L2_BEGIN: +L(dgemm_kernel_L2_BEGIN): mov counterJ , origN tst counterJ , #2 - ble .Ldgemm_kernel_L1_BEGIN + ble L(dgemm_kernel_L1_BEGIN) mov pCRow0, pC @@ -1280,11 +1280,11 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Ldgemm_kernel_L2_Mv2_BEGIN: +L(dgemm_kernel_L2_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Ldgemm_kernel_L2_Mv1_BEGIN + blt L(dgemm_kernel_L2_Mv1_BEGIN) mov counterI, origM @@ -1292,17 +1292,17 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ add pA2, pA1, temp, lsl #3 // pA1 = start of A array .align 5 -.Ldgemm_kernel_L2_Mv2_20: +L(dgemm_kernel_L2_Mv2_20): mov pB, origPB INITv2x2 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L2_Mv2_44 + ble L(dgemm_kernel_L2_Mv2_44) .align 5 -.Ldgemm_kernel_L2_Mv2_22: +L(dgemm_kernel_L2_Mv2_22): KERNELv2x2_SUB KERNELv2x2_SUB @@ -1314,57 +1314,57 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_Mv2_22 + bgt L(dgemm_kernel_L2_Mv2_22) -.Ldgemm_kernel_L2_Mv2_44: +L(dgemm_kernel_L2_Mv2_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L2_Mv2_100 + ble L(dgemm_kernel_L2_Mv2_100) .align 5 -.Ldgemm_kernel_L2_Mv2_46: +L(dgemm_kernel_L2_Mv2_46): KERNELv2x2_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L2_Mv2_46 + bne L(dgemm_kernel_L2_Mv2_46) -.Ldgemm_kernel_L2_Mv2_100: +L(dgemm_kernel_L2_Mv2_100): SAVEv2x2 mov pA1, pA2 // pA1 = pA2 mul temp, vec_len, origK // generate address of pA2 add pA2, pA1, temp, lsl #3 // -.Ldgemm_kernel_L2_Mv2_END: +L(dgemm_kernel_L2_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Ldgemm_kernel_L2_Mv2_20 + bge L(dgemm_kernel_L2_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Ldgemm_kernel_L2_END + beq L(dgemm_kernel_L2_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x2 kernel. -.Ldgemm_kernel_L2_Mv1_BEGIN: +L(dgemm_kernel_L2_Mv1_BEGIN): whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d .align 5 -.Ldgemm_kernel_L2_Mv1_20: +L(dgemm_kernel_L2_Mv1_20): mov pB, origPB INITv1x2 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L2_Mv1_44 + ble L(dgemm_kernel_L2_Mv1_44) .align 5 -.Ldgemm_kernel_L2_Mv1_22: +L(dgemm_kernel_L2_Mv1_22): KERNELv1x2_SUB KERNELv1x2_SUB @@ -1376,45 +1376,45 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_Mv1_22 + bgt L(dgemm_kernel_L2_Mv1_22) -.Ldgemm_kernel_L2_Mv1_44: +L(dgemm_kernel_L2_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L2_Mv1_100 + ble L(dgemm_kernel_L2_Mv1_100) .align 5 -.Ldgemm_kernel_L2_Mv1_46: +L(dgemm_kernel_L2_Mv1_46): KERNELv1x2_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L2_Mv1_46 + bne L(dgemm_kernel_L2_Mv1_46) -.Ldgemm_kernel_L2_Mv1_100: +L(dgemm_kernel_L2_Mv1_100): SAVEv1x2 -.Ldgemm_kernel_L2_Mv1_END: +L(dgemm_kernel_L2_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldgemm_kernel_L2_Mv1_20 + b.any L(dgemm_kernel_L2_Mv1_20) -.Ldgemm_kernel_L2_END: +L(dgemm_kernel_L2_END): add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 /******************************************************************************/ /* Repeat the same thing if 1 left in N */ .align 5 -.Ldgemm_kernel_L1_BEGIN: +L(dgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldgemm_kernel_L999 // done + ble L(dgemm_kernel_L999) // done mov pCRow0, pC @@ -1422,11 +1422,11 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Ldgemm_kernel_L1_Mv2_BEGIN: +L(dgemm_kernel_L1_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Ldgemm_kernel_L1_Mv1_BEGIN + blt L(dgemm_kernel_L1_Mv1_BEGIN) mov counterI, origM @@ -1435,17 +1435,17 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ .align 5 -.Ldgemm_kernel_L1_Mv2_20: +L(dgemm_kernel_L1_Mv2_20): mov pB, origPB INITv2x1 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Ldgemm_kernel_L1_Mv2_44 + ble L(dgemm_kernel_L1_Mv2_44) .align 5 -.Ldgemm_kernel_L1_Mv2_22: +L(dgemm_kernel_L1_Mv2_22): KERNELv2x1_SUB KERNELv2x1_SUB @@ -1457,57 +1457,57 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv2_22 + bgt L(dgemm_kernel_L1_Mv2_22) -.Ldgemm_kernel_L1_Mv2_44: +L(dgemm_kernel_L1_Mv2_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L1_Mv2_100 + ble L(dgemm_kernel_L1_Mv2_100) .align 5 -.Ldgemm_kernel_L1_Mv2_46: +L(dgemm_kernel_L1_Mv2_46): KERNELv2x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv2_46 + bgt L(dgemm_kernel_L1_Mv2_46) -.Ldgemm_kernel_L1_Mv2_100: +L(dgemm_kernel_L1_Mv2_100): SAVEv2x1 mov pA1, pA2 // pA1 = pA2 mul temp, vec_len, origK // generate address of pA2 add pA2, pA1, temp, lsl #3 // -.Ldgemm_kernel_L1_Mv2_END: +L(dgemm_kernel_L1_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Ldgemm_kernel_L1_Mv2_20 + bge L(dgemm_kernel_L1_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Ldgemm_kernel_L1_END + beq L(dgemm_kernel_L1_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x1 kernel. -.Ldgemm_kernel_L1_Mv1_BEGIN: +L(dgemm_kernel_L1_Mv1_BEGIN): whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d .align 5 -.Ldgemm_kernel_L1_Mv1_20: +L(dgemm_kernel_L1_Mv1_20): mov pB, origPB INITv1x1 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Ldgemm_kernel_L1_Mv1_44 + ble L(dgemm_kernel_L1_Mv1_44) .align 5 -.Ldgemm_kernel_L1_Mv1_22: +L(dgemm_kernel_L1_Mv1_22): KERNELv1x1_SUB KERNELv1x1_SUB @@ -1519,38 +1519,38 @@ With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv1_22 + bgt L(dgemm_kernel_L1_Mv1_22) -.Ldgemm_kernel_L1_Mv1_44: +L(dgemm_kernel_L1_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L1_Mv1_100 + ble L(dgemm_kernel_L1_Mv1_100) .align 5 -.Ldgemm_kernel_L1_Mv1_46: +L(dgemm_kernel_L1_Mv1_46): KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv1_46 + bgt L(dgemm_kernel_L1_Mv1_46) -.Ldgemm_kernel_L1_Mv1_100: +L(dgemm_kernel_L1_Mv1_100): SAVEv1x1 -.Ldgemm_kernel_L1_Mv1_END: +L(dgemm_kernel_L1_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldgemm_kernel_L1_Mv1_20 + b.any L(dgemm_kernel_L1_Mv1_20) -.Ldgemm_kernel_L1_END: +L(dgemm_kernel_L1_END): /******************************************************************************/ -.Ldgemm_kernel_L999: +L(dgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dgemm_ncopy_4.S b/kernel/arm64/dgemm_ncopy_4.S index 29d274d931..11e2d23a11 100644 --- a/kernel/arm64/dgemm_ncopy_4.S +++ b/kernel/arm64/dgemm_ncopy_4.S @@ -192,14 +192,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. lsl LDA, LDA, #3 // LDA = LDA * SIZE -.Ldgemm_ncopy_L4_BEGIN: +L(dgemm_ncopy_L4_BEGIN): asr J, N, #2 // J = N / 4 cmp J, #0 - ble .Ldgemm_ncopy_L2_BEGIN + ble L(dgemm_ncopy_L2_BEGIN) .align 5 -.Ldgemm_ncopy_L4_M4_BEGIN: +L(dgemm_ncopy_L4_M4_BEGIN): mov A01, A00 add A02, A01, LDA @@ -209,128 +209,128 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #2 // I = M / 4 cmp I, #0 - ble .Ldgemm_ncopy_L4_M4_40 + ble L(dgemm_ncopy_L4_M4_40) .align 5 -.Ldgemm_ncopy_L4_M4_20: +L(dgemm_ncopy_L4_M4_20): COPY4x4 subs I , I , #1 - bne .Ldgemm_ncopy_L4_M4_20 + bne L(dgemm_ncopy_L4_M4_20) -.Ldgemm_ncopy_L4_M4_40: +L(dgemm_ncopy_L4_M4_40): and I, M , #3 cmp I, #0 - ble .Ldgemm_ncopy_L4_M4_END + ble L(dgemm_ncopy_L4_M4_END) .align 5 -.Ldgemm_ncopy_L4_M4_60: +L(dgemm_ncopy_L4_M4_60): COPY1x4 subs I , I , #1 - bne .Ldgemm_ncopy_L4_M4_60 + bne L(dgemm_ncopy_L4_M4_60) -.Ldgemm_ncopy_L4_M4_END: +L(dgemm_ncopy_L4_M4_END): subs J , J, #1 // j-- - bne .Ldgemm_ncopy_L4_M4_BEGIN + bne L(dgemm_ncopy_L4_M4_BEGIN) /*********************************************************************************************/ -.Ldgemm_ncopy_L2_BEGIN: +L(dgemm_ncopy_L2_BEGIN): tst N, #3 - ble .Ldgemm_ncopy_L999 + ble L(dgemm_ncopy_L999) tst N, #2 - ble .Ldgemm_ncopy_L1_BEGIN + ble L(dgemm_ncopy_L1_BEGIN) -.Ldgemm_ncopy_L2_M4_BEGIN: +L(dgemm_ncopy_L2_M4_BEGIN): mov A01, A00 add A02, A01, LDA add A00, A02, LDA asr I, M, #2 // I = M / 4 cmp I, #0 - ble .Ldgemm_ncopy_L2_M4_40 + ble L(dgemm_ncopy_L2_M4_40) .align 5 -.Ldgemm_ncopy_L2_M4_20: +L(dgemm_ncopy_L2_M4_20): COPY4x2 subs I , I , #1 - bne .Ldgemm_ncopy_L2_M4_20 + bne L(dgemm_ncopy_L2_M4_20) -.Ldgemm_ncopy_L2_M4_40: +L(dgemm_ncopy_L2_M4_40): and I, M , #3 cmp I, #0 - ble .Ldgemm_ncopy_L2_M4_END + ble L(dgemm_ncopy_L2_M4_END) .align 5 -.Ldgemm_ncopy_L2_M4_60: +L(dgemm_ncopy_L2_M4_60): COPY1x2 subs I , I , #1 - bne .Ldgemm_ncopy_L2_M4_60 + bne L(dgemm_ncopy_L2_M4_60) -.Ldgemm_ncopy_L2_M4_END: +L(dgemm_ncopy_L2_M4_END): /*********************************************************************************************/ -.Ldgemm_ncopy_L1_BEGIN: +L(dgemm_ncopy_L1_BEGIN): tst N, #1 - ble .Ldgemm_ncopy_L999 + ble L(dgemm_ncopy_L999) -.Ldgemm_ncopy_L1_M4_BEGIN: +L(dgemm_ncopy_L1_M4_BEGIN): mov A01, A00 asr I, M, #2 // I = M / 4 cmp I, #0 - ble .Ldgemm_ncopy_L1_M4_40 + ble L(dgemm_ncopy_L1_M4_40) .align 5 -.Ldgemm_ncopy_L1_M4_20: +L(dgemm_ncopy_L1_M4_20): COPY4x1 subs I , I , #1 - bne .Ldgemm_ncopy_L1_M4_20 + bne L(dgemm_ncopy_L1_M4_20) -.Ldgemm_ncopy_L1_M4_40: +L(dgemm_ncopy_L1_M4_40): and I, M , #3 cmp I, #0 - ble .Ldgemm_ncopy_L1_M4_END + ble L(dgemm_ncopy_L1_M4_END) .align 5 -.Ldgemm_ncopy_L1_M4_60: +L(dgemm_ncopy_L1_M4_60): COPY1x1 subs I , I , #1 - bne .Ldgemm_ncopy_L1_M4_60 + bne L(dgemm_ncopy_L1_M4_60) -.Ldgemm_ncopy_L1_M4_END: +L(dgemm_ncopy_L1_M4_END): -.Ldgemm_ncopy_L999: +L(dgemm_ncopy_L999): mov x0, #0 RESTORE_REGS diff --git a/kernel/arm64/dgemm_ncopy_8.S b/kernel/arm64/dgemm_ncopy_8.S index 366424830d..f4dc62f2d3 100644 --- a/kernel/arm64/dgemm_ncopy_8.S +++ b/kernel/arm64/dgemm_ncopy_8.S @@ -353,13 +353,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. lsl LDA, LDA, #3 // LDA = LDA * SIZE -.Ldgemm_ncopy_L8_BEGIN: +L(dgemm_ncopy_L8_BEGIN): asr J, N, #3 // J = N / 8 cmp J, #0 - ble .Ldgemm_ncopy_L4_BEGIN + ble L(dgemm_ncopy_L4_BEGIN) -.Ldgemm_ncopy_L8_M8_BEGIN: +L(dgemm_ncopy_L8_M8_BEGIN): mov A01, A00 add A02, A01, LDA @@ -374,46 +374,46 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #3 // I = M / 8 cmp I, #0 - ble .Ldgemm_ncopy_L8_M8_40 + ble L(dgemm_ncopy_L8_M8_40) -.Ldgemm_ncopy_L8_M8_20: +L(dgemm_ncopy_L8_M8_20): COPY8x8 subs I , I , #1 - bne .Ldgemm_ncopy_L8_M8_20 + bne L(dgemm_ncopy_L8_M8_20) -.Ldgemm_ncopy_L8_M8_40: +L(dgemm_ncopy_L8_M8_40): and I, M , #7 cmp I, #0 - ble .Ldgemm_ncopy_L8_M8_END + ble L(dgemm_ncopy_L8_M8_END) -.Ldgemm_ncopy_L8_M8_60: +L(dgemm_ncopy_L8_M8_60): COPY1x8 subs I , I , #1 - bne .Ldgemm_ncopy_L8_M8_60 + bne L(dgemm_ncopy_L8_M8_60) -.Ldgemm_ncopy_L8_M8_END: +L(dgemm_ncopy_L8_M8_END): subs J , J, #1 // j-- - bne .Ldgemm_ncopy_L8_M8_BEGIN + bne L(dgemm_ncopy_L8_M8_BEGIN) /*********************************************************************************************/ -.Ldgemm_ncopy_L4_BEGIN: +L(dgemm_ncopy_L4_BEGIN): tst N, #7 - ble .Ldgemm_ncopy_L999 + ble L(dgemm_ncopy_L999) tst N, #4 - ble .Ldgemm_ncopy_L2_BEGIN + ble L(dgemm_ncopy_L2_BEGIN) -.Ldgemm_ncopy_L4_M8_BEGIN: +L(dgemm_ncopy_L4_M8_BEGIN): mov A01, A00 add A02, A01, LDA @@ -423,118 +423,118 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #3 // I = M / 8 cmp I, #0 - ble .Ldgemm_ncopy_L4_M8_40 + ble L(dgemm_ncopy_L4_M8_40) -.Ldgemm_ncopy_L4_M8_20: +L(dgemm_ncopy_L4_M8_20): COPY8x4 subs I , I , #1 - bne .Ldgemm_ncopy_L4_M8_20 + bne L(dgemm_ncopy_L4_M8_20) -.Ldgemm_ncopy_L4_M8_40: +L(dgemm_ncopy_L4_M8_40): and I, M , #7 cmp I, #0 - ble .Ldgemm_ncopy_L4_M8_END + ble L(dgemm_ncopy_L4_M8_END) -.Ldgemm_ncopy_L4_M8_60: +L(dgemm_ncopy_L4_M8_60): COPY1x4 subs I , I , #1 - bne .Ldgemm_ncopy_L4_M8_60 + bne L(dgemm_ncopy_L4_M8_60) -.Ldgemm_ncopy_L4_M8_END: +L(dgemm_ncopy_L4_M8_END): /*********************************************************************************************/ -.Ldgemm_ncopy_L2_BEGIN: +L(dgemm_ncopy_L2_BEGIN): tst N, #3 - ble .Ldgemm_ncopy_L999 + ble L(dgemm_ncopy_L999) tst N, #2 - ble .Ldgemm_ncopy_L1_BEGIN + ble L(dgemm_ncopy_L1_BEGIN) -.Ldgemm_ncopy_L2_M8_BEGIN: +L(dgemm_ncopy_L2_M8_BEGIN): mov A01, A00 add A02, A01, LDA add A00, A02, LDA asr I, M, #3 // I = M / 8 cmp I, #0 - ble .Ldgemm_ncopy_L2_M8_40 + ble L(dgemm_ncopy_L2_M8_40) -.Ldgemm_ncopy_L2_M8_20: +L(dgemm_ncopy_L2_M8_20): COPY8x2 subs I , I , #1 - bne .Ldgemm_ncopy_L2_M8_20 + bne L(dgemm_ncopy_L2_M8_20) -.Ldgemm_ncopy_L2_M8_40: +L(dgemm_ncopy_L2_M8_40): and I, M , #7 cmp I, #0 - ble .Ldgemm_ncopy_L2_M8_END + ble L(dgemm_ncopy_L2_M8_END) -.Ldgemm_ncopy_L2_M8_60: +L(dgemm_ncopy_L2_M8_60): COPY1x2 subs I , I , #1 - bne .Ldgemm_ncopy_L2_M8_60 + bne L(dgemm_ncopy_L2_M8_60) -.Ldgemm_ncopy_L2_M8_END: +L(dgemm_ncopy_L2_M8_END): /*********************************************************************************************/ -.Ldgemm_ncopy_L1_BEGIN: +L(dgemm_ncopy_L1_BEGIN): tst N, #1 - ble .Ldgemm_ncopy_L999 + ble L(dgemm_ncopy_L999) -.Ldgemm_ncopy_L1_M8_BEGIN: +L(dgemm_ncopy_L1_M8_BEGIN): mov A01, A00 asr I, M, #3 // I = M / 8 cmp I, #0 - ble .Ldgemm_ncopy_L1_M8_40 + ble L(dgemm_ncopy_L1_M8_40) -.Ldgemm_ncopy_L1_M8_20: +L(dgemm_ncopy_L1_M8_20): COPY8x1 subs I , I , #1 - bne .Ldgemm_ncopy_L1_M8_20 + bne L(dgemm_ncopy_L1_M8_20) -.Ldgemm_ncopy_L1_M8_40: +L(dgemm_ncopy_L1_M8_40): and I, M , #7 cmp I, #0 - ble .Ldgemm_ncopy_L1_M8_END + ble L(dgemm_ncopy_L1_M8_END) -.Ldgemm_ncopy_L1_M8_60: +L(dgemm_ncopy_L1_M8_60): COPY1x1 subs I , I , #1 - bne .Ldgemm_ncopy_L1_M8_60 + bne L(dgemm_ncopy_L1_M8_60) -.Ldgemm_ncopy_L1_M8_END: +L(dgemm_ncopy_L1_M8_END): -.Ldgemm_ncopy_L999: +L(dgemm_ncopy_L999): mov x0, #0 RESTORE_REGS diff --git a/kernel/arm64/dgemm_tcopy_4.S b/kernel/arm64/dgemm_tcopy_4.S index 7c9135287f..8ace3a67be 100644 --- a/kernel/arm64/dgemm_tcopy_4.S +++ b/kernel/arm64/dgemm_tcopy_4.S @@ -247,13 +247,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. lsl M4, M, #5 // M4 = M * 4 * SIZE -.Ldgemm_tcopy_L4_BEGIN: +L(dgemm_tcopy_L4_BEGIN): asr J, M, #2 // J = M / 4 cmp J, #0 - ble .Ldgemm_tcopy_L2_BEGIN + ble L(dgemm_tcopy_L2_BEGIN) .align 5 -.Ldgemm_tcopy_L4_M4_BEGIN: +L(dgemm_tcopy_L4_M4_BEGIN): mov A01, A add A02, A01, LDA @@ -266,51 +266,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, N, #2 // I = N / 4 cmp I, #0 - ble .Ldgemm_tcopy_L4_M4_40 + ble L(dgemm_tcopy_L4_M4_40) .align 5 -.Ldgemm_tcopy_L4_M4_20: +L(dgemm_tcopy_L4_M4_20): COPY4x4 subs I , I , #1 - bne .Ldgemm_tcopy_L4_M4_20 + bne L(dgemm_tcopy_L4_M4_20) -.Ldgemm_tcopy_L4_M4_40: +L(dgemm_tcopy_L4_M4_40): tst N , #2 - ble .Ldgemm_tcopy_L4_M4_60 + ble L(dgemm_tcopy_L4_M4_60) COPY2x4 -.Ldgemm_tcopy_L4_M4_60: +L(dgemm_tcopy_L4_M4_60): tst N, #1 - ble .Ldgemm_tcopy_L4_M4_END + ble L(dgemm_tcopy_L4_M4_END) COPY1x4 -.Ldgemm_tcopy_L4_M4_END: +L(dgemm_tcopy_L4_M4_END): subs J , J, #1 // j-- - bne .Ldgemm_tcopy_L4_M4_BEGIN + bne L(dgemm_tcopy_L4_M4_BEGIN) /*********************************************************************************************/ -.Ldgemm_tcopy_L2_BEGIN: +L(dgemm_tcopy_L2_BEGIN): tst M, #3 - ble .Ldgemm_tcopy_L999 + ble L(dgemm_tcopy_L999) tst M, #2 - ble .Ldgemm_tcopy_L1_BEGIN + ble L(dgemm_tcopy_L1_BEGIN) -.Ldgemm_tcopy_L2_M4_BEGIN: +L(dgemm_tcopy_L2_M4_BEGIN): mov A01, A add A02, A01, LDA add A, A02, LDA @@ -320,80 +320,80 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, N, #2 // I = N / 4 cmp I, #0 - ble .Ldgemm_tcopy_L2_M4_40 + ble L(dgemm_tcopy_L2_M4_40) .align 5 -.Ldgemm_tcopy_L2_M4_20: +L(dgemm_tcopy_L2_M4_20): COPY4x2 subs I , I , #1 - bne .Ldgemm_tcopy_L2_M4_20 + bne L(dgemm_tcopy_L2_M4_20) -.Ldgemm_tcopy_L2_M4_40: +L(dgemm_tcopy_L2_M4_40): tst N , #2 - ble .Ldgemm_tcopy_L2_M4_60 + ble L(dgemm_tcopy_L2_M4_60) COPY2x2 -.Ldgemm_tcopy_L2_M4_60: +L(dgemm_tcopy_L2_M4_60): tst N , #1 - ble .Ldgemm_tcopy_L2_M4_END + ble L(dgemm_tcopy_L2_M4_END) COPY1x2 -.Ldgemm_tcopy_L2_M4_END: +L(dgemm_tcopy_L2_M4_END): /*********************************************************************************************/ -.Ldgemm_tcopy_L1_BEGIN: +L(dgemm_tcopy_L1_BEGIN): tst M, #1 - ble .Ldgemm_tcopy_L999 + ble L(dgemm_tcopy_L999) -.Ldgemm_tcopy_L1_M4_BEGIN: +L(dgemm_tcopy_L1_M4_BEGIN): mov A01, A // A01 = A mov B01, B asr I, N, #2 // I = M / 4 cmp I, #0 - ble .Ldgemm_tcopy_L1_M4_40 + ble L(dgemm_tcopy_L1_M4_40) .align 5 -.Ldgemm_tcopy_L1_M4_20: +L(dgemm_tcopy_L1_M4_20): COPY4x1 subs I , I , #1 - bne .Ldgemm_tcopy_L1_M4_20 + bne L(dgemm_tcopy_L1_M4_20) -.Ldgemm_tcopy_L1_M4_40: +L(dgemm_tcopy_L1_M4_40): tst N , #2 - ble .Ldgemm_tcopy_L1_M4_60 + ble L(dgemm_tcopy_L1_M4_60) COPY2x1 -.Ldgemm_tcopy_L1_M4_60: +L(dgemm_tcopy_L1_M4_60): tst N , #1 - ble .Ldgemm_tcopy_L1_M4_END + ble L(dgemm_tcopy_L1_M4_END) COPY1x1 -.Ldgemm_tcopy_L1_M4_END: +L(dgemm_tcopy_L1_M4_END): -.Ldgemm_tcopy_L999: +L(dgemm_tcopy_L999): mov x0, #0 // set return value RESTORE_REGS ret diff --git a/kernel/arm64/dgemm_tcopy_8.S b/kernel/arm64/dgemm_tcopy_8.S index 7e5bf6080f..a04f71d4d7 100644 --- a/kernel/arm64/dgemm_tcopy_8.S +++ b/kernel/arm64/dgemm_tcopy_8.S @@ -453,13 +453,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. lsl M8, M, #6 // M8 = M * 8 * SIZE -.Ldgemm_tcopy_L8_BEGIN: +L(dgemm_tcopy_L8_BEGIN): asr J, M, #3 // J = M / 4 cmp J, #0 - ble .Ldgemm_tcopy_L4_BEGIN + ble L(dgemm_tcopy_L4_BEGIN) .align 5 -.Ldgemm_tcopy_L8_M8_BEGIN: +L(dgemm_tcopy_L8_M8_BEGIN): mov A01, A add A02, A01, LDA @@ -476,53 +476,53 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, N, #3 // I = N / 8 cmp I, #0 - ble .Ldgemm_tcopy_L8_M8_40 + ble L(dgemm_tcopy_L8_M8_40) .align 5 -.Ldgemm_tcopy_L8_M8_20: +L(dgemm_tcopy_L8_M8_20): COPY8x8 subs I , I , #1 - bne .Ldgemm_tcopy_L8_M8_20 + bne L(dgemm_tcopy_L8_M8_20) -.Ldgemm_tcopy_L8_M8_40: +L(dgemm_tcopy_L8_M8_40): tst N , #4 - ble .Ldgemm_tcopy_L8_M8_60 + ble L(dgemm_tcopy_L8_M8_60) COPY4x8 -.Ldgemm_tcopy_L8_M8_60: +L(dgemm_tcopy_L8_M8_60): tst N , #2 - ble .Ldgemm_tcopy_L8_M8_80 + ble L(dgemm_tcopy_L8_M8_80) COPY2x8 -.Ldgemm_tcopy_L8_M8_80: +L(dgemm_tcopy_L8_M8_80): tst N, #1 - ble .Ldgemm_tcopy_L8_M8_END + ble L(dgemm_tcopy_L8_M8_END) COPY1x8 -.Ldgemm_tcopy_L8_M8_END: +L(dgemm_tcopy_L8_M8_END): subs J , J, #1 // j-- - bne .Ldgemm_tcopy_L8_M8_BEGIN + bne L(dgemm_tcopy_L8_M8_BEGIN) /*********************************************************************************************/ -.Ldgemm_tcopy_L4_BEGIN: +L(dgemm_tcopy_L4_BEGIN): tst M, #7 - ble .Ldgemm_tcopy_L999 + ble L(dgemm_tcopy_L999) tst M, #4 - ble .Ldgemm_tcopy_L2_BEGIN + ble L(dgemm_tcopy_L2_BEGIN) -.Ldgemm_tcopy_L4_M8_BEGIN: +L(dgemm_tcopy_L4_M8_BEGIN): mov A01, A add A02, A01, LDA @@ -535,51 +535,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, N, #3 // I = N / 8 cmp I, #0 - ble .Ldgemm_tcopy_L4_M8_40 + ble L(dgemm_tcopy_L4_M8_40) .align 5 -.Ldgemm_tcopy_L4_M8_20: +L(dgemm_tcopy_L4_M8_20): COPY8x4 subs I , I , #1 - bne .Ldgemm_tcopy_L4_M8_20 + bne L(dgemm_tcopy_L4_M8_20) -.Ldgemm_tcopy_L4_M8_40: +L(dgemm_tcopy_L4_M8_40): tst N , #4 - ble .Ldgemm_tcopy_L4_M8_60 + ble L(dgemm_tcopy_L4_M8_60) COPY4x4 -.Ldgemm_tcopy_L4_M8_60: +L(dgemm_tcopy_L4_M8_60): tst N , #2 - ble .Ldgemm_tcopy_L4_M8_80 + ble L(dgemm_tcopy_L4_M8_80) COPY2x4 -.Ldgemm_tcopy_L4_M8_80: +L(dgemm_tcopy_L4_M8_80): tst N, #1 - ble .Ldgemm_tcopy_L4_M8_END + ble L(dgemm_tcopy_L4_M8_END) COPY1x4 -.Ldgemm_tcopy_L4_M8_END: +L(dgemm_tcopy_L4_M8_END): /*********************************************************************************************/ -.Ldgemm_tcopy_L2_BEGIN: +L(dgemm_tcopy_L2_BEGIN): tst M, #3 - ble .Ldgemm_tcopy_L999 + ble L(dgemm_tcopy_L999) tst M, #2 - ble .Ldgemm_tcopy_L1_BEGIN + ble L(dgemm_tcopy_L1_BEGIN) -.Ldgemm_tcopy_L2_M8_BEGIN: +L(dgemm_tcopy_L2_M8_BEGIN): mov A01, A add A02, A01, LDA add A, A02, LDA @@ -589,90 +589,90 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, N, #3 // I = N / 8 cmp I, #0 - ble .Ldgemm_tcopy_L2_M8_40 + ble L(dgemm_tcopy_L2_M8_40) .align 5 -.Ldgemm_tcopy_L2_M8_20: +L(dgemm_tcopy_L2_M8_20): COPY8x2 subs I , I , #1 - bne .Ldgemm_tcopy_L2_M8_20 + bne L(dgemm_tcopy_L2_M8_20) -.Ldgemm_tcopy_L2_M8_40: +L(dgemm_tcopy_L2_M8_40): tst N , #4 - ble .Ldgemm_tcopy_L2_M8_60 + ble L(dgemm_tcopy_L2_M8_60) COPY4x2 -.Ldgemm_tcopy_L2_M8_60: +L(dgemm_tcopy_L2_M8_60): tst N , #2 - ble .Ldgemm_tcopy_L2_M8_80 + ble L(dgemm_tcopy_L2_M8_80) COPY2x2 -.Ldgemm_tcopy_L2_M8_80: +L(dgemm_tcopy_L2_M8_80): tst N , #1 - ble .Ldgemm_tcopy_L2_M8_END + ble L(dgemm_tcopy_L2_M8_END) COPY1x2 -.Ldgemm_tcopy_L2_M8_END: +L(dgemm_tcopy_L2_M8_END): /*********************************************************************************************/ -.Ldgemm_tcopy_L1_BEGIN: +L(dgemm_tcopy_L1_BEGIN): tst M, #1 - ble .Ldgemm_tcopy_L999 + ble L(dgemm_tcopy_L999) -.Ldgemm_tcopy_L1_M8_BEGIN: +L(dgemm_tcopy_L1_M8_BEGIN): mov A01, A // A01 = A mov B01, B asr I, N, #3 // I = M / 8 cmp I, #0 - ble .Ldgemm_tcopy_L1_M8_40 + ble L(dgemm_tcopy_L1_M8_40) .align 5 -.Ldgemm_tcopy_L1_M8_20: +L(dgemm_tcopy_L1_M8_20): COPY8x1 subs I , I , #1 - bne .Ldgemm_tcopy_L1_M8_20 + bne L(dgemm_tcopy_L1_M8_20) -.Ldgemm_tcopy_L1_M8_40: +L(dgemm_tcopy_L1_M8_40): tst N , #4 - ble .Ldgemm_tcopy_L1_M8_60 + ble L(dgemm_tcopy_L1_M8_60) COPY4x1 -.Ldgemm_tcopy_L1_M8_60: +L(dgemm_tcopy_L1_M8_60): tst N , #2 - ble .Ldgemm_tcopy_L1_M8_80 + ble L(dgemm_tcopy_L1_M8_80) COPY2x1 -.Ldgemm_tcopy_L1_M8_80: +L(dgemm_tcopy_L1_M8_80): tst N , #1 - ble .Ldgemm_tcopy_L1_M8_END + ble L(dgemm_tcopy_L1_M8_END) COPY1x1 -.Ldgemm_tcopy_L1_M8_END: +L(dgemm_tcopy_L1_M8_END): -.Ldgemm_tcopy_L999: +L(dgemm_tcopy_L999): mov x0, #0 // set return value RESTORE_REGS ret diff --git a/kernel/arm64/dot.S b/kernel/arm64/dot.S index 3f01385fed..e5df75fd5f 100644 --- a/kernel/arm64/dot.S +++ b/kernel/arm64/dot.S @@ -158,51 +158,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif cmp N, xzr - ble .Ldot_kernel_L999 + ble L(dot_kernel_L999) cmp INC_X, #1 - bne .Ldot_kernel_S_BEGIN + bne L(dot_kernel_S_BEGIN) cmp INC_Y, #1 - bne .Ldot_kernel_S_BEGIN + bne L(dot_kernel_S_BEGIN) -.Ldot_kernel_F_BEGIN: +L(dot_kernel_F_BEGIN): asr I, N, #2 cmp I, xzr - beq .Ldot_kernel_F1 + beq L(dot_kernel_F1) -.Ldot_kernel_F4: +L(dot_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Ldot_kernel_F4 + bne L(dot_kernel_F4) KERNEL_F4_FINALIZE -.Ldot_kernel_F1: +L(dot_kernel_F1): ands I, N, #3 - ble .Ldot_kernel_L999 + ble L(dot_kernel_L999) -.Ldot_kernel_F10: +L(dot_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Ldot_kernel_F10 + bne L(dot_kernel_F10) ret -.Ldot_kernel_S_BEGIN: +L(dot_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Ldot_kernel_S1 + ble L(dot_kernel_S1) -.Ldot_kernel_S4: +L(dot_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -210,21 +210,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Ldot_kernel_S4 + bne L(dot_kernel_S4) -.Ldot_kernel_S1: +L(dot_kernel_S1): ands I, N, #3 - ble .Ldot_kernel_L999 + ble L(dot_kernel_L999) -.Ldot_kernel_S10: +L(dot_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Ldot_kernel_S10 + bne L(dot_kernel_S10) -.Ldot_kernel_L999: +L(dot_kernel_L999): ret diff --git a/kernel/arm64/dtrmm_kernel_4x4.S b/kernel/arm64/dtrmm_kernel_4x4.S index b528aeb182..fdfb56e6a4 100644 --- a/kernel/arm64/dtrmm_kernel_4x4.S +++ b/kernel/arm64/dtrmm_kernel_4x4.S @@ -549,11 +549,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Ldtrmm_kernel_L2_BEGIN + ble L(dtrmm_kernel_L2_BEGIN) /******************************************************************************/ -.Ldtrmm_kernel_L4_BEGIN: +L(dtrmm_kernel_L4_BEGIN): mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #2 @@ -563,14 +563,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldtrmm_kernel_L4_M4_BEGIN: +L(dtrmm_kernel_L4_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Ldtrmm_kernel_L4_M2_BEGIN + ble L(dtrmm_kernel_L4_M2_BEGIN) -.Ldtrmm_kernel_L4_M4_20: +L(dtrmm_kernel_L4_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -591,57 +591,57 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Ldtrmm_kernel_L4_M4_32 + blt L(dtrmm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Ldtrmm_kernel_L4_M4_22a + ble L(dtrmm_kernel_L4_M4_22a) .align 5 -.Ldtrmm_kernel_L4_M4_22: +L(dtrmm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M4_22 + bgt L(dtrmm_kernel_L4_M4_22) -.Ldtrmm_kernel_L4_M4_22a: +L(dtrmm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Ldtrmm_kernel_L4_M4_44 + b L(dtrmm_kernel_L4_M4_44) -.Ldtrmm_kernel_L4_M4_32: +L(dtrmm_kernel_L4_M4_32): tst counterL, #1 - ble .Ldtrmm_kernel_L4_M4_40 + ble L(dtrmm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Ldtrmm_kernel_L4_M4_44 + b L(dtrmm_kernel_L4_M4_44) -.Ldtrmm_kernel_L4_M4_40: +L(dtrmm_kernel_L4_M4_40): INIT4x4 -.Ldtrmm_kernel_L4_M4_44: +L(dtrmm_kernel_L4_M4_44): ands counterL , tempK, #1 - ble .Ldtrmm_kernel_L4_M4_100 + ble L(dtrmm_kernel_L4_M4_100) -.Ldtrmm_kernel_L4_M4_46: +L(dtrmm_kernel_L4_M4_46): KERNEL4x4_SUB -.Ldtrmm_kernel_L4_M4_100: +L(dtrmm_kernel_L4_M4_100): SAVE4x4 @@ -660,20 +660,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Ldtrmm_kernel_L4_M4_END: +L(dtrmm_kernel_L4_M4_END): subs counterI, counterI, #1 - bne .Ldtrmm_kernel_L4_M4_20 + bne L(dtrmm_kernel_L4_M4_20) -.Ldtrmm_kernel_L4_M2_BEGIN: +L(dtrmm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldtrmm_kernel_L4_END + ble L(dtrmm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L4_M1_BEGIN + ble L(dtrmm_kernel_L4_M1_BEGIN) -.Ldtrmm_kernel_L4_M2_20: +L(dtrmm_kernel_L4_M2_20): INIT2x4 @@ -697,9 +697,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L4_M2_40 + ble L(dtrmm_kernel_L4_M2_40) -.Ldtrmm_kernel_L4_M2_22: +L(dtrmm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -712,22 +712,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M2_22 + bgt L(dtrmm_kernel_L4_M2_22) -.Ldtrmm_kernel_L4_M2_40: +L(dtrmm_kernel_L4_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L4_M2_100 + ble L(dtrmm_kernel_L4_M2_100) -.Ldtrmm_kernel_L4_M2_42: +L(dtrmm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M2_42 + bgt L(dtrmm_kernel_L4_M2_42) -.Ldtrmm_kernel_L4_M2_100: +L(dtrmm_kernel_L4_M2_100): SAVE2x4 @@ -747,15 +747,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Ldtrmm_kernel_L4_M2_END: +L(dtrmm_kernel_L4_M2_END): -.Ldtrmm_kernel_L4_M1_BEGIN: +L(dtrmm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldtrmm_kernel_L4_END + ble L(dtrmm_kernel_L4_END) -.Ldtrmm_kernel_L4_M1_20: +L(dtrmm_kernel_L4_M1_20): INIT1x4 @@ -779,9 +779,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L4_M1_40 + ble L(dtrmm_kernel_L4_M1_40) -.Ldtrmm_kernel_L4_M1_22: +L(dtrmm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -793,22 +793,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M1_22 + bgt L(dtrmm_kernel_L4_M1_22) -.Ldtrmm_kernel_L4_M1_40: +L(dtrmm_kernel_L4_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L4_M1_100 + ble L(dtrmm_kernel_L4_M1_100) -.Ldtrmm_kernel_L4_M1_42: +L(dtrmm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M1_42 + bgt L(dtrmm_kernel_L4_M1_42) -.Ldtrmm_kernel_L4_M1_100: +L(dtrmm_kernel_L4_M1_100): SAVE1x4 @@ -828,7 +828,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Ldtrmm_kernel_L4_END: +L(dtrmm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 @@ -838,19 +838,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Ldtrmm_kernel_L4_BEGIN + bgt L(dtrmm_kernel_L4_BEGIN) /******************************************************************************/ -.Ldtrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(dtrmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Ldtrmm_kernel_L999 // error, N was less than 4? + ble L(dtrmm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Ldtrmm_kernel_L1_BEGIN + ble L(dtrmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -863,14 +863,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Ldtrmm_kernel_L2_M4_BEGIN: +L(dtrmm_kernel_L2_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI,#0 - ble .Ldtrmm_kernel_L2_M2_BEGIN + ble L(dtrmm_kernel_L2_M2_BEGIN) -.Ldtrmm_kernel_L2_M4_20: +L(dtrmm_kernel_L2_M4_20): INIT4x2 @@ -894,10 +894,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldtrmm_kernel_L2_M4_40 + ble L(dtrmm_kernel_L2_M4_40) .align 5 -.Ldtrmm_kernel_L2_M4_22: +L(dtrmm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -909,22 +909,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M4_22 + bgt L(dtrmm_kernel_L2_M4_22) -.Ldtrmm_kernel_L2_M4_40: +L(dtrmm_kernel_L2_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L2_M4_100 + ble L(dtrmm_kernel_L2_M4_100) -.Ldtrmm_kernel_L2_M4_42: +L(dtrmm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M4_42 + bgt L(dtrmm_kernel_L2_M4_42) -.Ldtrmm_kernel_L2_M4_100: +L(dtrmm_kernel_L2_M4_100): SAVE4x2 @@ -944,22 +944,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Ldtrmm_kernel_L2_M4_END: +L(dtrmm_kernel_L2_M4_END): subs counterI, counterI, #1 - bgt .Ldtrmm_kernel_L2_M4_20 + bgt L(dtrmm_kernel_L2_M4_20) -.Ldtrmm_kernel_L2_M2_BEGIN: +L(dtrmm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldtrmm_kernel_L2_END + ble L(dtrmm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L2_M1_BEGIN + ble L(dtrmm_kernel_L2_M1_BEGIN) -.Ldtrmm_kernel_L2_M2_20: +L(dtrmm_kernel_L2_M2_20): INIT2x2 @@ -983,9 +983,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldtrmm_kernel_L2_M2_40 + ble L(dtrmm_kernel_L2_M2_40) -.Ldtrmm_kernel_L2_M2_22: +L(dtrmm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -998,22 +998,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M2_22 + bgt L(dtrmm_kernel_L2_M2_22) -.Ldtrmm_kernel_L2_M2_40: +L(dtrmm_kernel_L2_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L2_M2_100 + ble L(dtrmm_kernel_L2_M2_100) -.Ldtrmm_kernel_L2_M2_42: +L(dtrmm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M2_42 + bgt L(dtrmm_kernel_L2_M2_42) -.Ldtrmm_kernel_L2_M2_100: +L(dtrmm_kernel_L2_M2_100): SAVE2x2 @@ -1033,15 +1033,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Ldtrmm_kernel_L2_M2_END: +L(dtrmm_kernel_L2_M2_END): -.Ldtrmm_kernel_L2_M1_BEGIN: +L(dtrmm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldtrmm_kernel_L2_END + ble L(dtrmm_kernel_L2_END) -.Ldtrmm_kernel_L2_M1_20: +L(dtrmm_kernel_L2_M1_20): INIT1x2 @@ -1065,9 +1065,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Ldtrmm_kernel_L2_M1_40 + ble L(dtrmm_kernel_L2_M1_40) -.Ldtrmm_kernel_L2_M1_22: +L(dtrmm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1079,22 +1079,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M1_22 + bgt L(dtrmm_kernel_L2_M1_22) -.Ldtrmm_kernel_L2_M1_40: +L(dtrmm_kernel_L2_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L2_M1_100 + ble L(dtrmm_kernel_L2_M1_100) -.Ldtrmm_kernel_L2_M1_42: +L(dtrmm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M1_42 + bgt L(dtrmm_kernel_L2_M1_42) -.Ldtrmm_kernel_L2_M1_100: +L(dtrmm_kernel_L2_M1_100): SAVE1x2 @@ -1114,7 +1114,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Ldtrmm_kernel_L2_END: +L(dtrmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -1122,11 +1122,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Ldtrmm_kernel_L1_BEGIN: +L(dtrmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldtrmm_kernel_L999 // done + ble L(dtrmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1138,14 +1138,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Ldtrmm_kernel_L1_M4_BEGIN: +L(dtrmm_kernel_L1_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Ldtrmm_kernel_L1_M2_BEGIN + ble L(dtrmm_kernel_L1_M2_BEGIN) -.Ldtrmm_kernel_L1_M4_20: +L(dtrmm_kernel_L1_M4_20): INIT4x1 @@ -1169,10 +1169,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L1_M4_40 + ble L(dtrmm_kernel_L1_M4_40) .align 5 -.Ldtrmm_kernel_L1_M4_22: +L(dtrmm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1184,22 +1184,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M4_22 + bgt L(dtrmm_kernel_L1_M4_22) -.Ldtrmm_kernel_L1_M4_40: +L(dtrmm_kernel_L1_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L1_M4_100 + ble L(dtrmm_kernel_L1_M4_100) -.Ldtrmm_kernel_L1_M4_42: +L(dtrmm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M4_42 + bgt L(dtrmm_kernel_L1_M4_42) -.Ldtrmm_kernel_L1_M4_100: +L(dtrmm_kernel_L1_M4_100): SAVE4x1 @@ -1220,22 +1220,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Ldtrmm_kernel_L1_M4_END: +L(dtrmm_kernel_L1_M4_END): subs counterI, counterI, #1 - bgt .Ldtrmm_kernel_L1_M4_20 + bgt L(dtrmm_kernel_L1_M4_20) -.Ldtrmm_kernel_L1_M2_BEGIN: +L(dtrmm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldtrmm_kernel_L1_END + ble L(dtrmm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L1_M1_BEGIN + ble L(dtrmm_kernel_L1_M1_BEGIN) -.Ldtrmm_kernel_L1_M2_20: +L(dtrmm_kernel_L1_M2_20): INIT2x1 @@ -1259,9 +1259,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L1_M2_40 + ble L(dtrmm_kernel_L1_M2_40) -.Ldtrmm_kernel_L1_M2_22: +L(dtrmm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1274,22 +1274,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M2_22 + bgt L(dtrmm_kernel_L1_M2_22) -.Ldtrmm_kernel_L1_M2_40: +L(dtrmm_kernel_L1_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L1_M2_100 + ble L(dtrmm_kernel_L1_M2_100) -.Ldtrmm_kernel_L1_M2_42: +L(dtrmm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M2_42 + bgt L(dtrmm_kernel_L1_M2_42) -.Ldtrmm_kernel_L1_M2_100: +L(dtrmm_kernel_L1_M2_100): SAVE2x1 @@ -1309,15 +1309,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Ldtrmm_kernel_L1_M2_END: +L(dtrmm_kernel_L1_M2_END): -.Ldtrmm_kernel_L1_M1_BEGIN: +L(dtrmm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldtrmm_kernel_L1_END + ble L(dtrmm_kernel_L1_END) -.Ldtrmm_kernel_L1_M1_20: +L(dtrmm_kernel_L1_M1_20): INIT1x1 @@ -1341,9 +1341,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L1_M1_40 + ble L(dtrmm_kernel_L1_M1_40) -.Ldtrmm_kernel_L1_M1_22: +L(dtrmm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1355,30 +1355,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M1_22 + bgt L(dtrmm_kernel_L1_M1_22) -.Ldtrmm_kernel_L1_M1_40: +L(dtrmm_kernel_L1_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L1_M1_100 + ble L(dtrmm_kernel_L1_M1_100) -.Ldtrmm_kernel_L1_M1_42: +L(dtrmm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M1_42 + bgt L(dtrmm_kernel_L1_M1_42) -.Ldtrmm_kernel_L1_M1_100: +L(dtrmm_kernel_L1_M1_100): SAVE1x1 -.Ldtrmm_kernel_L1_END: +L(dtrmm_kernel_L1_END): -.Ldtrmm_kernel_L999: +L(dtrmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dtrmm_kernel_4x8.S b/kernel/arm64/dtrmm_kernel_4x8.S index 47956dec57..969ffae452 100644 --- a/kernel/arm64/dtrmm_kernel_4x8.S +++ b/kernel/arm64/dtrmm_kernel_4x8.S @@ -900,11 +900,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Ldtrmm_kernel_L4_BEGIN + ble L(dtrmm_kernel_L4_BEGIN) /******************************************************************************/ -.Ldtrmm_kernel_L8_BEGIN: +L(dtrmm_kernel_L8_BEGIN): mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #3 @@ -915,14 +915,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldtrmm_kernel_L8_M4_BEGIN: +L(dtrmm_kernel_L8_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Ldtrmm_kernel_L8_M2_BEGIN + ble L(dtrmm_kernel_L8_M2_BEGIN) -.Ldtrmm_kernel_L8_M4_20: +L(dtrmm_kernel_L8_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -944,57 +944,57 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL, tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Ldtrmm_kernel_L8_M4_32 + blt L(dtrmm_kernel_L8_M4_32) KERNEL4x8_I // do one in the K KERNEL4x8_M2 // do another in the K subs counterL, counterL, #2 - ble .Ldtrmm_kernel_L8_M4_22a + ble L(dtrmm_kernel_L8_M4_22a) .align 5 -.Ldtrmm_kernel_L8_M4_22: +L(dtrmm_kernel_L8_M4_22): KERNEL4x8_M1 KERNEL4x8_M2 subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L8_M4_22 + bgt L(dtrmm_kernel_L8_M4_22) -.Ldtrmm_kernel_L8_M4_22a: +L(dtrmm_kernel_L8_M4_22a): KERNEL4x8_M1 KERNEL4x8_E - b .Ldtrmm_kernel_L8_M4_44 + b L(dtrmm_kernel_L8_M4_44) -.Ldtrmm_kernel_L8_M4_32: +L(dtrmm_kernel_L8_M4_32): tst counterL, #1 - ble .Ldtrmm_kernel_L8_M4_40 + ble L(dtrmm_kernel_L8_M4_40) KERNEL4x8_I KERNEL4x8_E - b .Ldtrmm_kernel_L8_M4_44 + b L(dtrmm_kernel_L8_M4_44) -.Ldtrmm_kernel_L8_M4_40: +L(dtrmm_kernel_L8_M4_40): INIT4x8 -.Ldtrmm_kernel_L8_M4_44: +L(dtrmm_kernel_L8_M4_44): ands counterL, tempK, #1 - ble .Ldtrmm_kernel_L8_M4_100 + ble L(dtrmm_kernel_L8_M4_100) -.Ldtrmm_kernel_L8_M4_46: +L(dtrmm_kernel_L8_M4_46): KERNEL4x8_SUB -.Ldtrmm_kernel_L8_M4_100: +L(dtrmm_kernel_L8_M4_100): SAVE4x8 @@ -1014,20 +1014,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Ldtrmm_kernel_L8_M4_END: +L(dtrmm_kernel_L8_M4_END): subs counterI, counterI, #1 - bne .Ldtrmm_kernel_L8_M4_20 + bne L(dtrmm_kernel_L8_M4_20) -.Ldtrmm_kernel_L8_M2_BEGIN: +L(dtrmm_kernel_L8_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldtrmm_kernel_L8_END + ble L(dtrmm_kernel_L8_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L8_M1_BEGIN + ble L(dtrmm_kernel_L8_M1_BEGIN) -.Ldtrmm_kernel_L8_M2_20: +L(dtrmm_kernel_L8_M2_20): INIT2x8 @@ -1051,9 +1051,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL, tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L8_M2_40 + ble L(dtrmm_kernel_L8_M2_40) -.Ldtrmm_kernel_L8_M2_22: +L(dtrmm_kernel_L8_M2_22): KERNEL2x8_SUB KERNEL2x8_SUB @@ -1066,22 +1066,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L8_M2_22 + bgt L(dtrmm_kernel_L8_M2_22) -.Ldtrmm_kernel_L8_M2_40: +L(dtrmm_kernel_L8_M2_40): ands counterL, tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L8_M2_100 + ble L(dtrmm_kernel_L8_M2_100) -.Ldtrmm_kernel_L8_M2_42: +L(dtrmm_kernel_L8_M2_42): KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L8_M2_42 + bgt L(dtrmm_kernel_L8_M2_42) -.Ldtrmm_kernel_L8_M2_100: +L(dtrmm_kernel_L8_M2_100): SAVE2x8 @@ -1102,15 +1102,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Ldtrmm_kernel_L8_M2_END: +L(dtrmm_kernel_L8_M2_END): -.Ldtrmm_kernel_L8_M1_BEGIN: +L(dtrmm_kernel_L8_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldtrmm_kernel_L8_END + ble L(dtrmm_kernel_L8_END) -.Ldtrmm_kernel_L8_M1_20: +L(dtrmm_kernel_L8_M1_20): INIT1x8 @@ -1134,9 +1134,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL, tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L8_M1_40 + ble L(dtrmm_kernel_L8_M1_40) -.Ldtrmm_kernel_L8_M1_22: +L(dtrmm_kernel_L8_M1_22): KERNEL1x8_SUB KERNEL1x8_SUB KERNEL1x8_SUB @@ -1148,22 +1148,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L8_M1_22 + bgt L(dtrmm_kernel_L8_M1_22) -.Ldtrmm_kernel_L8_M1_40: +L(dtrmm_kernel_L8_M1_40): ands counterL, tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L8_M1_100 + ble L(dtrmm_kernel_L8_M1_100) -.Ldtrmm_kernel_L8_M1_42: +L(dtrmm_kernel_L8_M1_42): KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L8_M1_42 + bgt L(dtrmm_kernel_L8_M1_42) -.Ldtrmm_kernel_L8_M1_100: +L(dtrmm_kernel_L8_M1_100): SAVE1x8 @@ -1183,7 +1183,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Ldtrmm_kernel_L8_END: +L(dtrmm_kernel_L8_END): lsl temp, origK, #6 add origPB, origPB, temp // B = B + K * 8 * 8 @@ -1193,19 +1193,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Ldtrmm_kernel_L8_BEGIN + bgt L(dtrmm_kernel_L8_BEGIN) /******************************************************************************/ -.Ldtrmm_kernel_L4_BEGIN: +L(dtrmm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #7 - ble .Ldtrmm_kernel_L999 + ble L(dtrmm_kernel_L999) tst counterJ , #4 - ble .Ldtrmm_kernel_L2_BEGIN + ble L(dtrmm_kernel_L2_BEGIN) mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #2 @@ -1216,14 +1216,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldtrmm_kernel_L4_M4_BEGIN: +L(dtrmm_kernel_L4_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Ldtrmm_kernel_L4_M2_BEGIN + ble L(dtrmm_kernel_L4_M2_BEGIN) -.Ldtrmm_kernel_L4_M4_20: +L(dtrmm_kernel_L4_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1244,57 +1244,57 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL, tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Ldtrmm_kernel_L4_M4_32 + blt L(dtrmm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Ldtrmm_kernel_L4_M4_22a + ble L(dtrmm_kernel_L4_M4_22a) .align 5 -.Ldtrmm_kernel_L4_M4_22: +L(dtrmm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M4_22 + bgt L(dtrmm_kernel_L4_M4_22) -.Ldtrmm_kernel_L4_M4_22a: +L(dtrmm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Ldtrmm_kernel_L4_M4_44 + b L(dtrmm_kernel_L4_M4_44) -.Ldtrmm_kernel_L4_M4_32: +L(dtrmm_kernel_L4_M4_32): tst counterL, #1 - ble .Ldtrmm_kernel_L4_M4_40 + ble L(dtrmm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Ldtrmm_kernel_L4_M4_44 + b L(dtrmm_kernel_L4_M4_44) -.Ldtrmm_kernel_L4_M4_40: +L(dtrmm_kernel_L4_M4_40): INIT4x4 -.Ldtrmm_kernel_L4_M4_44: +L(dtrmm_kernel_L4_M4_44): ands counterL , tempK, #1 - ble .Ldtrmm_kernel_L4_M4_100 + ble L(dtrmm_kernel_L4_M4_100) -.Ldtrmm_kernel_L4_M4_46: +L(dtrmm_kernel_L4_M4_46): KERNEL4x4_SUB -.Ldtrmm_kernel_L4_M4_100: +L(dtrmm_kernel_L4_M4_100): SAVE4x4 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) @@ -1312,20 +1312,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Ldtrmm_kernel_L4_M4_END: +L(dtrmm_kernel_L4_M4_END): subs counterI, counterI, #1 - bne .Ldtrmm_kernel_L4_M4_20 + bne L(dtrmm_kernel_L4_M4_20) -.Ldtrmm_kernel_L4_M2_BEGIN: +L(dtrmm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldtrmm_kernel_L4_END + ble L(dtrmm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L4_M1_BEGIN + ble L(dtrmm_kernel_L4_M1_BEGIN) -.Ldtrmm_kernel_L4_M2_20: +L(dtrmm_kernel_L4_M2_20): INIT2x4 @@ -1348,9 +1348,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L4_M2_40 + ble L(dtrmm_kernel_L4_M2_40) -.Ldtrmm_kernel_L4_M2_22: +L(dtrmm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1363,22 +1363,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M2_22 + bgt L(dtrmm_kernel_L4_M2_22) -.Ldtrmm_kernel_L4_M2_40: +L(dtrmm_kernel_L4_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L4_M2_100 + ble L(dtrmm_kernel_L4_M2_100) -.Ldtrmm_kernel_L4_M2_42: +L(dtrmm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M2_42 + bgt L(dtrmm_kernel_L4_M2_42) -.Ldtrmm_kernel_L4_M2_100: +L(dtrmm_kernel_L4_M2_100): SAVE2x4 @@ -1397,15 +1397,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #2 #endif -.Ldtrmm_kernel_L4_M2_END: +L(dtrmm_kernel_L4_M2_END): -.Ldtrmm_kernel_L4_M1_BEGIN: +L(dtrmm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldtrmm_kernel_L4_END + ble L(dtrmm_kernel_L4_END) -.Ldtrmm_kernel_L4_M1_20: +L(dtrmm_kernel_L4_M1_20): INIT1x4 @@ -1428,9 +1428,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L4_M1_40 + ble L(dtrmm_kernel_L4_M1_40) -.Ldtrmm_kernel_L4_M1_22: +L(dtrmm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1442,22 +1442,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M1_22 + bgt L(dtrmm_kernel_L4_M1_22) -.Ldtrmm_kernel_L4_M1_40: +L(dtrmm_kernel_L4_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L4_M1_100 + ble L(dtrmm_kernel_L4_M1_100) -.Ldtrmm_kernel_L4_M1_42: +L(dtrmm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M1_42 + bgt L(dtrmm_kernel_L4_M1_42) -.Ldtrmm_kernel_L4_M1_100: +L(dtrmm_kernel_L4_M1_100): SAVE1x4 @@ -1476,7 +1476,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #1 #endif -.Ldtrmm_kernel_L4_END: +L(dtrmm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 @@ -1486,14 +1486,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Ldtrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(dtrmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Ldtrmm_kernel_L999 // error, N was less than 4? + ble L(dtrmm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Ldtrmm_kernel_L1_BEGIN + ble L(dtrmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1505,14 +1505,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Ldtrmm_kernel_L2_M4_BEGIN: +L(dtrmm_kernel_L2_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI,#0 - ble .Ldtrmm_kernel_L2_M2_BEGIN + ble L(dtrmm_kernel_L2_M2_BEGIN) -.Ldtrmm_kernel_L2_M4_20: +L(dtrmm_kernel_L2_M4_20): INIT4x2 @@ -1535,10 +1535,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldtrmm_kernel_L2_M4_40 + ble L(dtrmm_kernel_L2_M4_40) .align 5 -.Ldtrmm_kernel_L2_M4_22: +L(dtrmm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1550,22 +1550,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M4_22 + bgt L(dtrmm_kernel_L2_M4_22) -.Ldtrmm_kernel_L2_M4_40: +L(dtrmm_kernel_L2_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L2_M4_100 + ble L(dtrmm_kernel_L2_M4_100) -.Ldtrmm_kernel_L2_M4_42: +L(dtrmm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M4_42 + bgt L(dtrmm_kernel_L2_M4_42) -.Ldtrmm_kernel_L2_M4_100: +L(dtrmm_kernel_L2_M4_100): SAVE4x2 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) @@ -1584,22 +1584,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Ldtrmm_kernel_L2_M4_END: +L(dtrmm_kernel_L2_M4_END): subs counterI, counterI, #1 - bgt .Ldtrmm_kernel_L2_M4_20 + bgt L(dtrmm_kernel_L2_M4_20) -.Ldtrmm_kernel_L2_M2_BEGIN: +L(dtrmm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldtrmm_kernel_L2_END + ble L(dtrmm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L2_M1_BEGIN + ble L(dtrmm_kernel_L2_M1_BEGIN) -.Ldtrmm_kernel_L2_M2_20: +L(dtrmm_kernel_L2_M2_20): INIT2x2 @@ -1622,9 +1622,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldtrmm_kernel_L2_M2_40 + ble L(dtrmm_kernel_L2_M2_40) -.Ldtrmm_kernel_L2_M2_22: +L(dtrmm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1637,22 +1637,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M2_22 + bgt L(dtrmm_kernel_L2_M2_22) -.Ldtrmm_kernel_L2_M2_40: +L(dtrmm_kernel_L2_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L2_M2_100 + ble L(dtrmm_kernel_L2_M2_100) -.Ldtrmm_kernel_L2_M2_42: +L(dtrmm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M2_42 + bgt L(dtrmm_kernel_L2_M2_42) -.Ldtrmm_kernel_L2_M2_100: +L(dtrmm_kernel_L2_M2_100): SAVE2x2 @@ -1671,15 +1671,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #2 #endif -.Ldtrmm_kernel_L2_M2_END: +L(dtrmm_kernel_L2_M2_END): -.Ldtrmm_kernel_L2_M1_BEGIN: +L(dtrmm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldtrmm_kernel_L2_END + ble L(dtrmm_kernel_L2_END) -.Ldtrmm_kernel_L2_M1_20: +L(dtrmm_kernel_L2_M1_20): INIT1x2 @@ -1702,9 +1702,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Ldtrmm_kernel_L2_M1_40 + ble L(dtrmm_kernel_L2_M1_40) -.Ldtrmm_kernel_L2_M1_22: +L(dtrmm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1716,22 +1716,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M1_22 + bgt L(dtrmm_kernel_L2_M1_22) -.Ldtrmm_kernel_L2_M1_40: +L(dtrmm_kernel_L2_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L2_M1_100 + ble L(dtrmm_kernel_L2_M1_100) -.Ldtrmm_kernel_L2_M1_42: +L(dtrmm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M1_42 + bgt L(dtrmm_kernel_L2_M1_42) -.Ldtrmm_kernel_L2_M1_100: +L(dtrmm_kernel_L2_M1_100): SAVE1x2 @@ -1750,7 +1750,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #1 #endif -.Ldtrmm_kernel_L2_END: +L(dtrmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -1758,11 +1758,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Ldtrmm_kernel_L1_BEGIN: +L(dtrmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldtrmm_kernel_L999 // done + ble L(dtrmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1773,14 +1773,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = A -.Ldtrmm_kernel_L1_M4_BEGIN: +L(dtrmm_kernel_L1_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Ldtrmm_kernel_L1_M2_BEGIN + ble L(dtrmm_kernel_L1_M2_BEGIN) -.Ldtrmm_kernel_L1_M4_20: +L(dtrmm_kernel_L1_M4_20): INIT4x1 @@ -1802,10 +1802,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L1_M4_40 + ble L(dtrmm_kernel_L1_M4_40) .align 5 -.Ldtrmm_kernel_L1_M4_22: +L(dtrmm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1817,22 +1817,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M4_22 + bgt L(dtrmm_kernel_L1_M4_22) -.Ldtrmm_kernel_L1_M4_40: +L(dtrmm_kernel_L1_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L1_M4_100 + ble L(dtrmm_kernel_L1_M4_100) -.Ldtrmm_kernel_L1_M4_42: +L(dtrmm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M4_42 + bgt L(dtrmm_kernel_L1_M4_42) -.Ldtrmm_kernel_L1_M4_100: +L(dtrmm_kernel_L1_M4_100): SAVE4x1 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) @@ -1851,22 +1851,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Ldtrmm_kernel_L1_M4_END: +L(dtrmm_kernel_L1_M4_END): subs counterI, counterI, #1 - bgt .Ldtrmm_kernel_L1_M4_20 + bgt L(dtrmm_kernel_L1_M4_20) -.Ldtrmm_kernel_L1_M2_BEGIN: +L(dtrmm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldtrmm_kernel_L1_END + ble L(dtrmm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L1_M1_BEGIN + ble L(dtrmm_kernel_L1_M1_BEGIN) -.Ldtrmm_kernel_L1_M2_20: +L(dtrmm_kernel_L1_M2_20): INIT2x1 @@ -1889,9 +1889,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L1_M2_40 + ble L(dtrmm_kernel_L1_M2_40) -.Ldtrmm_kernel_L1_M2_22: +L(dtrmm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1904,22 +1904,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M2_22 + bgt L(dtrmm_kernel_L1_M2_22) -.Ldtrmm_kernel_L1_M2_40: +L(dtrmm_kernel_L1_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L1_M2_100 + ble L(dtrmm_kernel_L1_M2_100) -.Ldtrmm_kernel_L1_M2_42: +L(dtrmm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M2_42 + bgt L(dtrmm_kernel_L1_M2_42) -.Ldtrmm_kernel_L1_M2_100: +L(dtrmm_kernel_L1_M2_100): SAVE2x1 @@ -1938,15 +1938,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #2 #endif -.Ldtrmm_kernel_L1_M2_END: +L(dtrmm_kernel_L1_M2_END): -.Ldtrmm_kernel_L1_M1_BEGIN: +L(dtrmm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldtrmm_kernel_L1_END + ble L(dtrmm_kernel_L1_END) -.Ldtrmm_kernel_L1_M1_20: +L(dtrmm_kernel_L1_M1_20): INIT1x1 @@ -1969,9 +1969,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L1_M1_40 + ble L(dtrmm_kernel_L1_M1_40) -.Ldtrmm_kernel_L1_M1_22: +L(dtrmm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1983,30 +1983,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M1_22 + bgt L(dtrmm_kernel_L1_M1_22) -.Ldtrmm_kernel_L1_M1_40: +L(dtrmm_kernel_L1_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L1_M1_100 + ble L(dtrmm_kernel_L1_M1_100) -.Ldtrmm_kernel_L1_M1_42: +L(dtrmm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M1_42 + bgt L(dtrmm_kernel_L1_M1_42) -.Ldtrmm_kernel_L1_M1_100: +L(dtrmm_kernel_L1_M1_100): SAVE1x1 -.Ldtrmm_kernel_L1_END: +L(dtrmm_kernel_L1_END): -.Ldtrmm_kernel_L999: +L(dtrmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dtrmm_kernel_6x8_cortexa72.S b/kernel/arm64/dtrmm_kernel_6x8_cortexa72.S index 749e58a809..5cdf668652 100644 --- a/kernel/arm64/dtrmm_kernel_6x8_cortexa72.S +++ b/kernel/arm64/dtrmm_kernel_6x8_cortexa72.S @@ -715,9 +715,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 - cbz counterJ, .Ldtrmm6x8_L4_BEGIN + cbz counterJ, L(dtrmm6x8_L4_BEGIN) -.Ldtrmm6x8_L8_BEGIN: +L(dtrmm6x8_L8_BEGIN): mov pCRow0, pC add pC, pC, LDC, lsl #3 #if defined(LEFT) @@ -727,9 +727,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov temp, #6 udiv counterI, origM, temp - cbz counterI, .Ldtrmm6x8_L8_Mrem + cbz counterI, L(dtrmm6x8_L8_Mrem) -.Ldtrmm6x8_L8_M6: +L(dtrmm6x8_L8_M6): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -751,12 +751,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT6x8 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L8_M6s -.Ldtrmm6x8_L8_M6l: + cbz counterL, L(dtrmm6x8_L8_M6s) +L(dtrmm6x8_L8_M6l): KERNEL6x8_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L8_M6l -.Ldtrmm6x8_L8_M6s: + bgt L(dtrmm6x8_L8_M6l) +L(dtrmm6x8_L8_M6s): SAVE6x8 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -777,14 +777,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterI, counterI, #1 - bgt .Ldtrmm6x8_L8_M6 + bgt L(dtrmm6x8_L8_M6) -.Ldtrmm6x8_L8_Mrem: +L(dtrmm6x8_L8_Mrem): mov temp, #6 udiv counterI, origM, temp msub counterI, counterI, temp, origM cmp counterI, #4 - blt .Ldtrmm6x8_L8_M2c + blt L(dtrmm6x8_L8_M2c) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -806,12 +806,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT4x8 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L8_M4s -.Ldtrmm6x8_L8_M4l: + cbz counterL, L(dtrmm6x8_L8_M4s) +L(dtrmm6x8_L8_M4l): KERNEL4x8_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L8_M4l -.Ldtrmm6x8_L8_M4s: + bgt L(dtrmm6x8_L8_M4l) +L(dtrmm6x8_L8_M4s): SAVE4x8 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -832,9 +832,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif sub counterI, counterI, #4 -.Ldtrmm6x8_L8_M2c: +L(dtrmm6x8_L8_M2c): cmp counterI, #2 - blt .Ldtrmm6x8_L8_M1c + blt L(dtrmm6x8_L8_M1c) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -856,12 +856,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT2x8 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L8_M2s -.Ldtrmm6x8_L8_M2l: + cbz counterL, L(dtrmm6x8_L8_M2s) +L(dtrmm6x8_L8_M2l): KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L8_M2l -.Ldtrmm6x8_L8_M2s: + bgt L(dtrmm6x8_L8_M2l) +L(dtrmm6x8_L8_M2s): SAVE2x8 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -882,8 +882,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif sub counterI, counterI, #2 -.Ldtrmm6x8_L8_M1c: - cbz counterI, .Ldtrmm6x8_L8_END +L(dtrmm6x8_L8_M1c): + cbz counterI, L(dtrmm6x8_L8_END) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -905,12 +905,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT1x8 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L8_M1s -.Ldtrmm6x8_L8_M1l: + cbz counterL, L(dtrmm6x8_L8_M1s) +L(dtrmm6x8_L8_M1l): KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L8_M1l -.Ldtrmm6x8_L8_M1s: + bgt L(dtrmm6x8_L8_M1l) +L(dtrmm6x8_L8_M1s): SAVE1x8 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -930,20 +930,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Ldtrmm6x8_L8_END: +L(dtrmm6x8_L8_END): lsl temp, origK, #6 add origPB, origPB, temp #if !defined(LEFT) add tempOffset, tempOffset, #8 #endif subs counterJ, counterJ, #1 - bgt .Ldtrmm6x8_L8_BEGIN + bgt L(dtrmm6x8_L8_BEGIN) -.Ldtrmm6x8_L4_BEGIN: +L(dtrmm6x8_L4_BEGIN): tst origN, #7 - beq .Ldtrmm6x8_L999 + beq L(dtrmm6x8_L999) tst origN, #4 - beq .Ldtrmm6x8_L2_BEGIN + beq L(dtrmm6x8_L2_BEGIN) mov pCRow0, pC add pC, pC, LDC, lsl #2 @@ -954,8 +954,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov temp, #6 udiv counterI, origM, temp - cbz counterI, .Ldtrmm6x8_L4_Mrem -.Ldtrmm6x8_L4_M6: + cbz counterI, L(dtrmm6x8_L4_Mrem) +L(dtrmm6x8_L4_M6): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -977,12 +977,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT6x4 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L4_M6s -.Ldtrmm6x8_L4_M6l: + cbz counterL, L(dtrmm6x8_L4_M6s) +L(dtrmm6x8_L4_M6l): KERNEL6x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L4_M6l -.Ldtrmm6x8_L4_M6s: + bgt L(dtrmm6x8_L4_M6l) +L(dtrmm6x8_L4_M6s): SAVE6x4 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1003,13 +1003,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterI, counterI, #1 - bgt .Ldtrmm6x8_L4_M6 -.Ldtrmm6x8_L4_Mrem: + bgt L(dtrmm6x8_L4_M6) +L(dtrmm6x8_L4_Mrem): mov temp, #6 udiv counterI, origM, temp msub counterI, counterI, temp, origM cmp counterI, #4 - blt .Ldtrmm6x8_L4_M2c + blt L(dtrmm6x8_L4_M2c) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -1031,12 +1031,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT4x4 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L4_M4s -.Ldtrmm6x8_L4_M4l: + cbz counterL, L(dtrmm6x8_L4_M4s) +L(dtrmm6x8_L4_M4l): KERNEL4x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L4_M4l -.Ldtrmm6x8_L4_M4s: + bgt L(dtrmm6x8_L4_M4l) +L(dtrmm6x8_L4_M4s): SAVE4x4 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1057,9 +1057,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif sub counterI, counterI, #4 -.Ldtrmm6x8_L4_M2c: +L(dtrmm6x8_L4_M2c): cmp counterI, #2 - blt .Ldtrmm6x8_L4_M1c + blt L(dtrmm6x8_L4_M1c) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -1081,12 +1081,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT2x4 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L4_M2s -.Ldtrmm6x8_L4_M2l: + cbz counterL, L(dtrmm6x8_L4_M2s) +L(dtrmm6x8_L4_M2l): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L4_M2l -.Ldtrmm6x8_L4_M2s: + bgt L(dtrmm6x8_L4_M2l) +L(dtrmm6x8_L4_M2s): SAVE2x4 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1107,8 +1107,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif sub counterI, counterI, #2 -.Ldtrmm6x8_L4_M1c: - cbz counterI, .Ldtrmm6x8_L4_END +L(dtrmm6x8_L4_M1c): + cbz counterI, L(dtrmm6x8_L4_END) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -1130,12 +1130,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT1x4 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L4_M1s -.Ldtrmm6x8_L4_M1l: + cbz counterL, L(dtrmm6x8_L4_M1s) +L(dtrmm6x8_L4_M1l): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L4_M1l -.Ldtrmm6x8_L4_M1s: + bgt L(dtrmm6x8_L4_M1l) +L(dtrmm6x8_L4_M1s): SAVE1x4 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1155,15 +1155,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Ldtrmm6x8_L4_END: +L(dtrmm6x8_L4_END): add origPB, origPB, origK, lsl #5 #if !defined(LEFT) add tempOffset, tempOffset, #4 #endif -.Ldtrmm6x8_L2_BEGIN: +L(dtrmm6x8_L2_BEGIN): tst origN, #2 - beq .Ldtrmm6x8_L1_BEGIN + beq L(dtrmm6x8_L1_BEGIN) mov pCRow0, pC add pC, pC, LDC, lsl #1 #if defined(LEFT) @@ -1172,8 +1172,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA mov temp, #6 udiv counterI, origM, temp - cbz counterI, .Ldtrmm6x8_L2_Mrem -.Ldtrmm6x8_L2_M6: + cbz counterI, L(dtrmm6x8_L2_Mrem) +L(dtrmm6x8_L2_M6): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -1195,12 +1195,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT6x2 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L2_M6s -.Ldtrmm6x8_L2_M6l: + cbz counterL, L(dtrmm6x8_L2_M6s) +L(dtrmm6x8_L2_M6l): KERNEL6x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L2_M6l -.Ldtrmm6x8_L2_M6s: + bgt L(dtrmm6x8_L2_M6l) +L(dtrmm6x8_L2_M6s): SAVE6x2 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1221,13 +1221,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterI, counterI, #1 - bgt .Ldtrmm6x8_L2_M6 -.Ldtrmm6x8_L2_Mrem: + bgt L(dtrmm6x8_L2_M6) +L(dtrmm6x8_L2_Mrem): mov temp, #6 udiv counterI, origM, temp msub counterI, counterI, temp, origM cmp counterI, #4 - blt .Ldtrmm6x8_L2_M2c + blt L(dtrmm6x8_L2_M2c) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -1249,12 +1249,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT4x2 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L2_M4s -.Ldtrmm6x8_L2_M4l: + cbz counterL, L(dtrmm6x8_L2_M4s) +L(dtrmm6x8_L2_M4l): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L2_M4l -.Ldtrmm6x8_L2_M4s: + bgt L(dtrmm6x8_L2_M4l) +L(dtrmm6x8_L2_M4s): SAVE4x2 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1275,9 +1275,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif sub counterI, counterI, #4 -.Ldtrmm6x8_L2_M2c: +L(dtrmm6x8_L2_M2c): cmp counterI, #2 - blt .Ldtrmm6x8_L2_M1c + blt L(dtrmm6x8_L2_M1c) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -1299,12 +1299,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT2x2 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L2_M2s -.Ldtrmm6x8_L2_M2l: + cbz counterL, L(dtrmm6x8_L2_M2s) +L(dtrmm6x8_L2_M2l): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L2_M2l -.Ldtrmm6x8_L2_M2s: + bgt L(dtrmm6x8_L2_M2l) +L(dtrmm6x8_L2_M2s): SAVE2x2 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1325,8 +1325,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif sub counterI, counterI, #2 -.Ldtrmm6x8_L2_M1c: - cbz counterI, .Ldtrmm6x8_L2_END +L(dtrmm6x8_L2_M1c): + cbz counterI, L(dtrmm6x8_L2_END) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -1348,12 +1348,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT1x2 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L2_M1s -.Ldtrmm6x8_L2_M1l: + cbz counterL, L(dtrmm6x8_L2_M1s) +L(dtrmm6x8_L2_M1l): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L2_M1l -.Ldtrmm6x8_L2_M1s: + bgt L(dtrmm6x8_L2_M1l) +L(dtrmm6x8_L2_M1s): SAVE1x2 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1373,15 +1373,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Ldtrmm6x8_L2_END: +L(dtrmm6x8_L2_END): add origPB, origPB, origK, lsl #4 #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif -.Ldtrmm6x8_L1_BEGIN: +L(dtrmm6x8_L1_BEGIN): tst origN, #1 - beq .Ldtrmm6x8_L999 + beq L(dtrmm6x8_L999) mov pCRow0, pC #if defined(LEFT) mov tempOffset, offset @@ -1389,8 +1389,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA mov temp, #6 udiv counterI, origM, temp - cbz counterI, .Ldtrmm6x8_L1_Mrem -.Ldtrmm6x8_L1_M6: + cbz counterI, L(dtrmm6x8_L1_Mrem) +L(dtrmm6x8_L1_M6): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -1412,12 +1412,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT6x1 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L1_M6s -.Ldtrmm6x8_L1_M6l: + cbz counterL, L(dtrmm6x8_L1_M6s) +L(dtrmm6x8_L1_M6l): KERNEL6x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L1_M6l -.Ldtrmm6x8_L1_M6s: + bgt L(dtrmm6x8_L1_M6l) +L(dtrmm6x8_L1_M6s): SAVE6x1 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1438,13 +1438,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterI, counterI, #1 - bgt .Ldtrmm6x8_L1_M6 -.Ldtrmm6x8_L1_Mrem: + bgt L(dtrmm6x8_L1_M6) +L(dtrmm6x8_L1_Mrem): mov temp, #6 udiv counterI, origM, temp msub counterI, counterI, temp, origM cmp counterI, #4 - blt .Ldtrmm6x8_L1_M2c + blt L(dtrmm6x8_L1_M2c) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -1466,12 +1466,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT4x1 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L1_M4s -.Ldtrmm6x8_L1_M4l: + cbz counterL, L(dtrmm6x8_L1_M4s) +L(dtrmm6x8_L1_M4l): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L1_M4l -.Ldtrmm6x8_L1_M4s: + bgt L(dtrmm6x8_L1_M4l) +L(dtrmm6x8_L1_M4s): SAVE4x1 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1492,9 +1492,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif sub counterI, counterI, #4 -.Ldtrmm6x8_L1_M2c: +L(dtrmm6x8_L1_M2c): cmp counterI, #2 - blt .Ldtrmm6x8_L1_M1c + blt L(dtrmm6x8_L1_M1c) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -1516,12 +1516,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT2x1 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L1_M2s -.Ldtrmm6x8_L1_M2l: + cbz counterL, L(dtrmm6x8_L1_M2s) +L(dtrmm6x8_L1_M2l): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L1_M2l -.Ldtrmm6x8_L1_M2s: + bgt L(dtrmm6x8_L1_M2l) +L(dtrmm6x8_L1_M2s): SAVE2x1 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1542,8 +1542,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif sub counterI, counterI, #2 -.Ldtrmm6x8_L1_M1c: - cbz counterI, .Ldtrmm6x8_L999 +L(dtrmm6x8_L1_M1c): + cbz counterI, L(dtrmm6x8_L999) #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB #else @@ -1565,12 +1565,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT1x1 mov counterL, tempK - cbz counterL, .Ldtrmm6x8_L1_M1s -.Ldtrmm6x8_L1_M1l: + cbz counterL, L(dtrmm6x8_L1_M1s) +L(dtrmm6x8_L1_M1l): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm6x8_L1_M1l -.Ldtrmm6x8_L1_M1s: + bgt L(dtrmm6x8_L1_M1l) +L(dtrmm6x8_L1_M1s): SAVE1x1 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -1590,7 +1590,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Ldtrmm6x8_L999: +L(dtrmm6x8_L999): mov x0, #0 ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dtrmm_kernel_8x4.S b/kernel/arm64/dtrmm_kernel_8x4.S index 3d953266ca..45c861dff7 100644 --- a/kernel/arm64/dtrmm_kernel_8x4.S +++ b/kernel/arm64/dtrmm_kernel_8x4.S @@ -830,11 +830,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Ldtrmm_kernel_L2_BEGIN + ble L(dtrmm_kernel_L2_BEGIN) /******************************************************************************/ -.Ldtrmm_kernel_L4_BEGIN: +L(dtrmm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -848,15 +848,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = start of A array -.Ldtrmm_kernel_L4_M8_BEGIN: +L(dtrmm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Ldtrmm_kernel_L4_M4_BEGIN + ble L(dtrmm_kernel_L4_M4_BEGIN) .align 5 -.Ldtrmm_kernel_L4_M8_20: +L(dtrmm_kernel_L4_M8_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -878,7 +878,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Ldtrmm_kernel_L4_M8_32 + blt L(dtrmm_kernel_L4_M8_32) KERNEL8x4_I // do one in the K KERNEL8x4_M2 // do another in the K @@ -890,10 +890,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M2 subs counterL, counterL, #2 // subtract 2 - ble .Ldtrmm_kernel_L4_M8_22a + ble L(dtrmm_kernel_L4_M8_22a) .align 5 -.Ldtrmm_kernel_L4_M8_22: +L(dtrmm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 @@ -905,10 +905,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M8_22 + bgt L(dtrmm_kernel_L4_M8_22) .align 5 -.Ldtrmm_kernel_L4_M8_22a: +L(dtrmm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_M2 @@ -919,13 +919,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Ldtrmm_kernel_L4_M8_44 + b L(dtrmm_kernel_L4_M8_44) .align 5 -.Ldtrmm_kernel_L4_M8_32: +L(dtrmm_kernel_L4_M8_32): tst counterL, #1 - ble .Ldtrmm_kernel_L4_M8_40 + ble L(dtrmm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_M2 @@ -936,26 +936,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_M1 KERNEL8x4_E - b .Ldtrmm_kernel_L4_M8_44 + b L(dtrmm_kernel_L4_M8_44) -.Ldtrmm_kernel_L4_M8_40: +L(dtrmm_kernel_L4_M8_40): INIT8x4 -.Ldtrmm_kernel_L4_M8_44: +L(dtrmm_kernel_L4_M8_44): ands counterL , tempK, #7 - ble .Ldtrmm_kernel_L4_M8_100 + ble L(dtrmm_kernel_L4_M8_100) .align 5 -.Ldtrmm_kernel_L4_M8_46: +L(dtrmm_kernel_L4_M8_46): KERNEL8x4_SUB subs counterL, counterL, #1 - bne .Ldtrmm_kernel_L4_M8_46 + bne L(dtrmm_kernel_L4_M8_46) -.Ldtrmm_kernel_L4_M8_100: +L(dtrmm_kernel_L4_M8_100): SAVE8x4 @@ -978,20 +978,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] -.Ldtrmm_kernel_L4_M8_END: +L(dtrmm_kernel_L4_M8_END): subs counterI, counterI, #1 - bne .Ldtrmm_kernel_L4_M8_20 + bne L(dtrmm_kernel_L4_M8_20) -.Ldtrmm_kernel_L4_M4_BEGIN: +L(dtrmm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Ldtrmm_kernel_L4_END + ble L(dtrmm_kernel_L4_END) tst counterI, #4 - ble .Ldtrmm_kernel_L4_M2_BEGIN + ble L(dtrmm_kernel_L4_M2_BEGIN) -.Ldtrmm_kernel_L4_M4_20: +L(dtrmm_kernel_L4_M4_20): INIT4x4 @@ -1014,9 +1014,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L4_M4_40 + ble L(dtrmm_kernel_L4_M4_40) -.Ldtrmm_kernel_L4_M4_22: +L(dtrmm_kernel_L4_M4_22): KERNEL4x4_SUB KERNEL4x4_SUB @@ -1029,22 +1029,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M4_22 + bgt L(dtrmm_kernel_L4_M4_22) -.Ldtrmm_kernel_L4_M4_40: +L(dtrmm_kernel_L4_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L4_M4_100 + ble L(dtrmm_kernel_L4_M4_100) -.Ldtrmm_kernel_L4_M4_42: +L(dtrmm_kernel_L4_M4_42): KERNEL4x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M4_42 + bgt L(dtrmm_kernel_L4_M4_42) -.Ldtrmm_kernel_L4_M4_100: +L(dtrmm_kernel_L4_M4_100): SAVE4x4 @@ -1063,19 +1063,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Ldtrmm_kernel_L4_M4_END: +L(dtrmm_kernel_L4_M4_END): -.Ldtrmm_kernel_L4_M2_BEGIN: +L(dtrmm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldtrmm_kernel_L4_END + ble L(dtrmm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L4_M1_BEGIN + ble L(dtrmm_kernel_L4_M1_BEGIN) -.Ldtrmm_kernel_L4_M2_20: +L(dtrmm_kernel_L4_M2_20): INIT2x4 @@ -1098,9 +1098,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L4_M2_40 + ble L(dtrmm_kernel_L4_M2_40) -.Ldtrmm_kernel_L4_M2_22: +L(dtrmm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1113,22 +1113,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M2_22 + bgt L(dtrmm_kernel_L4_M2_22) -.Ldtrmm_kernel_L4_M2_40: +L(dtrmm_kernel_L4_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L4_M2_100 + ble L(dtrmm_kernel_L4_M2_100) -.Ldtrmm_kernel_L4_M2_42: +L(dtrmm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M2_42 + bgt L(dtrmm_kernel_L4_M2_42) -.Ldtrmm_kernel_L4_M2_100: +L(dtrmm_kernel_L4_M2_100): SAVE2x4 @@ -1148,15 +1148,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Ldtrmm_kernel_L4_M2_END: +L(dtrmm_kernel_L4_M2_END): -.Ldtrmm_kernel_L4_M1_BEGIN: +L(dtrmm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldtrmm_kernel_L4_END + ble L(dtrmm_kernel_L4_END) -.Ldtrmm_kernel_L4_M1_20: +L(dtrmm_kernel_L4_M1_20): INIT1x4 @@ -1180,9 +1180,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L4_M1_40 + ble L(dtrmm_kernel_L4_M1_40) -.Ldtrmm_kernel_L4_M1_22: +L(dtrmm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1194,22 +1194,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M1_22 + bgt L(dtrmm_kernel_L4_M1_22) -.Ldtrmm_kernel_L4_M1_40: +L(dtrmm_kernel_L4_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L4_M1_100 + ble L(dtrmm_kernel_L4_M1_100) -.Ldtrmm_kernel_L4_M1_42: +L(dtrmm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_M1_42 + bgt L(dtrmm_kernel_L4_M1_42) -.Ldtrmm_kernel_L4_M1_100: +L(dtrmm_kernel_L4_M1_100): SAVE1x4 @@ -1229,7 +1229,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Ldtrmm_kernel_L4_END: +L(dtrmm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 @@ -1239,19 +1239,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Ldtrmm_kernel_L4_BEGIN + bgt L(dtrmm_kernel_L4_BEGIN) /******************************************************************************/ -.Ldtrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(dtrmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Ldtrmm_kernel_L999 // error, N was less than 4? + ble L(dtrmm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Ldtrmm_kernel_L1_BEGIN + ble L(dtrmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1262,14 +1262,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = A -.Ldtrmm_kernel_L2_M8_BEGIN: +L(dtrmm_kernel_L2_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Ldtrmm_kernel_L2_M4_BEGIN + ble L(dtrmm_kernel_L2_M4_BEGIN) -.Ldtrmm_kernel_L2_M8_20: +L(dtrmm_kernel_L2_M8_20): INIT8x2 @@ -1293,10 +1293,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldtrmm_kernel_L2_M8_40 + ble L(dtrmm_kernel_L2_M8_40) .align 5 -.Ldtrmm_kernel_L2_M8_22: +L(dtrmm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB KERNEL8x2_SUB @@ -1308,22 +1308,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M8_22 + bgt L(dtrmm_kernel_L2_M8_22) -.Ldtrmm_kernel_L2_M8_40: +L(dtrmm_kernel_L2_M8_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L2_M8_100 + ble L(dtrmm_kernel_L2_M8_100) -.Ldtrmm_kernel_L2_M8_42: +L(dtrmm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M8_42 + bgt L(dtrmm_kernel_L2_M8_42) -.Ldtrmm_kernel_L2_M8_100: +L(dtrmm_kernel_L2_M8_100): SAVE8x2 @@ -1343,21 +1343,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #8 #endif -.Ldtrmm_kernel_L2_M8_END: +L(dtrmm_kernel_L2_M8_END): subs counterI, counterI, #1 - bgt .Ldtrmm_kernel_L2_M8_20 + bgt L(dtrmm_kernel_L2_M8_20) -.Ldtrmm_kernel_L2_M4_BEGIN: +L(dtrmm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Ldtrmm_kernel_L2_END + ble L(dtrmm_kernel_L2_END) tst counterI, #4 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L2_M2_BEGIN + ble L(dtrmm_kernel_L2_M2_BEGIN) -.Ldtrmm_kernel_L2_M4_20: +L(dtrmm_kernel_L2_M4_20): INIT4x2 @@ -1381,10 +1381,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldtrmm_kernel_L2_M4_40 + ble L(dtrmm_kernel_L2_M4_40) .align 5 -.Ldtrmm_kernel_L2_M4_22: +L(dtrmm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1396,22 +1396,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M4_22 + bgt L(dtrmm_kernel_L2_M4_22) -.Ldtrmm_kernel_L2_M4_40: +L(dtrmm_kernel_L2_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L2_M4_100 + ble L(dtrmm_kernel_L2_M4_100) -.Ldtrmm_kernel_L2_M4_42: +L(dtrmm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M4_42 + bgt L(dtrmm_kernel_L2_M4_42) -.Ldtrmm_kernel_L2_M4_100: +L(dtrmm_kernel_L2_M4_100): SAVE4x2 @@ -1431,19 +1431,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Ldtrmm_kernel_L2_M4_END: +L(dtrmm_kernel_L2_M4_END): -.Ldtrmm_kernel_L2_M2_BEGIN: +L(dtrmm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldtrmm_kernel_L2_END + ble L(dtrmm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L2_M1_BEGIN + ble L(dtrmm_kernel_L2_M1_BEGIN) -.Ldtrmm_kernel_L2_M2_20: +L(dtrmm_kernel_L2_M2_20): INIT2x2 @@ -1467,9 +1467,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Ldtrmm_kernel_L2_M2_40 + ble L(dtrmm_kernel_L2_M2_40) -.Ldtrmm_kernel_L2_M2_22: +L(dtrmm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1482,22 +1482,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M2_22 + bgt L(dtrmm_kernel_L2_M2_22) -.Ldtrmm_kernel_L2_M2_40: +L(dtrmm_kernel_L2_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L2_M2_100 + ble L(dtrmm_kernel_L2_M2_100) -.Ldtrmm_kernel_L2_M2_42: +L(dtrmm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M2_42 + bgt L(dtrmm_kernel_L2_M2_42) -.Ldtrmm_kernel_L2_M2_100: +L(dtrmm_kernel_L2_M2_100): SAVE2x2 @@ -1517,15 +1517,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Ldtrmm_kernel_L2_M2_END: +L(dtrmm_kernel_L2_M2_END): -.Ldtrmm_kernel_L2_M1_BEGIN: +L(dtrmm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldtrmm_kernel_L2_END + ble L(dtrmm_kernel_L2_END) -.Ldtrmm_kernel_L2_M1_20: +L(dtrmm_kernel_L2_M1_20): INIT1x2 @@ -1549,9 +1549,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Ldtrmm_kernel_L2_M1_40 + ble L(dtrmm_kernel_L2_M1_40) -.Ldtrmm_kernel_L2_M1_22: +L(dtrmm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1563,22 +1563,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M1_22 + bgt L(dtrmm_kernel_L2_M1_22) -.Ldtrmm_kernel_L2_M1_40: +L(dtrmm_kernel_L2_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L2_M1_100 + ble L(dtrmm_kernel_L2_M1_100) -.Ldtrmm_kernel_L2_M1_42: +L(dtrmm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_M1_42 + bgt L(dtrmm_kernel_L2_M1_42) -.Ldtrmm_kernel_L2_M1_100: +L(dtrmm_kernel_L2_M1_100): SAVE1x2 @@ -1598,7 +1598,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Ldtrmm_kernel_L2_END: +L(dtrmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -1606,11 +1606,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Ldtrmm_kernel_L1_BEGIN: +L(dtrmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldtrmm_kernel_L999 // done + ble L(dtrmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C add pC , pC , LDC // Update pC to point to next @@ -1620,14 +1620,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = A -.Ldtrmm_kernel_L1_M8_BEGIN: +L(dtrmm_kernel_L1_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Ldtrmm_kernel_L1_M4_BEGIN + ble L(dtrmm_kernel_L1_M4_BEGIN) -.Ldtrmm_kernel_L1_M8_20: +L(dtrmm_kernel_L1_M8_20): INIT8x1 @@ -1651,10 +1651,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L1_M8_40 + ble L(dtrmm_kernel_L1_M8_40) .align 5 -.Ldtrmm_kernel_L1_M8_22: +L(dtrmm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -1666,22 +1666,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M8_22 + bgt L(dtrmm_kernel_L1_M8_22) -.Ldtrmm_kernel_L1_M8_40: +L(dtrmm_kernel_L1_M8_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L1_M8_100 + ble L(dtrmm_kernel_L1_M8_100) -.Ldtrmm_kernel_L1_M8_42: +L(dtrmm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M8_42 + bgt L(dtrmm_kernel_L1_M8_42) -.Ldtrmm_kernel_L1_M8_100: +L(dtrmm_kernel_L1_M8_100): SAVE8x1 @@ -1701,21 +1701,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #8 #endif -.Ldtrmm_kernel_L1_M8_END: +L(dtrmm_kernel_L1_M8_END): subs counterI, counterI, #1 - bgt .Ldtrmm_kernel_L1_M8_20 + bgt L(dtrmm_kernel_L1_M8_20) -.Ldtrmm_kernel_L1_M4_BEGIN: +L(dtrmm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Ldtrmm_kernel_L1_END + ble L(dtrmm_kernel_L1_END) tst counterI, #4 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L1_M2_BEGIN + ble L(dtrmm_kernel_L1_M2_BEGIN) -.Ldtrmm_kernel_L1_M4_20: +L(dtrmm_kernel_L1_M4_20): INIT4x1 @@ -1738,10 +1738,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L1_M4_40 + ble L(dtrmm_kernel_L1_M4_40) .align 5 -.Ldtrmm_kernel_L1_M4_22: +L(dtrmm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1753,22 +1753,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M4_22 + bgt L(dtrmm_kernel_L1_M4_22) -.Ldtrmm_kernel_L1_M4_40: +L(dtrmm_kernel_L1_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L1_M4_100 + ble L(dtrmm_kernel_L1_M4_100) -.Ldtrmm_kernel_L1_M4_42: +L(dtrmm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M4_42 + bgt L(dtrmm_kernel_L1_M4_42) -.Ldtrmm_kernel_L1_M4_100: +L(dtrmm_kernel_L1_M4_100): SAVE4x1 @@ -1788,18 +1788,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Ldtrmm_kernel_L1_M4_END: +L(dtrmm_kernel_L1_M4_END): -.Ldtrmm_kernel_L1_M2_BEGIN: +L(dtrmm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Ldtrmm_kernel_L1_END + ble L(dtrmm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Ldtrmm_kernel_L1_M1_BEGIN + ble L(dtrmm_kernel_L1_M1_BEGIN) -.Ldtrmm_kernel_L1_M2_20: +L(dtrmm_kernel_L1_M2_20): INIT2x1 @@ -1823,9 +1823,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L1_M2_40 + ble L(dtrmm_kernel_L1_M2_40) -.Ldtrmm_kernel_L1_M2_22: +L(dtrmm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1838,22 +1838,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M2_22 + bgt L(dtrmm_kernel_L1_M2_22) -.Ldtrmm_kernel_L1_M2_40: +L(dtrmm_kernel_L1_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L1_M2_100 + ble L(dtrmm_kernel_L1_M2_100) -.Ldtrmm_kernel_L1_M2_42: +L(dtrmm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M2_42 + bgt L(dtrmm_kernel_L1_M2_42) -.Ldtrmm_kernel_L1_M2_100: +L(dtrmm_kernel_L1_M2_100): SAVE2x1 @@ -1873,15 +1873,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Ldtrmm_kernel_L1_M2_END: +L(dtrmm_kernel_L1_M2_END): -.Ldtrmm_kernel_L1_M1_BEGIN: +L(dtrmm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Ldtrmm_kernel_L1_END + ble L(dtrmm_kernel_L1_END) -.Ldtrmm_kernel_L1_M1_20: +L(dtrmm_kernel_L1_M1_20): INIT1x1 @@ -1905,9 +1905,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Ldtrmm_kernel_L1_M1_40 + ble L(dtrmm_kernel_L1_M1_40) -.Ldtrmm_kernel_L1_M1_22: +L(dtrmm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1919,30 +1919,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M1_22 + bgt L(dtrmm_kernel_L1_M1_22) -.Ldtrmm_kernel_L1_M1_40: +L(dtrmm_kernel_L1_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Ldtrmm_kernel_L1_M1_100 + ble L(dtrmm_kernel_L1_M1_100) -.Ldtrmm_kernel_L1_M1_42: +L(dtrmm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_M1_42 + bgt L(dtrmm_kernel_L1_M1_42) -.Ldtrmm_kernel_L1_M1_100: +L(dtrmm_kernel_L1_M1_100): SAVE1x1 -.Ldtrmm_kernel_L1_END: +L(dtrmm_kernel_L1_END): -.Ldtrmm_kernel_L999: +L(dtrmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/dtrmm_kernel_sve_v1x8.S b/kernel/arm64/dtrmm_kernel_sve_v1x8.S index 1f8c9b20fa..644d227138 100644 --- a/kernel/arm64/dtrmm_kernel_sve_v1x8.S +++ b/kernel/arm64/dtrmm_kernel_sve_v1x8.S @@ -472,13 +472,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Ldtrmm_kernel_L4_BEGIN + ble L(dtrmm_kernel_L4_BEGIN) /******************************************************************************/ /* Repeat this as long as there are 8 left in N */ .align 5 -.Ldtrmm_kernel_L8_BEGIN: +L(dtrmm_kernel_L8_BEGIN): mov pCRow0, pC add pC, pC, LDC, lsl #3 // add 8 x LDC @@ -489,7 +489,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldtrmm_kernel_L8_Mv1_BEGIN: +L(dtrmm_kernel_L8_Mv1_BEGIN): /* Loop over M is done in an SVE fashion. This has the benefit of the last M%SVE_LEN iterations being done in a single sweep */ mov counterI, #0 @@ -497,7 +497,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension .align 5 -.Ldtrmm_kernel_L8_Mv1_20: +L(dtrmm_kernel_L8_Mv1_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -521,7 +521,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Ldtrmm_kernel_L8_Mv1_32 + blt L(dtrmm_kernel_L8_Mv1_32) KERNELv1x8_I KERNELv1x8_M2 @@ -533,10 +533,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Ldtrmm_kernel_L8_Mv1_22a + ble L(dtrmm_kernel_L8_Mv1_22a) .align 5 -.Ldtrmm_kernel_L8_Mv1_22: +L(dtrmm_kernel_L8_Mv1_22): KERNELv1x8_M1 KERNELv1x8_M2 @@ -548,10 +548,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M2 subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L8_Mv1_22 + bgt L(dtrmm_kernel_L8_Mv1_22) .align 5 -.Ldtrmm_kernel_L8_Mv1_22a: +L(dtrmm_kernel_L8_Mv1_22a): KERNELv1x8_M1 KERNELv1x8_M2 @@ -562,13 +562,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M1 KERNELv1x8_E - b .Ldtrmm_kernel_L8_Mv1_44 + b L(dtrmm_kernel_L8_Mv1_44) .align 5 -.Ldtrmm_kernel_L8_Mv1_32: +L(dtrmm_kernel_L8_Mv1_32): tst counterL, #1 - ble .Ldtrmm_kernel_L8_Mv1_40 + ble L(dtrmm_kernel_L8_Mv1_40) KERNELv1x8_I KERNELv1x8_M2 @@ -580,26 +580,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_E - b .Ldtrmm_kernel_L8_Mv1_44 + b L(dtrmm_kernel_L8_Mv1_44) -.Ldtrmm_kernel_L8_Mv1_40: +L(dtrmm_kernel_L8_Mv1_40): INITv1x8 -.Ldtrmm_kernel_L8_Mv1_44: +L(dtrmm_kernel_L8_Mv1_44): ands counterL , tempK, #7 - ble .Ldtrmm_kernel_L8_Mv1_100 + ble L(dtrmm_kernel_L8_Mv1_100) .align 5 -.Ldtrmm_kernel_L8_Mv1_46: +L(dtrmm_kernel_L8_Mv1_46): KERNELv1x8_SUB subs counterL, counterL, #1 - bne .Ldtrmm_kernel_L8_Mv1_46 + bne L(dtrmm_kernel_L8_Mv1_46) -.Ldtrmm_kernel_L8_Mv1_100: +L(dtrmm_kernel_L8_Mv1_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] @@ -622,14 +622,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, lanes #endif -.Ldtrmm_kernel_L8_Mv1_END: +L(dtrmm_kernel_L8_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldtrmm_kernel_L8_Mv1_20 + b.any L(dtrmm_kernel_L8_Mv1_20) -.Ldtrmm_kernel_L8_END: +L(dtrmm_kernel_L8_END): lsl temp, origK, #6 add origPB, origPB, temp // B = B + K * 8 * 8 @@ -639,17 +639,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Ldtrmm_kernel_L8_BEGIN + bgt L(dtrmm_kernel_L8_BEGIN) /******************************************************************************/ /* Repeat the same thing if 4 left in N */ .align 5 -.Ldtrmm_kernel_L4_BEGIN: +L(dtrmm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #4 - ble .Ldtrmm_kernel_L2_BEGIN + ble L(dtrmm_kernel_L2_BEGIN) #if defined(LEFT) mov tempOffset, offset @@ -661,14 +661,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldtrmm_kernel_L4_Mv1_BEGIN: +L(dtrmm_kernel_L4_Mv1_BEGIN): mov counterI, #0 whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d .align 5 -.Ldtrmm_kernel_L4_Mv1_20: +L(dtrmm_kernel_L4_Mv1_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -692,10 +692,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldtrmm_kernel_L4_Mv1_44 + ble L(dtrmm_kernel_L4_Mv1_44) .align 5 -.Ldtrmm_kernel_L4_Mv1_22: +L(dtrmm_kernel_L4_Mv1_22): KERNELv1x4_SUB KERNELv1x4_SUB @@ -707,22 +707,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L4_Mv1_22 + bgt L(dtrmm_kernel_L4_Mv1_22) -.Ldtrmm_kernel_L4_Mv1_44: +L(dtrmm_kernel_L4_Mv1_44): ands counterL , tempK, #7 - ble .Ldtrmm_kernel_L4_Mv1_100 + ble L(dtrmm_kernel_L4_Mv1_100) .align 5 -.Ldtrmm_kernel_L4_Mv1_46: +L(dtrmm_kernel_L4_Mv1_46): KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Ldtrmm_kernel_L4_Mv1_46 + bne L(dtrmm_kernel_L4_Mv1_46) -.Ldtrmm_kernel_L4_Mv1_100: +L(dtrmm_kernel_L4_Mv1_100): SAVEv1x4 @@ -742,15 +742,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, lanes #endif -.Ldtrmm_kernel_L4_Mv1_END: +L(dtrmm_kernel_L4_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldtrmm_kernel_L4_Mv1_20 + b.any L(dtrmm_kernel_L4_Mv1_20) -.Ldtrmm_kernel_L4_END: +L(dtrmm_kernel_L4_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 4 * 8 #if !defined(LEFT) @@ -761,11 +761,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /* Repeat the same thing if 2 left in N */ .align 5 -.Ldtrmm_kernel_L2_BEGIN: +L(dtrmm_kernel_L2_BEGIN): mov counterJ , origN tst counterJ , #2 - ble .Ldtrmm_kernel_L1_BEGIN + ble L(dtrmm_kernel_L1_BEGIN) mov pCRow0, pC @@ -777,14 +777,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldtrmm_kernel_L2_Mv1_BEGIN: +L(dtrmm_kernel_L2_Mv1_BEGIN): mov counterI, #0 whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d .align 5 -.Ldtrmm_kernel_L2_Mv1_20: +L(dtrmm_kernel_L2_Mv1_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -808,10 +808,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldtrmm_kernel_L2_Mv1_44 + ble L(dtrmm_kernel_L2_Mv1_44) .align 5 -.Ldtrmm_kernel_L2_Mv1_22: +L(dtrmm_kernel_L2_Mv1_22): KERNELv1x2_SUB KERNELv1x2_SUB @@ -823,22 +823,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L2_Mv1_22 + bgt L(dtrmm_kernel_L2_Mv1_22) -.Ldtrmm_kernel_L2_Mv1_44: +L(dtrmm_kernel_L2_Mv1_44): ands counterL , tempK, #7 - ble .Ldtrmm_kernel_L2_Mv1_100 + ble L(dtrmm_kernel_L2_Mv1_100) .align 5 -.Ldtrmm_kernel_L2_Mv1_46: +L(dtrmm_kernel_L2_Mv1_46): KERNELv1x2_SUB subs counterL, counterL, #1 - bne .Ldtrmm_kernel_L2_Mv1_46 + bne L(dtrmm_kernel_L2_Mv1_46) -.Ldtrmm_kernel_L2_Mv1_100: +L(dtrmm_kernel_L2_Mv1_100): SAVEv1x2 @@ -859,15 +859,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.Ldtrmm_kernel_L2_Mv1_END: +L(dtrmm_kernel_L2_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldtrmm_kernel_L2_Mv1_20 + b.any L(dtrmm_kernel_L2_Mv1_20) -.Ldtrmm_kernel_L2_END: +L(dtrmm_kernel_L2_END): add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 #if !defined(LEFT) add tempOffset, tempOffset, #2 @@ -877,11 +877,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /* Repeat the same thing if 1 left in N */ .align 5 -.Ldtrmm_kernel_L1_BEGIN: +L(dtrmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldtrmm_kernel_L999 // done + ble L(dtrmm_kernel_L999) // done mov pCRow0, pC @@ -893,14 +893,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldtrmm_kernel_L1_Mv1_BEGIN: +L(dtrmm_kernel_L1_Mv1_BEGIN): mov counterI, #0 whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d .align 5 -.Ldtrmm_kernel_L1_Mv1_20: +L(dtrmm_kernel_L1_Mv1_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -924,10 +924,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Ldtrmm_kernel_L1_Mv1_44 + ble L(dtrmm_kernel_L1_Mv1_44) .align 5 -.Ldtrmm_kernel_L1_Mv1_22: +L(dtrmm_kernel_L1_Mv1_22): KERNELv1x1_SUB KERNELv1x1_SUB @@ -939,22 +939,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_Mv1_22 + bgt L(dtrmm_kernel_L1_Mv1_22) -.Ldtrmm_kernel_L1_Mv1_44: +L(dtrmm_kernel_L1_Mv1_44): ands counterL , tempK, #7 - ble .Ldtrmm_kernel_L1_Mv1_100 + ble L(dtrmm_kernel_L1_Mv1_100) .align 5 -.Ldtrmm_kernel_L1_Mv1_46: +L(dtrmm_kernel_L1_Mv1_46): KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Ldtrmm_kernel_L1_Mv1_46 + bgt L(dtrmm_kernel_L1_Mv1_46) -.Ldtrmm_kernel_L1_Mv1_100: +L(dtrmm_kernel_L1_Mv1_100): SAVEv1x1 @@ -976,19 +976,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Ldtrmm_kernel_L1_Mv1_END: +L(dtrmm_kernel_L1_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Ldtrmm_kernel_L1_Mv1_20 + b.any L(dtrmm_kernel_L1_Mv1_20) -.Ldtrmm_kernel_L1_END: +L(dtrmm_kernel_L1_END): /******************************************************************************/ -.Ldtrmm_kernel_L999: +L(dtrmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/gemv_n.S b/kernel/arm64/gemv_n.S index 658551f4f0..ef791e22e7 100644 --- a/kernel/arm64/gemv_n.S +++ b/kernel/arm64/gemv_n.S @@ -203,18 +203,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SAVE_REGS cmp N, xzr - ble .Lgemv_n_kernel_L999 + ble L(gemv_n_kernel_L999) cmp M, xzr - ble .Lgemv_n_kernel_L999 + ble L(gemv_n_kernel_L999) lsl LDA, LDA, #SHZ lsl INC_X, INC_X, #SHZ mov J, N cmp INC_Y, #1 - bne .Lgemv_n_kernel_S_BEGIN + bne L(gemv_n_kernel_S_BEGIN) -.Lgemv_n_kernel_F_LOOP: +L(gemv_n_kernel_F_LOOP): ld1 TEMPV, [X], INC_X fmul TEMP, ALPHA, TEMP @@ -229,57 +229,57 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov Y_IPTR, Y mov Y_OPTR, Y -.Lgemv_n_kernel_F32: +L(gemv_n_kernel_F32): asr I, M, #5 cmp I, xzr - beq .Lgemv_n_kernel_F4 + beq L(gemv_n_kernel_F4) -.Lgemv_n_kernel_F320: +L(gemv_n_kernel_F320): KERNEL_F16 KERNEL_F16 subs I, I, #1 - bne .Lgemv_n_kernel_F320 + bne L(gemv_n_kernel_F320) -.Lgemv_n_kernel_F4: +L(gemv_n_kernel_F4): ands I, M, #31 asr I, I, #2 cmp I, xzr - beq .Lgemv_n_kernel_F1 + beq L(gemv_n_kernel_F1) -.Lgemv_n_kernel_F40: +L(gemv_n_kernel_F40): KERNEL_F4 subs I, I, #1 - bne .Lgemv_n_kernel_F40 + bne L(gemv_n_kernel_F40) -.Lgemv_n_kernel_F1: +L(gemv_n_kernel_F1): ands I, M, #3 - ble .Lgemv_n_kernel_F_END + ble L(gemv_n_kernel_F_END) -.Lgemv_n_kernel_F10: +L(gemv_n_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lgemv_n_kernel_F10 + bne L(gemv_n_kernel_F10) -.Lgemv_n_kernel_F_END: +L(gemv_n_kernel_F_END): add A, A, LDA subs J, J, #1 - bne .Lgemv_n_kernel_F_LOOP + bne L(gemv_n_kernel_F_LOOP) - b .Lgemv_n_kernel_L999 + b L(gemv_n_kernel_L999) -.Lgemv_n_kernel_S_BEGIN: +L(gemv_n_kernel_S_BEGIN): INIT_S -.Lgemv_n_kernel_S_LOOP: +L(gemv_n_kernel_S_LOOP): ld1 TEMPV, [X], INC_X fmul TEMP, ALPHA, TEMP @@ -288,9 +288,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #2 cmp I, xzr - ble .Lgemv_n_kernel_S1 + ble L(gemv_n_kernel_S1) -.Lgemv_n_kernel_S4: +L(gemv_n_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -298,27 +298,27 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lgemv_n_kernel_S4 + bne L(gemv_n_kernel_S4) -.Lgemv_n_kernel_S1: +L(gemv_n_kernel_S1): ands I, M, #3 - ble .Lgemv_n_kernel_S_END + ble L(gemv_n_kernel_S_END) -.Lgemv_n_kernel_S10: +L(gemv_n_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lgemv_n_kernel_S10 + bne L(gemv_n_kernel_S10) -.Lgemv_n_kernel_S_END: +L(gemv_n_kernel_S_END): add A, A, LDA subs J, J, #1 - bne .Lgemv_n_kernel_S_LOOP + bne L(gemv_n_kernel_S_LOOP) -.Lgemv_n_kernel_L999: +L(gemv_n_kernel_L999): mov w0, wzr diff --git a/kernel/arm64/gemv_t.S b/kernel/arm64/gemv_t.S index a98eef49b0..18559a1f93 100644 --- a/kernel/arm64/gemv_t.S +++ b/kernel/arm64/gemv_t.S @@ -242,18 +242,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SAVE_REGS cmp N, xzr - ble .Lgemv_t_kernel_L999 + ble L(gemv_t_kernel_L999) cmp M, xzr - ble .Lgemv_t_kernel_L999 + ble L(gemv_t_kernel_L999) lsl LDA, LDA, #SHZ lsl INC_Y, INC_Y, #SHZ mov J, N cmp INC_X, #1 - bne .Lgemv_t_kernel_S_BEGIN + bne L(gemv_t_kernel_S_BEGIN) -.Lgemv_t_kernel_F_LOOP: +L(gemv_t_kernel_F_LOOP): fmov TEMP, REG0 fmov TEMP1, REG0 @@ -263,64 +263,64 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov A_PTR, A mov X_PTR, X -.Lgemv_t_kernel_F32: +L(gemv_t_kernel_F32): asr I, M, #5 cmp I, xzr - beq .Lgemv_t_kernel_F8 + beq L(gemv_t_kernel_F8) -.Lgemv_t_kernel_F320: +L(gemv_t_kernel_F320): KERNEL_F32 subs I, I, #1 - bne .Lgemv_t_kernel_F320 + bne L(gemv_t_kernel_F320) KERNEL_F32_FINALIZE -.Lgemv_t_kernel_F8: +L(gemv_t_kernel_F8): ands I, M, #31 asr I, I, #3 cmp I, xzr - beq .Lgemv_t_kernel_F1 + beq L(gemv_t_kernel_F1) -.Lgemv_t_kernel_F80: +L(gemv_t_kernel_F80): KERNEL_F8 subs I, I, #1 - bne .Lgemv_t_kernel_F80 + bne L(gemv_t_kernel_F80) -.Lgemv_t_kernel_F1: +L(gemv_t_kernel_F1): KERNEL_F8_FINALIZE ands I, M, #7 - ble .Lgemv_t_kernel_F_END + ble L(gemv_t_kernel_F_END) -.Lgemv_t_kernel_F10: +L(gemv_t_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lgemv_t_kernel_F10 + bne L(gemv_t_kernel_F10) -.Lgemv_t_kernel_F_END: +L(gemv_t_kernel_F_END): ld1 TMPV1, [Y] add A, A, LDA subs J, J, #1 fmadd TMP1, ALPHA, TEMP, TMP1 st1 TMPV1, [Y], INC_Y - bne .Lgemv_t_kernel_F_LOOP + bne L(gemv_t_kernel_F_LOOP) - b .Lgemv_t_kernel_L999 + b L(gemv_t_kernel_L999) -.Lgemv_t_kernel_S_BEGIN: +L(gemv_t_kernel_S_BEGIN): INIT_S -.Lgemv_t_kernel_S_LOOP: +L(gemv_t_kernel_S_LOOP): fmov TEMP, REG0 mov A_PTR, A @@ -328,9 +328,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #2 cmp I, xzr - ble .Lgemv_t_kernel_S1 + ble L(gemv_t_kernel_S1) -.Lgemv_t_kernel_S4: +L(gemv_t_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -338,30 +338,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lgemv_t_kernel_S4 + bne L(gemv_t_kernel_S4) -.Lgemv_t_kernel_S1: +L(gemv_t_kernel_S1): ands I, M, #3 - ble .Lgemv_t_kernel_S_END + ble L(gemv_t_kernel_S_END) -.Lgemv_t_kernel_S10: +L(gemv_t_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lgemv_t_kernel_S10 + bne L(gemv_t_kernel_S10) -.Lgemv_t_kernel_S_END: +L(gemv_t_kernel_S_END): ld1 TMPV1, [Y] add A, A, LDA subs J, J, #1 fmadd TMP1, ALPHA, TEMP, TMP1 st1 TMPV1, [Y], INC_Y - bne .Lgemv_t_kernel_S_LOOP + bne L(gemv_t_kernel_S_LOOP) -.Lgemv_t_kernel_L999: +L(gemv_t_kernel_L999): RESTORE_REGS diff --git a/kernel/arm64/iamax.S b/kernel/arm64/iamax.S index 31d0cd6466..b089505b12 100644 --- a/kernel/arm64/iamax.S +++ b/kernel/arm64/iamax.S @@ -230,62 +230,62 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Liamax_kernel_zero + ble L(iamax_kernel_zero) cmp INC_X, xzr - ble .Liamax_kernel_zero + ble L(iamax_kernel_zero) cmp INC_X, #1 - bne .Liamax_kernel_S_BEGIN + bne L(iamax_kernel_S_BEGIN) mov x7, X -.Liamax_kernel_F_BEGIN: +L(iamax_kernel_F_BEGIN): INIT_S subs N, N, #1 - ble .Liamax_kernel_L999 + ble L(iamax_kernel_L999) asr I, N, #3 cmp I, xzr - beq .Liamax_kernel_F1 + beq L(iamax_kernel_F1) add Z, Z, #1 -.Liamax_kernel_F8: +L(iamax_kernel_F8): KERNEL_F8 subs I, I, #1 - bne .Liamax_kernel_F8 + bne L(iamax_kernel_F8) KERNEL_F8_FINALIZE sub Z, Z, #1 -.Liamax_kernel_F1: +L(iamax_kernel_F1): ands I, N, #7 - ble .Liamax_kernel_L999 + ble L(iamax_kernel_L999) -.Liamax_kernel_F10: +L(iamax_kernel_F10): KERNEL_S1 subs I, I, #1 - bne .Liamax_kernel_F10 + bne L(iamax_kernel_F10) - b .Liamax_kernel_L999 + b L(iamax_kernel_L999) -.Liamax_kernel_S_BEGIN: +L(iamax_kernel_S_BEGIN): INIT_S subs N, N, #1 - ble .Liamax_kernel_L999 + ble L(iamax_kernel_L999) asr I, N, #2 cmp I, xzr - ble .Liamax_kernel_S1 + ble L(iamax_kernel_S1) -.Liamax_kernel_S4: +L(iamax_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -293,25 +293,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Liamax_kernel_S4 + bne L(iamax_kernel_S4) -.Liamax_kernel_S1: +L(iamax_kernel_S1): ands I, N, #3 - ble .Liamax_kernel_L999 + ble L(iamax_kernel_L999) -.Liamax_kernel_S10: +L(iamax_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Liamax_kernel_S10 + bne L(iamax_kernel_S10) -.Liamax_kernel_L999: +L(iamax_kernel_L999): mov x0, INDEX ret -.Liamax_kernel_zero: +L(iamax_kernel_zero): mov x0, xzr ret diff --git a/kernel/arm64/izamax.S b/kernel/arm64/izamax.S index 42fa4e7111..904d6fc701 100644 --- a/kernel/arm64/izamax.S +++ b/kernel/arm64/izamax.S @@ -276,64 +276,64 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Lizamax_kernel_zero + ble L(izamax_kernel_zero) cmp INC_X, xzr - ble .Lizamax_kernel_zero + ble L(izamax_kernel_zero) cmp INC_X, #1 - bne .Lizamax_kernel_S_BEGIN + bne L(izamax_kernel_S_BEGIN) mov x7, X -.Lizamax_kernel_F_BEGIN: +L(izamax_kernel_F_BEGIN): INIT_S subs N, N, #1 - ble .Lizamax_kernel_L999 + ble L(izamax_kernel_L999) asr I, N, #3 cmp I, xzr - ble .Lizamax_kernel_F1 + ble L(izamax_kernel_F1) add Z, Z, #1 -.Lizamax_kernel_F8: +L(izamax_kernel_F8): KERNEL_F8 subs I, I, #1 - bne .Lizamax_kernel_F8 + bne L(izamax_kernel_F8) KERNEL_F8_FINALIZE sub Z, Z, #1 -.Lizamax_kernel_F1: +L(izamax_kernel_F1): ands I, N, #7 - ble .Lizamax_kernel_L999 + ble L(izamax_kernel_L999) -.Lizamax_kernel_F10: +L(izamax_kernel_F10): KERNEL_S1 subs I, I, #1 - bne .Lizamax_kernel_F10 + bne L(izamax_kernel_F10) - b .Lizamax_kernel_L999 + b L(izamax_kernel_L999) -.Lizamax_kernel_S_BEGIN: +L(izamax_kernel_S_BEGIN): INIT_S subs N, N, #1 - ble .Lizamax_kernel_L999 + ble L(izamax_kernel_L999) asr I, N, #2 cmp I, xzr - ble .Lizamax_kernel_S1 + ble L(izamax_kernel_S1) -.Lizamax_kernel_S4: +L(izamax_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -341,26 +341,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lizamax_kernel_S4 + bne L(izamax_kernel_S4) -.Lizamax_kernel_S1: +L(izamax_kernel_S1): ands I, N, #3 - ble .Lizamax_kernel_L999 + ble L(izamax_kernel_L999) -.Lizamax_kernel_S10: +L(izamax_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lizamax_kernel_S10 + bne L(izamax_kernel_S10) -.Lizamax_kernel_L999: +L(izamax_kernel_L999): mov x0, INDEX ret -.Lizamax_kernel_zero: +L(izamax_kernel_zero): mov x0, xzr ret diff --git a/kernel/arm64/nrm2.S b/kernel/arm64/nrm2.S index 93218ff6e0..a5e9404fff 100644 --- a/kernel/arm64/nrm2.S +++ b/kernel/arm64/nrm2.S @@ -133,44 +133,44 @@ KERNEL_S1_NEXT: INIT cmp N, #0 - ble .Lnrm2_kernel_L999 + ble L(nrm2_kernel_L999) cmp INC_X, #0 - beq .Lnrm2_kernel_L999 + beq L(nrm2_kernel_L999) cmp INC_X, #1 - bne .Lnrm2_kernel_S_BEGIN + bne L(nrm2_kernel_S_BEGIN) -.Lnrm2_kernel_F_BEGIN: +L(nrm2_kernel_F_BEGIN): asr I, N, #3 // I = N / 8 cmp I, xzr - ble .Lnrm2_kernel_F1 + ble L(nrm2_kernel_F1) -.Lnrm2_kernel_F8: +L(nrm2_kernel_F8): KERNEL_F8 subs I, I, #1 - bne .Lnrm2_kernel_F8 + bne L(nrm2_kernel_F8) -.Lnrm2_kernel_F1: +L(nrm2_kernel_F1): ands I, N, #7 - ble .Lnrm2_kernel_L999 + ble L(nrm2_kernel_L999) -.Lnrm2_kernel_F10: +L(nrm2_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lnrm2_kernel_F10 + bne L(nrm2_kernel_F10) - b .Lnrm2_kernel_L999 + b L(nrm2_kernel_L999) -.Lnrm2_kernel_S_BEGIN: +L(nrm2_kernel_S_BEGIN): INIT_S @@ -178,15 +178,15 @@ KERNEL_S1_NEXT: .align 5 -.Lnrm2_kernel_S10: +L(nrm2_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lnrm2_kernel_S10 + bne L(nrm2_kernel_S10) -.Lnrm2_kernel_L999: +L(nrm2_kernel_L999): fsqrt SSQ, SSQ fmul SSQ, SCALE, SSQ #if !defined(DOUBLE) diff --git a/kernel/arm64/rot.S b/kernel/arm64/rot.S index 00c3085fa3..33b404db4e 100644 --- a/kernel/arm64/rot.S +++ b/kernel/arm64/rot.S @@ -165,48 +165,48 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Lrot_kernel_L999 + ble L(rot_kernel_L999) INIT cmp INC_X, #1 - bne .Lrot_kernel_S_BEGIN + bne L(rot_kernel_S_BEGIN) cmp INC_Y, #1 - bne .Lrot_kernel_S_BEGIN + bne L(rot_kernel_S_BEGIN) -.Lrot_kernel_F_BEGIN: +L(rot_kernel_F_BEGIN): asr I, N, #2 cmp I, xzr - beq .Lrot_kernel_F1 + beq L(rot_kernel_F1) KERNEL_INIT_F4 -.Lrot_kernel_F4: +L(rot_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lrot_kernel_F4 + bne L(rot_kernel_F4) -.Lrot_kernel_F1: +L(rot_kernel_F1): ands I, N, #3 - ble .Lrot_kernel_L999 + ble L(rot_kernel_L999) INIT_F1 -.Lrot_kernel_F10: +L(rot_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lrot_kernel_F10 + bne L(rot_kernel_F10) mov w0, wzr ret -.Lrot_kernel_S_BEGIN: +L(rot_kernel_S_BEGIN): INIT_S INIT_F1 @@ -214,9 +214,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, N, #2 cmp I, xzr - ble .Lrot_kernel_S1 + ble L(rot_kernel_S1) -.Lrot_kernel_S4: +L(rot_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -224,22 +224,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lrot_kernel_S4 + bne L(rot_kernel_S4) -.Lrot_kernel_S1: +L(rot_kernel_S1): ands I, N, #3 - ble .Lrot_kernel_L999 + ble L(rot_kernel_L999) -.Lrot_kernel_S10: +L(rot_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lrot_kernel_S10 + bne L(rot_kernel_S10) -.Lrot_kernel_L999: +L(rot_kernel_L999): mov w0, wzr ret diff --git a/kernel/arm64/scal.S b/kernel/arm64/scal.S index 33400b6309..757667e5bb 100644 --- a/kernel/arm64/scal.S +++ b/kernel/arm64/scal.S @@ -166,91 +166,91 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Lscal_kernel_L999 + ble L(scal_kernel_L999) ldr FLAG, [sp] cmp FLAG, #1 - beq .Lscal_kernel_nansafe + beq L(scal_kernel_nansafe) fcmp DA, #0.0 - beq .Lscal_kernel_zero + beq L(scal_kernel_zero) -.Lscal_kernel_nansafe: +L(scal_kernel_nansafe): cmp INC_X, #1 - bne .Lscal_kernel_S_BEGIN + bne L(scal_kernel_S_BEGIN) -.Lscal_kernel_F_BEGIN: +L(scal_kernel_F_BEGIN): asr I, N, #3 cmp I, xzr - beq .Lscal_kernel_F1 + beq L(scal_kernel_F1) KERNEL_INIT_F8 -.Lscal_kernel_F8: +L(scal_kernel_F8): KERNEL_F8 subs I, I, #1 - bne .Lscal_kernel_F8 + bne L(scal_kernel_F8) -.Lscal_kernel_F1: +L(scal_kernel_F1): ands I, N, #7 - ble .Lscal_kernel_L999 + ble L(scal_kernel_L999) -.Lscal_kernel_F10: +L(scal_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lscal_kernel_F10 + bne L(scal_kernel_F10) mov w0, wzr ret -.Lscal_kernel_S_BEGIN: +L(scal_kernel_S_BEGIN): INIT_S mov X_COPY, X asr I, N, #2 cmp I, xzr - ble .Lscal_kernel_S1 + ble L(scal_kernel_S1) -.Lscal_kernel_S4: +L(scal_kernel_S4): KERNEL_S4 subs I, I, #1 - bne .Lscal_kernel_S4 + bne L(scal_kernel_S4) -.Lscal_kernel_S1: +L(scal_kernel_S1): ands I, N, #3 - ble .Lscal_kernel_L999 + ble L(scal_kernel_L999) -.Lscal_kernel_S10: +L(scal_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lscal_kernel_S10 + bne L(scal_kernel_S10) -.Lscal_kernel_L999: +L(scal_kernel_L999): mov w0, wzr ret -.Lscal_kernel_zero: +L(scal_kernel_zero): INIT_S -.Lscal_kernel_Z1: +L(scal_kernel_Z1): st1 DAV, [X], INC_X subs N, N, #1 - bne .Lscal_kernel_Z1 + bne L(scal_kernel_Z1) mov w0, wzr ret diff --git a/kernel/arm64/sgemm_beta.S b/kernel/arm64/sgemm_beta.S index 574485bc44..d892d97ab0 100644 --- a/kernel/arm64/sgemm_beta.S +++ b/kernel/arm64/sgemm_beta.S @@ -102,33 +102,33 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ldr LDC, [sp] SAVE_REGS -.Lgemm_beta_BEGIN: +L(gemm_beta_BEGIN): fmov beta0, BETA cmp N, #0 - ble .Lgemm_beta_L999 + ble L(gemm_beta_L999) fcmp BETA, #0.0 - beq .Lgemm_beta_zero_01 + beq L(gemm_beta_zero_01) -.Lgemm_beta_01: +L(gemm_beta_01): lsl LDC, LDC, #2 .align 5 -.Lgemm_beta_02: +L(gemm_beta_02): mov A01, C00 add C00, C00, LDC asr I, M, #5 cmp I, #0 - ble .Lgemm_beta_04 + ble L(gemm_beta_04) add A02, A01, #32 add A03, A02, #32 add A04, A03, #32 .align 5 -.Lgemm_beta_03: +L(gemm_beta_03): prfm PLDL1KEEP, [A01, prfm_size] @@ -161,17 +161,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add A04, A04, calc_size subs I , I , #1 - bne .Lgemm_beta_03 + bne L(gemm_beta_03) .align 5 -.Lgemm_beta_04: +L(gemm_beta_04): and I, M , #31 cmp I, #0 - ble .Lgemm_beta_06 + ble L(gemm_beta_06) .align 5 -.Lgemm_beta_05: +L(gemm_beta_05): ldr s12, [A01] fmul s12, s12, beta0 @@ -179,42 +179,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add A01, A01, #4 subs I , I , #1 - bne .Lgemm_beta_05 + bne L(gemm_beta_05) .align 5 -.Lgemm_beta_06: +L(gemm_beta_06): subs N , N, #1 // N-- - bne .Lgemm_beta_02 + bne L(gemm_beta_02) .align 5 -.Lgemm_beta_L999: +L(gemm_beta_L999): mov x0, #0 RESTORE_REGS ret .align 5 -.Lgemm_beta_zero_01: +L(gemm_beta_zero_01): INIT_ZERO lsl LDC, LDC, #2 .align 5 -.Lgemm_beta_zero_02: +L(gemm_beta_zero_02): mov A01, C00 add C00, C00, LDC asr I, M, #5 cmp I, #0 - ble .Lgemm_beta_zero_04 + ble L(gemm_beta_zero_04) add A02, A01, #32 add A03, A02, #32 add A04, A03, #32 .align 5 -.Lgemm_beta_zero_03: +L(gemm_beta_zero_03): st1 {v0.4s, v1.4s}, [A01] add A01, A01, calc_size @@ -226,32 +226,32 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add A04, A04, calc_size subs I, I, #1 - bne .Lgemm_beta_zero_03 + bne L(gemm_beta_zero_03) .align 5 -.Lgemm_beta_zero_04: +L(gemm_beta_zero_04): and I, M, #31 cmp I, #0 - ble .Lgemm_beta_zero_06 + ble L(gemm_beta_zero_06) .align 5 -.Lgemm_beta_zero_05: +L(gemm_beta_zero_05): str beta0, [A01] add A01, A01, #4 subs I, I, #1 - bne .Lgemm_beta_zero_05 + bne L(gemm_beta_zero_05) .align 5 -.Lgemm_beta_zero_06: +L(gemm_beta_zero_06): subs N, N, #1 - bne .Lgemm_beta_zero_02 + bne L(gemm_beta_zero_02) .align 5 -.Lgemm_beta_zero_L999: +L(gemm_beta_zero_L999): mov x0, #0 RESTORE_REGS ret diff --git a/kernel/arm64/sgemm_kernel_16x4.S b/kernel/arm64/sgemm_kernel_16x4.S index 99099ea6fd..3ad1f008a0 100644 --- a/kernel/arm64/sgemm_kernel_16x4.S +++ b/kernel/arm64/sgemm_kernel_16x4.S @@ -1070,7 +1070,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE -.Lsgemm_kernel_begin: +L(sgemm_kernel_begin): .align 5 add sp, sp, #-(11 * 16) @@ -1098,11 +1098,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lsgemm_kernel_L2_BEGIN + ble L(sgemm_kernel_L2_BEGIN) /******************************************************************************/ -.Lsgemm_kernel_L4_BEGIN: +L(sgemm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -1112,21 +1112,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lsgemm_kernel_L4_M16_BEGIN: +L(sgemm_kernel_L4_M16_BEGIN): mov counterI, origM asr counterI, counterI, #4 // counterI = counterI / 16 cmp counterI, #0 - ble .Lsgemm_kernel_L4_M8_BEGIN + ble L(sgemm_kernel_L4_M8_BEGIN) .align 5 -.Lsgemm_kernel_L4_M16_20: +L(sgemm_kernel_L4_M16_20): mov pB, origPB asr counterL , origK, #3 cmp counterL , #2 - blt .Lsgemm_kernel_L4_M16_32 + blt L(sgemm_kernel_L4_M16_32) KERNEL16x4_I KERNEL16x4_M2 @@ -1138,10 +1138,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x4_M2 subs counterL, counterL, #2 - ble .Lsgemm_kernel_L4_M16_22a + ble L(sgemm_kernel_L4_M16_22a) .align 5 -.Lsgemm_kernel_L4_M16_22: +L(sgemm_kernel_L4_M16_22): KERNEL16x4_M1 KERNEL16x4_M2 @@ -1153,10 +1153,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x4_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M16_22 + bgt L(sgemm_kernel_L4_M16_22) .align 5 -.Lsgemm_kernel_L4_M16_22a: +L(sgemm_kernel_L4_M16_22a): KERNEL16x4_M1 KERNEL16x4_M2 @@ -1167,13 +1167,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x4_M1 KERNEL16x4_E - b .Lsgemm_kernel_L4_M16_44 + b L(sgemm_kernel_L4_M16_44) .align 5 -.Lsgemm_kernel_L4_M16_32: +L(sgemm_kernel_L4_M16_32): tst counterL, #1 - ble .Lsgemm_kernel_L4_M16_40 + ble L(sgemm_kernel_L4_M16_40) KERNEL16x4_I KERNEL16x4_M2 @@ -1184,187 +1184,187 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x4_M1 KERNEL16x4_E - b .Lsgemm_kernel_L4_M16_44 + b L(sgemm_kernel_L4_M16_44) -.Lsgemm_kernel_L4_M16_40: +L(sgemm_kernel_L4_M16_40): INIT16x4 -.Lsgemm_kernel_L4_M16_44: +L(sgemm_kernel_L4_M16_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L4_M16_100 + ble L(sgemm_kernel_L4_M16_100) .align 5 -.Lsgemm_kernel_L4_M16_46: +L(sgemm_kernel_L4_M16_46): KERNEL16x4_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L4_M16_46 + bne L(sgemm_kernel_L4_M16_46) -.Lsgemm_kernel_L4_M16_100: +L(sgemm_kernel_L4_M16_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVE16x4 -.Lsgemm_kernel_L4_M16_END: +L(sgemm_kernel_L4_M16_END): subs counterI, counterI, #1 - bne .Lsgemm_kernel_L4_M16_20 + bne L(sgemm_kernel_L4_M16_20) //------------------------------------------------------------------------------ -.Lsgemm_kernel_L4_M8_BEGIN: +L(sgemm_kernel_L4_M8_BEGIN): mov counterI, origM tst counterI , #15 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #8 - ble .Lsgemm_kernel_L4_M4_BEGIN + ble L(sgemm_kernel_L4_M4_BEGIN) -.Lsgemm_kernel_L4_M8_20: +L(sgemm_kernel_L4_M8_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L4_M8_32 + blt L(sgemm_kernel_L4_M8_32) KERNEL8x4_I // do one in the K KERNEL8x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L4_M8_22a + ble L(sgemm_kernel_L4_M8_22a) .align 5 -.Lsgemm_kernel_L4_M8_22: +L(sgemm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M8_22 + bgt L(sgemm_kernel_L4_M8_22) -.Lsgemm_kernel_L4_M8_22a: +L(sgemm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_E - b .Lsgemm_kernel_L4_M8_44 + b L(sgemm_kernel_L4_M8_44) -.Lsgemm_kernel_L4_M8_32: +L(sgemm_kernel_L4_M8_32): tst counterL, #1 - ble .Lsgemm_kernel_L4_M8_40 + ble L(sgemm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_E - b .Lsgemm_kernel_L4_M8_44 + b L(sgemm_kernel_L4_M8_44) -.Lsgemm_kernel_L4_M8_40: +L(sgemm_kernel_L4_M8_40): INIT8x4 -.Lsgemm_kernel_L4_M8_44: +L(sgemm_kernel_L4_M8_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L4_M8_100 + ble L(sgemm_kernel_L4_M8_100) -.Lsgemm_kernel_L4_M8_46: +L(sgemm_kernel_L4_M8_46): KERNEL8x4_SUB -.Lsgemm_kernel_L4_M8_100: +L(sgemm_kernel_L4_M8_100): SAVE8x4 -.Lsgemm_kernel_L4_M8_END: +L(sgemm_kernel_L4_M8_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L4_M4_BEGIN: +L(sgemm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #4 - ble .Lsgemm_kernel_L4_M2_BEGIN + ble L(sgemm_kernel_L4_M2_BEGIN) -.Lsgemm_kernel_L4_M4_20: +L(sgemm_kernel_L4_M4_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L4_M4_32 + blt L(sgemm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L4_M4_22a + ble L(sgemm_kernel_L4_M4_22a) .align 5 -.Lsgemm_kernel_L4_M4_22: +L(sgemm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M4_22 + bgt L(sgemm_kernel_L4_M4_22) -.Lsgemm_kernel_L4_M4_22a: +L(sgemm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lsgemm_kernel_L4_M4_44 + b L(sgemm_kernel_L4_M4_44) -.Lsgemm_kernel_L4_M4_32: +L(sgemm_kernel_L4_M4_32): tst counterL, #1 - ble .Lsgemm_kernel_L4_M4_40 + ble L(sgemm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lsgemm_kernel_L4_M4_44 + b L(sgemm_kernel_L4_M4_44) -.Lsgemm_kernel_L4_M4_40: +L(sgemm_kernel_L4_M4_40): INIT4x4 -.Lsgemm_kernel_L4_M4_44: +L(sgemm_kernel_L4_M4_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L4_M4_100 + ble L(sgemm_kernel_L4_M4_100) -.Lsgemm_kernel_L4_M4_46: +L(sgemm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lsgemm_kernel_L4_M4_100: +L(sgemm_kernel_L4_M4_100): SAVE4x4 -.Lsgemm_kernel_L4_M4_END: +L(sgemm_kernel_L4_M4_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L4_M2_BEGIN: +L(sgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L4_M1_BEGIN + ble L(sgemm_kernel_L4_M1_BEGIN) -.Lsgemm_kernel_L4_M2_20: +L(sgemm_kernel_L4_M2_20): INIT2x4 @@ -1372,9 +1372,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L4_M2_40 + ble L(sgemm_kernel_L4_M2_40) -.Lsgemm_kernel_L4_M2_22: +L(sgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1387,34 +1387,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M2_22 + bgt L(sgemm_kernel_L4_M2_22) -.Lsgemm_kernel_L4_M2_40: +L(sgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M2_100 + ble L(sgemm_kernel_L4_M2_100) -.Lsgemm_kernel_L4_M2_42: +L(sgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M2_42 + bgt L(sgemm_kernel_L4_M2_42) -.Lsgemm_kernel_L4_M2_100: +L(sgemm_kernel_L4_M2_100): SAVE2x4 -.Lsgemm_kernel_L4_M2_END: +L(sgemm_kernel_L4_M2_END): -.Lsgemm_kernel_L4_M1_BEGIN: +L(sgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) -.Lsgemm_kernel_L4_M1_20: +L(sgemm_kernel_L4_M1_20): INIT1x4 @@ -1422,9 +1422,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L4_M1_40 + ble L(sgemm_kernel_L4_M1_40) -.Lsgemm_kernel_L4_M1_22: +L(sgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1436,42 +1436,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M1_22 + bgt L(sgemm_kernel_L4_M1_22) -.Lsgemm_kernel_L4_M1_40: +L(sgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M1_100 + ble L(sgemm_kernel_L4_M1_100) -.Lsgemm_kernel_L4_M1_42: +L(sgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M1_42 + bgt L(sgemm_kernel_L4_M1_42) -.Lsgemm_kernel_L4_M1_100: +L(sgemm_kernel_L4_M1_100): SAVE1x4 -.Lsgemm_kernel_L4_END: +L(sgemm_kernel_L4_END): add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4 subs counterJ, counterJ , #1 // j-- - bgt .Lsgemm_kernel_L4_BEGIN + bgt L(sgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Lsgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(sgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lsgemm_kernel_L999 + ble L(sgemm_kernel_L999) tst counterJ , #2 - ble .Lsgemm_kernel_L1_BEGIN + ble L(sgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1479,14 +1479,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lsgemm_kernel_L2_M16_BEGIN: +L(sgemm_kernel_L2_M16_BEGIN): mov counterI, origM asr counterI, counterI, #4 // counterI = counterI / 16 cmp counterI,#0 - ble .Lsgemm_kernel_L2_M8_BEGIN + ble L(sgemm_kernel_L2_M8_BEGIN) -.Lsgemm_kernel_L2_M16_20: +L(sgemm_kernel_L2_M16_20): INIT16x2 @@ -1494,10 +1494,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M16_40 + ble L(sgemm_kernel_L2_M16_40) .align 5 -.Lsgemm_kernel_L2_M16_22: +L(sgemm_kernel_L2_M16_22): KERNEL16x2_SUB KERNEL16x2_SUB KERNEL16x2_SUB @@ -1509,41 +1509,41 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M16_22 + bgt L(sgemm_kernel_L2_M16_22) -.Lsgemm_kernel_L2_M16_40: +L(sgemm_kernel_L2_M16_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M16_100 + ble L(sgemm_kernel_L2_M16_100) -.Lsgemm_kernel_L2_M16_42: +L(sgemm_kernel_L2_M16_42): KERNEL16x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M16_42 + bgt L(sgemm_kernel_L2_M16_42) -.Lsgemm_kernel_L2_M16_100: +L(sgemm_kernel_L2_M16_100): SAVE16x2 -.Lsgemm_kernel_L2_M16_END: +L(sgemm_kernel_L2_M16_END): subs counterI, counterI, #1 - bgt .Lsgemm_kernel_L2_M16_20 + bgt L(sgemm_kernel_L2_M16_20) //------------------------------------------------------------------------------ -.Lsgemm_kernel_L2_M8_BEGIN: +L(sgemm_kernel_L2_M8_BEGIN): mov counterI, origM tst counterI , #15 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) tst counterI, #8 - ble .Lsgemm_kernel_L2_M4_BEGIN + ble L(sgemm_kernel_L2_M4_BEGIN) -.Lsgemm_kernel_L2_M8_20: +L(sgemm_kernel_L2_M8_20): INIT8x2 @@ -1551,10 +1551,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M8_40 + ble L(sgemm_kernel_L2_M8_40) .align 5 -.Lsgemm_kernel_L2_M8_22: +L(sgemm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB KERNEL8x2_SUB @@ -1566,38 +1566,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M8_22 + bgt L(sgemm_kernel_L2_M8_22) -.Lsgemm_kernel_L2_M8_40: +L(sgemm_kernel_L2_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M8_100 + ble L(sgemm_kernel_L2_M8_100) -.Lsgemm_kernel_L2_M8_42: +L(sgemm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M8_42 + bgt L(sgemm_kernel_L2_M8_42) -.Lsgemm_kernel_L2_M8_100: +L(sgemm_kernel_L2_M8_100): SAVE8x2 -.Lsgemm_kernel_L2_M8_END: +L(sgemm_kernel_L2_M8_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L2_M4_BEGIN: +L(sgemm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) tst counterI, #4 - ble .Lsgemm_kernel_L2_M2_BEGIN + ble L(sgemm_kernel_L2_M2_BEGIN) -.Lsgemm_kernel_L2_M4_20: +L(sgemm_kernel_L2_M4_20): INIT4x2 @@ -1605,10 +1605,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M4_40 + ble L(sgemm_kernel_L2_M4_40) .align 5 -.Lsgemm_kernel_L2_M4_22: +L(sgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1620,40 +1620,40 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M4_22 + bgt L(sgemm_kernel_L2_M4_22) -.Lsgemm_kernel_L2_M4_40: +L(sgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M4_100 + ble L(sgemm_kernel_L2_M4_100) -.Lsgemm_kernel_L2_M4_42: +L(sgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M4_42 + bgt L(sgemm_kernel_L2_M4_42) -.Lsgemm_kernel_L2_M4_100: +L(sgemm_kernel_L2_M4_100): SAVE4x2 -.Lsgemm_kernel_L2_M4_END: +L(sgemm_kernel_L2_M4_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L2_M2_BEGIN: +L(sgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L2_M1_BEGIN + ble L(sgemm_kernel_L2_M1_BEGIN) -.Lsgemm_kernel_L2_M2_20: +L(sgemm_kernel_L2_M2_20): INIT2x2 @@ -1661,9 +1661,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M2_40 + ble L(sgemm_kernel_L2_M2_40) -.Lsgemm_kernel_L2_M2_22: +L(sgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1676,34 +1676,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M2_22 + bgt L(sgemm_kernel_L2_M2_22) -.Lsgemm_kernel_L2_M2_40: +L(sgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M2_100 + ble L(sgemm_kernel_L2_M2_100) -.Lsgemm_kernel_L2_M2_42: +L(sgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M2_42 + bgt L(sgemm_kernel_L2_M2_42) -.Lsgemm_kernel_L2_M2_100: +L(sgemm_kernel_L2_M2_100): SAVE2x2 -.Lsgemm_kernel_L2_M2_END: +L(sgemm_kernel_L2_M2_END): -.Lsgemm_kernel_L2_M1_BEGIN: +L(sgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) -.Lsgemm_kernel_L2_M1_20: +L(sgemm_kernel_L2_M1_20): INIT1x2 @@ -1711,9 +1711,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lsgemm_kernel_L2_M1_40 + ble L(sgemm_kernel_L2_M1_40) -.Lsgemm_kernel_L2_M1_22: +L(sgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1725,36 +1725,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M1_22 + bgt L(sgemm_kernel_L2_M1_22) -.Lsgemm_kernel_L2_M1_40: +L(sgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M1_100 + ble L(sgemm_kernel_L2_M1_100) -.Lsgemm_kernel_L2_M1_42: +L(sgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M1_42 + bgt L(sgemm_kernel_L2_M1_42) -.Lsgemm_kernel_L2_M1_100: +L(sgemm_kernel_L2_M1_100): SAVE1x2 -.Lsgemm_kernel_L2_END: +L(sgemm_kernel_L2_END): add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 /******************************************************************************/ -.Lsgemm_kernel_L1_BEGIN: +L(sgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lsgemm_kernel_L999 // done + ble L(sgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1762,14 +1762,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lsgemm_kernel_L1_M16_BEGIN: +L(sgemm_kernel_L1_M16_BEGIN): mov counterI, origM asr counterI, counterI, #4 // counterI = counterI / 16 cmp counterI, #0 - ble .Lsgemm_kernel_L1_M8_BEGIN + ble L(sgemm_kernel_L1_M8_BEGIN) -.Lsgemm_kernel_L1_M16_20: +L(sgemm_kernel_L1_M16_20): INIT16x1 @@ -1777,10 +1777,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M16_40 + ble L(sgemm_kernel_L1_M16_40) .align 5 -.Lsgemm_kernel_L1_M16_22: +L(sgemm_kernel_L1_M16_22): KERNEL16x1_SUB KERNEL16x1_SUB KERNEL16x1_SUB @@ -1792,42 +1792,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M16_22 + bgt L(sgemm_kernel_L1_M16_22) -.Lsgemm_kernel_L1_M16_40: +L(sgemm_kernel_L1_M16_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M16_100 + ble L(sgemm_kernel_L1_M16_100) -.Lsgemm_kernel_L1_M16_42: +L(sgemm_kernel_L1_M16_42): KERNEL16x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M16_42 + bgt L(sgemm_kernel_L1_M16_42) -.Lsgemm_kernel_L1_M16_100: +L(sgemm_kernel_L1_M16_100): SAVE16x1 -.Lsgemm_kernel_L1_M16_END: +L(sgemm_kernel_L1_M16_END): subs counterI, counterI, #1 - bgt .Lsgemm_kernel_L1_M16_20 + bgt L(sgemm_kernel_L1_M16_20) //------------------------------------------------------------------------------ -.Lsgemm_kernel_L1_M8_BEGIN: +L(sgemm_kernel_L1_M8_BEGIN): mov counterI, origM tst counterI , #15 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) tst counterI, #8 - ble .Lsgemm_kernel_L1_M4_BEGIN + ble L(sgemm_kernel_L1_M4_BEGIN) -.Lsgemm_kernel_L1_M8_20: +L(sgemm_kernel_L1_M8_20): INIT8x1 @@ -1835,10 +1835,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M8_40 + ble L(sgemm_kernel_L1_M8_40) .align 5 -.Lsgemm_kernel_L1_M8_22: +L(sgemm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -1850,38 +1850,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M8_22 + bgt L(sgemm_kernel_L1_M8_22) -.Lsgemm_kernel_L1_M8_40: +L(sgemm_kernel_L1_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M8_100 + ble L(sgemm_kernel_L1_M8_100) -.Lsgemm_kernel_L1_M8_42: +L(sgemm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M8_42 + bgt L(sgemm_kernel_L1_M8_42) -.Lsgemm_kernel_L1_M8_100: +L(sgemm_kernel_L1_M8_100): SAVE8x1 -.Lsgemm_kernel_L1_M8_END: +L(sgemm_kernel_L1_M8_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L1_M4_BEGIN: +L(sgemm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) tst counterI, #4 - ble .Lsgemm_kernel_L1_M2_BEGIN + ble L(sgemm_kernel_L1_M2_BEGIN) -.Lsgemm_kernel_L1_M4_20: +L(sgemm_kernel_L1_M4_20): INIT4x1 @@ -1889,10 +1889,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M4_40 + ble L(sgemm_kernel_L1_M4_40) .align 5 -.Lsgemm_kernel_L1_M4_22: +L(sgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1904,39 +1904,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M4_22 + bgt L(sgemm_kernel_L1_M4_22) -.Lsgemm_kernel_L1_M4_40: +L(sgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M4_100 + ble L(sgemm_kernel_L1_M4_100) -.Lsgemm_kernel_L1_M4_42: +L(sgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M4_42 + bgt L(sgemm_kernel_L1_M4_42) -.Lsgemm_kernel_L1_M4_100: +L(sgemm_kernel_L1_M4_100): SAVE4x1 -.Lsgemm_kernel_L1_M4_END: +L(sgemm_kernel_L1_M4_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L1_M2_BEGIN: +L(sgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L1_M1_BEGIN + ble L(sgemm_kernel_L1_M1_BEGIN) -.Lsgemm_kernel_L1_M2_20: +L(sgemm_kernel_L1_M2_20): INIT2x1 @@ -1944,9 +1944,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M2_40 + ble L(sgemm_kernel_L1_M2_40) -.Lsgemm_kernel_L1_M2_22: +L(sgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1959,34 +1959,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M2_22 + bgt L(sgemm_kernel_L1_M2_22) -.Lsgemm_kernel_L1_M2_40: +L(sgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M2_100 + ble L(sgemm_kernel_L1_M2_100) -.Lsgemm_kernel_L1_M2_42: +L(sgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M2_42 + bgt L(sgemm_kernel_L1_M2_42) -.Lsgemm_kernel_L1_M2_100: +L(sgemm_kernel_L1_M2_100): SAVE2x1 -.Lsgemm_kernel_L1_M2_END: +L(sgemm_kernel_L1_M2_END): -.Lsgemm_kernel_L1_M1_BEGIN: +L(sgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) -.Lsgemm_kernel_L1_M1_20: +L(sgemm_kernel_L1_M1_20): INIT1x1 @@ -1994,9 +1994,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M1_40 + ble L(sgemm_kernel_L1_M1_40) -.Lsgemm_kernel_L1_M1_22: +L(sgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -2008,28 +2008,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M1_22 + bgt L(sgemm_kernel_L1_M1_22) -.Lsgemm_kernel_L1_M1_40: +L(sgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M1_100 + ble L(sgemm_kernel_L1_M1_100) -.Lsgemm_kernel_L1_M1_42: +L(sgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M1_42 + bgt L(sgemm_kernel_L1_M1_42) -.Lsgemm_kernel_L1_M1_100: +L(sgemm_kernel_L1_M1_100): SAVE1x1 -.Lsgemm_kernel_L1_END: +L(sgemm_kernel_L1_END): -.Lsgemm_kernel_L999: +L(sgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/sgemm_kernel_16x4_thunderx2t99.S b/kernel/arm64/sgemm_kernel_16x4_thunderx2t99.S index 144d4bcd69..e2a225f384 100644 --- a/kernel/arm64/sgemm_kernel_16x4_thunderx2t99.S +++ b/kernel/arm64/sgemm_kernel_16x4_thunderx2t99.S @@ -1117,7 +1117,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE -.Lsgemm_kernel_begin: +L(sgemm_kernel_begin): .align 5 add sp, sp, #-(11 * 16) @@ -1145,11 +1145,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lsgemm_kernel_L2_BEGIN + ble L(sgemm_kernel_L2_BEGIN) /******************************************************************************/ -.Lsgemm_kernel_L4_BEGIN: +L(sgemm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -1159,21 +1159,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lsgemm_kernel_L4_M16_BEGIN: +L(sgemm_kernel_L4_M16_BEGIN): mov counterI, origM asr counterI, counterI, #4 // counterI = counterI / 16 cmp counterI, #0 - ble .Lsgemm_kernel_L4_M8_BEGIN + ble L(sgemm_kernel_L4_M8_BEGIN) .align 5 -.Lsgemm_kernel_L4_M16_20: +L(sgemm_kernel_L4_M16_20): mov pB, origPB asr counterL , origK, #4 // L = K / 16 cmp counterL , #2 - blt .Lsgemm_kernel_L4_M16_32 + blt L(sgemm_kernel_L4_M16_32) KERNEL16x4_I KERNEL16x4_M2 @@ -1182,18 +1182,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x4_M1_M2_x1 subs counterL, counterL, #2 - ble .Lsgemm_kernel_L4_M16_22a + ble L(sgemm_kernel_L4_M16_22a) .align 5 -.Lsgemm_kernel_L4_M16_22: +L(sgemm_kernel_L4_M16_22): KERNEL16x4_M1_M2_x8 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M16_22 + bgt L(sgemm_kernel_L4_M16_22) .align 5 -.Lsgemm_kernel_L4_M16_22a: +L(sgemm_kernel_L4_M16_22a): KERNEL16x4_M1_M2_x4 KERNEL16x4_M1_M2_x2 @@ -1201,13 +1201,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x4_M1 KERNEL16x4_E - b .Lsgemm_kernel_L4_M16_44 + b L(sgemm_kernel_L4_M16_44) .align 5 -.Lsgemm_kernel_L4_M16_32: +L(sgemm_kernel_L4_M16_32): tst counterL, #1 - ble .Lsgemm_kernel_L4_M16_40 + ble L(sgemm_kernel_L4_M16_40) KERNEL16x4_I KERNEL16x4_M2 @@ -1216,187 +1216,187 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x4_M1 KERNEL16x4_E - b .Lsgemm_kernel_L4_M16_44 + b L(sgemm_kernel_L4_M16_44) -.Lsgemm_kernel_L4_M16_40: +L(sgemm_kernel_L4_M16_40): INIT16x4 -.Lsgemm_kernel_L4_M16_44: +L(sgemm_kernel_L4_M16_44): ands counterL , origK, #15 - ble .Lsgemm_kernel_L4_M16_100 + ble L(sgemm_kernel_L4_M16_100) .align 5 -.Lsgemm_kernel_L4_M16_46: +L(sgemm_kernel_L4_M16_46): KERNEL16x4_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L4_M16_46 + bne L(sgemm_kernel_L4_M16_46) -.Lsgemm_kernel_L4_M16_100: +L(sgemm_kernel_L4_M16_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVE16x4 -.Lsgemm_kernel_L4_M16_END: +L(sgemm_kernel_L4_M16_END): subs counterI, counterI, #1 - bne .Lsgemm_kernel_L4_M16_20 + bne L(sgemm_kernel_L4_M16_20) //------------------------------------------------------------------------------ -.Lsgemm_kernel_L4_M8_BEGIN: +L(sgemm_kernel_L4_M8_BEGIN): mov counterI, origM tst counterI , #15 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #8 - ble .Lsgemm_kernel_L4_M4_BEGIN + ble L(sgemm_kernel_L4_M4_BEGIN) -.Lsgemm_kernel_L4_M8_20: +L(sgemm_kernel_L4_M8_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L4_M8_32 + blt L(sgemm_kernel_L4_M8_32) KERNEL8x4_I // do one in the K KERNEL8x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L4_M8_22a + ble L(sgemm_kernel_L4_M8_22a) .align 5 -.Lsgemm_kernel_L4_M8_22: +L(sgemm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M8_22 + bgt L(sgemm_kernel_L4_M8_22) -.Lsgemm_kernel_L4_M8_22a: +L(sgemm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_E - b .Lsgemm_kernel_L4_M8_44 + b L(sgemm_kernel_L4_M8_44) -.Lsgemm_kernel_L4_M8_32: +L(sgemm_kernel_L4_M8_32): tst counterL, #1 - ble .Lsgemm_kernel_L4_M8_40 + ble L(sgemm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_E - b .Lsgemm_kernel_L4_M8_44 + b L(sgemm_kernel_L4_M8_44) -.Lsgemm_kernel_L4_M8_40: +L(sgemm_kernel_L4_M8_40): INIT8x4 -.Lsgemm_kernel_L4_M8_44: +L(sgemm_kernel_L4_M8_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L4_M8_100 + ble L(sgemm_kernel_L4_M8_100) -.Lsgemm_kernel_L4_M8_46: +L(sgemm_kernel_L4_M8_46): KERNEL8x4_SUB -.Lsgemm_kernel_L4_M8_100: +L(sgemm_kernel_L4_M8_100): SAVE8x4 -.Lsgemm_kernel_L4_M8_END: +L(sgemm_kernel_L4_M8_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L4_M4_BEGIN: +L(sgemm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #4 - ble .Lsgemm_kernel_L4_M2_BEGIN + ble L(sgemm_kernel_L4_M2_BEGIN) -.Lsgemm_kernel_L4_M4_20: +L(sgemm_kernel_L4_M4_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L4_M4_32 + blt L(sgemm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L4_M4_22a + ble L(sgemm_kernel_L4_M4_22a) .align 5 -.Lsgemm_kernel_L4_M4_22: +L(sgemm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M4_22 + bgt L(sgemm_kernel_L4_M4_22) -.Lsgemm_kernel_L4_M4_22a: +L(sgemm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lsgemm_kernel_L4_M4_44 + b L(sgemm_kernel_L4_M4_44) -.Lsgemm_kernel_L4_M4_32: +L(sgemm_kernel_L4_M4_32): tst counterL, #1 - ble .Lsgemm_kernel_L4_M4_40 + ble L(sgemm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lsgemm_kernel_L4_M4_44 + b L(sgemm_kernel_L4_M4_44) -.Lsgemm_kernel_L4_M4_40: +L(sgemm_kernel_L4_M4_40): INIT4x4 -.Lsgemm_kernel_L4_M4_44: +L(sgemm_kernel_L4_M4_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L4_M4_100 + ble L(sgemm_kernel_L4_M4_100) -.Lsgemm_kernel_L4_M4_46: +L(sgemm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lsgemm_kernel_L4_M4_100: +L(sgemm_kernel_L4_M4_100): SAVE4x4 -.Lsgemm_kernel_L4_M4_END: +L(sgemm_kernel_L4_M4_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L4_M2_BEGIN: +L(sgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L4_M1_BEGIN + ble L(sgemm_kernel_L4_M1_BEGIN) -.Lsgemm_kernel_L4_M2_20: +L(sgemm_kernel_L4_M2_20): INIT2x4 @@ -1404,9 +1404,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L4_M2_40 + ble L(sgemm_kernel_L4_M2_40) -.Lsgemm_kernel_L4_M2_22: +L(sgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1419,34 +1419,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M2_22 + bgt L(sgemm_kernel_L4_M2_22) -.Lsgemm_kernel_L4_M2_40: +L(sgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M2_100 + ble L(sgemm_kernel_L4_M2_100) -.Lsgemm_kernel_L4_M2_42: +L(sgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M2_42 + bgt L(sgemm_kernel_L4_M2_42) -.Lsgemm_kernel_L4_M2_100: +L(sgemm_kernel_L4_M2_100): SAVE2x4 -.Lsgemm_kernel_L4_M2_END: +L(sgemm_kernel_L4_M2_END): -.Lsgemm_kernel_L4_M1_BEGIN: +L(sgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) -.Lsgemm_kernel_L4_M1_20: +L(sgemm_kernel_L4_M1_20): INIT1x4 @@ -1454,9 +1454,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L4_M1_40 + ble L(sgemm_kernel_L4_M1_40) -.Lsgemm_kernel_L4_M1_22: +L(sgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1468,42 +1468,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M1_22 + bgt L(sgemm_kernel_L4_M1_22) -.Lsgemm_kernel_L4_M1_40: +L(sgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M1_100 + ble L(sgemm_kernel_L4_M1_100) -.Lsgemm_kernel_L4_M1_42: +L(sgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M1_42 + bgt L(sgemm_kernel_L4_M1_42) -.Lsgemm_kernel_L4_M1_100: +L(sgemm_kernel_L4_M1_100): SAVE1x4 -.Lsgemm_kernel_L4_END: +L(sgemm_kernel_L4_END): add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4 subs counterJ, counterJ , #1 // j-- - bgt .Lsgemm_kernel_L4_BEGIN + bgt L(sgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Lsgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(sgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lsgemm_kernel_L999 + ble L(sgemm_kernel_L999) tst counterJ , #2 - ble .Lsgemm_kernel_L1_BEGIN + ble L(sgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1511,14 +1511,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lsgemm_kernel_L2_M16_BEGIN: +L(sgemm_kernel_L2_M16_BEGIN): mov counterI, origM asr counterI, counterI, #4 // counterI = counterI / 16 cmp counterI,#0 - ble .Lsgemm_kernel_L2_M8_BEGIN + ble L(sgemm_kernel_L2_M8_BEGIN) -.Lsgemm_kernel_L2_M16_20: +L(sgemm_kernel_L2_M16_20): INIT16x2 @@ -1526,10 +1526,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M16_40 + ble L(sgemm_kernel_L2_M16_40) .align 5 -.Lsgemm_kernel_L2_M16_22: +L(sgemm_kernel_L2_M16_22): KERNEL16x2_SUB KERNEL16x2_SUB KERNEL16x2_SUB @@ -1541,41 +1541,41 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M16_22 + bgt L(sgemm_kernel_L2_M16_22) -.Lsgemm_kernel_L2_M16_40: +L(sgemm_kernel_L2_M16_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M16_100 + ble L(sgemm_kernel_L2_M16_100) -.Lsgemm_kernel_L2_M16_42: +L(sgemm_kernel_L2_M16_42): KERNEL16x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M16_42 + bgt L(sgemm_kernel_L2_M16_42) -.Lsgemm_kernel_L2_M16_100: +L(sgemm_kernel_L2_M16_100): SAVE16x2 -.Lsgemm_kernel_L2_M16_END: +L(sgemm_kernel_L2_M16_END): subs counterI, counterI, #1 - bgt .Lsgemm_kernel_L2_M16_20 + bgt L(sgemm_kernel_L2_M16_20) //------------------------------------------------------------------------------ -.Lsgemm_kernel_L2_M8_BEGIN: +L(sgemm_kernel_L2_M8_BEGIN): mov counterI, origM tst counterI , #15 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) tst counterI, #8 - ble .Lsgemm_kernel_L2_M4_BEGIN + ble L(sgemm_kernel_L2_M4_BEGIN) -.Lsgemm_kernel_L2_M8_20: +L(sgemm_kernel_L2_M8_20): INIT8x2 @@ -1583,10 +1583,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M8_40 + ble L(sgemm_kernel_L2_M8_40) .align 5 -.Lsgemm_kernel_L2_M8_22: +L(sgemm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB KERNEL8x2_SUB @@ -1598,38 +1598,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M8_22 + bgt L(sgemm_kernel_L2_M8_22) -.Lsgemm_kernel_L2_M8_40: +L(sgemm_kernel_L2_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M8_100 + ble L(sgemm_kernel_L2_M8_100) -.Lsgemm_kernel_L2_M8_42: +L(sgemm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M8_42 + bgt L(sgemm_kernel_L2_M8_42) -.Lsgemm_kernel_L2_M8_100: +L(sgemm_kernel_L2_M8_100): SAVE8x2 -.Lsgemm_kernel_L2_M8_END: +L(sgemm_kernel_L2_M8_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L2_M4_BEGIN: +L(sgemm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) tst counterI, #4 - ble .Lsgemm_kernel_L2_M2_BEGIN + ble L(sgemm_kernel_L2_M2_BEGIN) -.Lsgemm_kernel_L2_M4_20: +L(sgemm_kernel_L2_M4_20): INIT4x2 @@ -1637,10 +1637,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M4_40 + ble L(sgemm_kernel_L2_M4_40) .align 5 -.Lsgemm_kernel_L2_M4_22: +L(sgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1652,40 +1652,40 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M4_22 + bgt L(sgemm_kernel_L2_M4_22) -.Lsgemm_kernel_L2_M4_40: +L(sgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M4_100 + ble L(sgemm_kernel_L2_M4_100) -.Lsgemm_kernel_L2_M4_42: +L(sgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M4_42 + bgt L(sgemm_kernel_L2_M4_42) -.Lsgemm_kernel_L2_M4_100: +L(sgemm_kernel_L2_M4_100): SAVE4x2 -.Lsgemm_kernel_L2_M4_END: +L(sgemm_kernel_L2_M4_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L2_M2_BEGIN: +L(sgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L2_M1_BEGIN + ble L(sgemm_kernel_L2_M1_BEGIN) -.Lsgemm_kernel_L2_M2_20: +L(sgemm_kernel_L2_M2_20): INIT2x2 @@ -1693,9 +1693,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M2_40 + ble L(sgemm_kernel_L2_M2_40) -.Lsgemm_kernel_L2_M2_22: +L(sgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1708,34 +1708,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M2_22 + bgt L(sgemm_kernel_L2_M2_22) -.Lsgemm_kernel_L2_M2_40: +L(sgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M2_100 + ble L(sgemm_kernel_L2_M2_100) -.Lsgemm_kernel_L2_M2_42: +L(sgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M2_42 + bgt L(sgemm_kernel_L2_M2_42) -.Lsgemm_kernel_L2_M2_100: +L(sgemm_kernel_L2_M2_100): SAVE2x2 -.Lsgemm_kernel_L2_M2_END: +L(sgemm_kernel_L2_M2_END): -.Lsgemm_kernel_L2_M1_BEGIN: +L(sgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) -.Lsgemm_kernel_L2_M1_20: +L(sgemm_kernel_L2_M1_20): INIT1x2 @@ -1743,9 +1743,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lsgemm_kernel_L2_M1_40 + ble L(sgemm_kernel_L2_M1_40) -.Lsgemm_kernel_L2_M1_22: +L(sgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1757,36 +1757,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M1_22 + bgt L(sgemm_kernel_L2_M1_22) -.Lsgemm_kernel_L2_M1_40: +L(sgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M1_100 + ble L(sgemm_kernel_L2_M1_100) -.Lsgemm_kernel_L2_M1_42: +L(sgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M1_42 + bgt L(sgemm_kernel_L2_M1_42) -.Lsgemm_kernel_L2_M1_100: +L(sgemm_kernel_L2_M1_100): SAVE1x2 -.Lsgemm_kernel_L2_END: +L(sgemm_kernel_L2_END): add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 /******************************************************************************/ -.Lsgemm_kernel_L1_BEGIN: +L(sgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lsgemm_kernel_L999 // done + ble L(sgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1794,14 +1794,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lsgemm_kernel_L1_M16_BEGIN: +L(sgemm_kernel_L1_M16_BEGIN): mov counterI, origM asr counterI, counterI, #4 // counterI = counterI / 16 cmp counterI, #0 - ble .Lsgemm_kernel_L1_M8_BEGIN + ble L(sgemm_kernel_L1_M8_BEGIN) -.Lsgemm_kernel_L1_M16_20: +L(sgemm_kernel_L1_M16_20): INIT16x1 @@ -1809,10 +1809,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M16_40 + ble L(sgemm_kernel_L1_M16_40) .align 5 -.Lsgemm_kernel_L1_M16_22: +L(sgemm_kernel_L1_M16_22): KERNEL16x1_SUB KERNEL16x1_SUB KERNEL16x1_SUB @@ -1824,42 +1824,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M16_22 + bgt L(sgemm_kernel_L1_M16_22) -.Lsgemm_kernel_L1_M16_40: +L(sgemm_kernel_L1_M16_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M16_100 + ble L(sgemm_kernel_L1_M16_100) -.Lsgemm_kernel_L1_M16_42: +L(sgemm_kernel_L1_M16_42): KERNEL16x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M16_42 + bgt L(sgemm_kernel_L1_M16_42) -.Lsgemm_kernel_L1_M16_100: +L(sgemm_kernel_L1_M16_100): SAVE16x1 -.Lsgemm_kernel_L1_M16_END: +L(sgemm_kernel_L1_M16_END): subs counterI, counterI, #1 - bgt .Lsgemm_kernel_L1_M16_20 + bgt L(sgemm_kernel_L1_M16_20) //------------------------------------------------------------------------------ -.Lsgemm_kernel_L1_M8_BEGIN: +L(sgemm_kernel_L1_M8_BEGIN): mov counterI, origM tst counterI , #15 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) tst counterI, #8 - ble .Lsgemm_kernel_L1_M4_BEGIN + ble L(sgemm_kernel_L1_M4_BEGIN) -.Lsgemm_kernel_L1_M8_20: +L(sgemm_kernel_L1_M8_20): INIT8x1 @@ -1867,10 +1867,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M8_40 + ble L(sgemm_kernel_L1_M8_40) .align 5 -.Lsgemm_kernel_L1_M8_22: +L(sgemm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -1882,38 +1882,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M8_22 + bgt L(sgemm_kernel_L1_M8_22) -.Lsgemm_kernel_L1_M8_40: +L(sgemm_kernel_L1_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M8_100 + ble L(sgemm_kernel_L1_M8_100) -.Lsgemm_kernel_L1_M8_42: +L(sgemm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M8_42 + bgt L(sgemm_kernel_L1_M8_42) -.Lsgemm_kernel_L1_M8_100: +L(sgemm_kernel_L1_M8_100): SAVE8x1 -.Lsgemm_kernel_L1_M8_END: +L(sgemm_kernel_L1_M8_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L1_M4_BEGIN: +L(sgemm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) tst counterI, #4 - ble .Lsgemm_kernel_L1_M2_BEGIN + ble L(sgemm_kernel_L1_M2_BEGIN) -.Lsgemm_kernel_L1_M4_20: +L(sgemm_kernel_L1_M4_20): INIT4x1 @@ -1921,10 +1921,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M4_40 + ble L(sgemm_kernel_L1_M4_40) .align 5 -.Lsgemm_kernel_L1_M4_22: +L(sgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1936,39 +1936,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M4_22 + bgt L(sgemm_kernel_L1_M4_22) -.Lsgemm_kernel_L1_M4_40: +L(sgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M4_100 + ble L(sgemm_kernel_L1_M4_100) -.Lsgemm_kernel_L1_M4_42: +L(sgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M4_42 + bgt L(sgemm_kernel_L1_M4_42) -.Lsgemm_kernel_L1_M4_100: +L(sgemm_kernel_L1_M4_100): SAVE4x1 -.Lsgemm_kernel_L1_M4_END: +L(sgemm_kernel_L1_M4_END): //------------------------------------------------------------------------------ -.Lsgemm_kernel_L1_M2_BEGIN: +L(sgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L1_M1_BEGIN + ble L(sgemm_kernel_L1_M1_BEGIN) -.Lsgemm_kernel_L1_M2_20: +L(sgemm_kernel_L1_M2_20): INIT2x1 @@ -1976,9 +1976,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M2_40 + ble L(sgemm_kernel_L1_M2_40) -.Lsgemm_kernel_L1_M2_22: +L(sgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1991,34 +1991,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M2_22 + bgt L(sgemm_kernel_L1_M2_22) -.Lsgemm_kernel_L1_M2_40: +L(sgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M2_100 + ble L(sgemm_kernel_L1_M2_100) -.Lsgemm_kernel_L1_M2_42: +L(sgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M2_42 + bgt L(sgemm_kernel_L1_M2_42) -.Lsgemm_kernel_L1_M2_100: +L(sgemm_kernel_L1_M2_100): SAVE2x1 -.Lsgemm_kernel_L1_M2_END: +L(sgemm_kernel_L1_M2_END): -.Lsgemm_kernel_L1_M1_BEGIN: +L(sgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) -.Lsgemm_kernel_L1_M1_20: +L(sgemm_kernel_L1_M1_20): INIT1x1 @@ -2026,9 +2026,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M1_40 + ble L(sgemm_kernel_L1_M1_40) -.Lsgemm_kernel_L1_M1_22: +L(sgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -2040,28 +2040,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M1_22 + bgt L(sgemm_kernel_L1_M1_22) -.Lsgemm_kernel_L1_M1_40: +L(sgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M1_100 + ble L(sgemm_kernel_L1_M1_100) -.Lsgemm_kernel_L1_M1_42: +L(sgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M1_42 + bgt L(sgemm_kernel_L1_M1_42) -.Lsgemm_kernel_L1_M1_100: +L(sgemm_kernel_L1_M1_100): SAVE1x1 -.Lsgemm_kernel_L1_END: +L(sgemm_kernel_L1_END): -.Lsgemm_kernel_L999: +L(sgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/sgemm_kernel_4x4.S b/kernel/arm64/sgemm_kernel_4x4.S index 76c11f1e14..354aaa2994 100644 --- a/kernel/arm64/sgemm_kernel_4x4.S +++ b/kernel/arm64/sgemm_kernel_4x4.S @@ -892,11 +892,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lsgemm_kernel_L2_BEGIN + ble L(sgemm_kernel_L2_BEGIN) /******************************************************************************/ -.Lsgemm_kernel_L4_BEGIN: +L(sgemm_kernel_L4_BEGIN): mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #2 @@ -906,73 +906,73 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add pA_2, temp, pA_1 add pA_3, temp, pA_2 -.Lsgemm_kernel_L4_M16_BEGIN: +L(sgemm_kernel_L4_M16_BEGIN): mov counterI, origM asr counterI, counterI, #4 // counterI = counterI / 16 cmp counterI, #0 - ble .Lsgemm_kernel_L4_M8_BEGIN + ble L(sgemm_kernel_L4_M8_BEGIN) -.Lsgemm_kernel_L4_M16_20: +L(sgemm_kernel_L4_M16_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L4_M16_32 + blt L(sgemm_kernel_L4_M16_32) KERNEL16x4_I // do one in the K KERNEL16x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L4_M16_22a + ble L(sgemm_kernel_L4_M16_22a) .align 5 -.Lsgemm_kernel_L4_M16_22: +L(sgemm_kernel_L4_M16_22): KERNEL16x4_M1 KERNEL16x4_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M16_22 + bgt L(sgemm_kernel_L4_M16_22) -.Lsgemm_kernel_L4_M16_22a: +L(sgemm_kernel_L4_M16_22a): KERNEL16x4_M1 KERNEL16x4_E - b .Lsgemm_kernel_L4_M16_44 + b L(sgemm_kernel_L4_M16_44) -.Lsgemm_kernel_L4_M16_32: +L(sgemm_kernel_L4_M16_32): tst counterL, #1 - ble .Lsgemm_kernel_L4_M16_40 + ble L(sgemm_kernel_L4_M16_40) KERNEL16x4_I KERNEL16x4_E - b .Lsgemm_kernel_L4_M16_44 + b L(sgemm_kernel_L4_M16_44) -.Lsgemm_kernel_L4_M16_40: +L(sgemm_kernel_L4_M16_40): INIT16x4 -.Lsgemm_kernel_L4_M16_44: +L(sgemm_kernel_L4_M16_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L4_M16_100 + ble L(sgemm_kernel_L4_M16_100) -.Lsgemm_kernel_L4_M16_46: +L(sgemm_kernel_L4_M16_46): KERNEL16x4_SUB -.Lsgemm_kernel_L4_M16_100: +L(sgemm_kernel_L4_M16_100): SAVE16x4 -.Lsgemm_kernel_L4_M16_END: +L(sgemm_kernel_L4_M16_END): lsl temp, origK, #4 // k * 4 * 4 = Four rows of A add pA_0, pA_0, temp add pA_0, pA_0, temp @@ -981,26 +981,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add pA_2, pA_1, temp add pA_3, pA_2, temp subs counterI, counterI, #1 - bne .Lsgemm_kernel_L4_M16_20 + bne L(sgemm_kernel_L4_M16_20) -.Lsgemm_kernel_L4_M8_BEGIN: +L(sgemm_kernel_L4_M8_BEGIN): mov counterI, origM tst counterI , #15 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #8 - ble .Lsgemm_kernel_L4_M4_BEGIN + ble L(sgemm_kernel_L4_M4_BEGIN) -.Lsgemm_kernel_L4_M8_20: +L(sgemm_kernel_L4_M8_20): INIT8x4 mov pB, origPB asr counterL, origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lsgemm_kernel_L4_M8_40 + ble L(sgemm_kernel_L4_M8_40) -.Lsgemm_kernel_L4_M8_22: +L(sgemm_kernel_L4_M8_22): KERNEL8x4_SUB KERNEL8x4_SUB @@ -1013,47 +1013,47 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M8_22 + bgt L(sgemm_kernel_L4_M8_22) -.Lsgemm_kernel_L4_M8_40: +L(sgemm_kernel_L4_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M8_100 + ble L(sgemm_kernel_L4_M8_100) -.Lsgemm_kernel_L4_M8_42: +L(sgemm_kernel_L4_M8_42): KERNEL8x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M8_42 + bgt L(sgemm_kernel_L4_M8_42) -.Lsgemm_kernel_L4_M8_100: +L(sgemm_kernel_L4_M8_100): SAVE8x4 -.Lsgemm_kernel_L4_M8_END: +L(sgemm_kernel_L4_M8_END): lsl temp, origK, #4 // k * 4 * 4 add pA_0, pA_0, temp -.Lsgemm_kernel_L4_M4_BEGIN: +L(sgemm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #4 - ble .Lsgemm_kernel_L4_M2_BEGIN + ble L(sgemm_kernel_L4_M2_BEGIN) -.Lsgemm_kernel_L4_M4_20: +L(sgemm_kernel_L4_M4_20): INIT4x4 mov pB, origPB asr counterL, origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lsgemm_kernel_L4_M4_40 + ble L(sgemm_kernel_L4_M4_40) -.Lsgemm_kernel_L4_M4_22: +L(sgemm_kernel_L4_M4_22): KERNEL4x4_SUB KERNEL4x4_SUB @@ -1066,47 +1066,47 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M4_22 + bgt L(sgemm_kernel_L4_M4_22) -.Lsgemm_kernel_L4_M4_40: +L(sgemm_kernel_L4_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M4_100 + ble L(sgemm_kernel_L4_M4_100) -.Lsgemm_kernel_L4_M4_42: +L(sgemm_kernel_L4_M4_42): KERNEL4x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M4_42 + bgt L(sgemm_kernel_L4_M4_42) -.Lsgemm_kernel_L4_M4_100: +L(sgemm_kernel_L4_M4_100): SAVE4x4 -.Lsgemm_kernel_L4_M4_END: +L(sgemm_kernel_L4_M4_END): -.Lsgemm_kernel_L4_M2_BEGIN: +L(sgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L4_M1_BEGIN + ble L(sgemm_kernel_L4_M1_BEGIN) -.Lsgemm_kernel_L4_M2_20: +L(sgemm_kernel_L4_M2_20): INIT2x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L4_M2_40 + ble L(sgemm_kernel_L4_M2_40) -.Lsgemm_kernel_L4_M2_22: +L(sgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1119,43 +1119,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M2_22 + bgt L(sgemm_kernel_L4_M2_22) -.Lsgemm_kernel_L4_M2_40: +L(sgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M2_100 + ble L(sgemm_kernel_L4_M2_100) -.Lsgemm_kernel_L4_M2_42: +L(sgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M2_42 + bgt L(sgemm_kernel_L4_M2_42) -.Lsgemm_kernel_L4_M2_100: +L(sgemm_kernel_L4_M2_100): SAVE2x4 -.Lsgemm_kernel_L4_M2_END: +L(sgemm_kernel_L4_M2_END): -.Lsgemm_kernel_L4_M1_BEGIN: +L(sgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) -.Lsgemm_kernel_L4_M1_20: +L(sgemm_kernel_L4_M1_20): INIT1x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L4_M1_40 + ble L(sgemm_kernel_L4_M1_40) -.Lsgemm_kernel_L4_M1_22: +L(sgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1167,45 +1167,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M1_22 + bgt L(sgemm_kernel_L4_M1_22) -.Lsgemm_kernel_L4_M1_40: +L(sgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M1_100 + ble L(sgemm_kernel_L4_M1_100) -.Lsgemm_kernel_L4_M1_42: +L(sgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M1_42 + bgt L(sgemm_kernel_L4_M1_42) -.Lsgemm_kernel_L4_M1_100: +L(sgemm_kernel_L4_M1_100): SAVE1x4 -.Lsgemm_kernel_L4_END: +L(sgemm_kernel_L4_END): lsl temp, origK, #4 add origPB, origPB, temp // B = B + K * 4 * 4 subs counterJ, counterJ , #1 // j-- - bgt .Lsgemm_kernel_L4_BEGIN + bgt L(sgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Lsgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(sgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lsgemm_kernel_L999 + ble L(sgemm_kernel_L999) tst counterJ , #2 - ble .Lsgemm_kernel_L1_BEGIN + ble L(sgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1215,24 +1215,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lsgemm_kernel_L2_M4_BEGIN: +L(sgemm_kernel_L2_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI,#0 - ble .Lsgemm_kernel_L2_M2_BEGIN + ble L(sgemm_kernel_L2_M2_BEGIN) -.Lsgemm_kernel_L2_M4_20: +L(sgemm_kernel_L2_M4_20): INIT4x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M4_40 + ble L(sgemm_kernel_L2_M4_40) .align 5 -.Lsgemm_kernel_L2_M4_22: +L(sgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1244,50 +1244,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M4_22 + bgt L(sgemm_kernel_L2_M4_22) -.Lsgemm_kernel_L2_M4_40: +L(sgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M4_100 + ble L(sgemm_kernel_L2_M4_100) -.Lsgemm_kernel_L2_M4_42: +L(sgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M4_42 + bgt L(sgemm_kernel_L2_M4_42) -.Lsgemm_kernel_L2_M4_100: +L(sgemm_kernel_L2_M4_100): SAVE4x2 -.Lsgemm_kernel_L2_M4_END: +L(sgemm_kernel_L2_M4_END): subs counterI, counterI, #1 - bgt .Lsgemm_kernel_L2_M4_20 + bgt L(sgemm_kernel_L2_M4_20) -.Lsgemm_kernel_L2_M2_BEGIN: +L(sgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L2_M1_BEGIN + ble L(sgemm_kernel_L2_M1_BEGIN) -.Lsgemm_kernel_L2_M2_20: +L(sgemm_kernel_L2_M2_20): INIT2x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M2_40 + ble L(sgemm_kernel_L2_M2_40) -.Lsgemm_kernel_L2_M2_22: +L(sgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1300,43 +1300,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M2_22 + bgt L(sgemm_kernel_L2_M2_22) -.Lsgemm_kernel_L2_M2_40: +L(sgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M2_100 + ble L(sgemm_kernel_L2_M2_100) -.Lsgemm_kernel_L2_M2_42: +L(sgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M2_42 + bgt L(sgemm_kernel_L2_M2_42) -.Lsgemm_kernel_L2_M2_100: +L(sgemm_kernel_L2_M2_100): SAVE2x2 -.Lsgemm_kernel_L2_M2_END: +L(sgemm_kernel_L2_M2_END): -.Lsgemm_kernel_L2_M1_BEGIN: +L(sgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) -.Lsgemm_kernel_L2_M1_20: +L(sgemm_kernel_L2_M1_20): INIT1x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lsgemm_kernel_L2_M1_40 + ble L(sgemm_kernel_L2_M1_40) -.Lsgemm_kernel_L2_M1_22: +L(sgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1348,36 +1348,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M1_22 + bgt L(sgemm_kernel_L2_M1_22) -.Lsgemm_kernel_L2_M1_40: +L(sgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M1_100 + ble L(sgemm_kernel_L2_M1_100) -.Lsgemm_kernel_L2_M1_42: +L(sgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M1_42 + bgt L(sgemm_kernel_L2_M1_42) -.Lsgemm_kernel_L2_M1_100: +L(sgemm_kernel_L2_M1_100): SAVE1x2 -.Lsgemm_kernel_L2_END: +L(sgemm_kernel_L2_END): add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 /******************************************************************************/ -.Lsgemm_kernel_L1_BEGIN: +L(sgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lsgemm_kernel_L999 // done + ble L(sgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1387,24 +1387,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lsgemm_kernel_L1_M4_BEGIN: +L(sgemm_kernel_L1_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lsgemm_kernel_L1_M2_BEGIN + ble L(sgemm_kernel_L1_M2_BEGIN) -.Lsgemm_kernel_L1_M4_20: +L(sgemm_kernel_L1_M4_20): INIT4x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M4_40 + ble L(sgemm_kernel_L1_M4_40) .align 5 -.Lsgemm_kernel_L1_M4_22: +L(sgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1416,50 +1416,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M4_22 + bgt L(sgemm_kernel_L1_M4_22) -.Lsgemm_kernel_L1_M4_40: +L(sgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M4_100 + ble L(sgemm_kernel_L1_M4_100) -.Lsgemm_kernel_L1_M4_42: +L(sgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M4_42 + bgt L(sgemm_kernel_L1_M4_42) -.Lsgemm_kernel_L1_M4_100: +L(sgemm_kernel_L1_M4_100): SAVE4x1 -.Lsgemm_kernel_L1_M4_END: +L(sgemm_kernel_L1_M4_END): subs counterI, counterI, #1 - bgt .Lsgemm_kernel_L1_M4_20 + bgt L(sgemm_kernel_L1_M4_20) -.Lsgemm_kernel_L1_M2_BEGIN: +L(sgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L1_M1_BEGIN + ble L(sgemm_kernel_L1_M1_BEGIN) -.Lsgemm_kernel_L1_M2_20: +L(sgemm_kernel_L1_M2_20): INIT2x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M2_40 + ble L(sgemm_kernel_L1_M2_40) -.Lsgemm_kernel_L1_M2_22: +L(sgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1472,43 +1472,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M2_22 + bgt L(sgemm_kernel_L1_M2_22) -.Lsgemm_kernel_L1_M2_40: +L(sgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M2_100 + ble L(sgemm_kernel_L1_M2_100) -.Lsgemm_kernel_L1_M2_42: +L(sgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M2_42 + bgt L(sgemm_kernel_L1_M2_42) -.Lsgemm_kernel_L1_M2_100: +L(sgemm_kernel_L1_M2_100): SAVE2x1 -.Lsgemm_kernel_L1_M2_END: +L(sgemm_kernel_L1_M2_END): -.Lsgemm_kernel_L1_M1_BEGIN: +L(sgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) -.Lsgemm_kernel_L1_M1_20: +L(sgemm_kernel_L1_M1_20): INIT1x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M1_40 + ble L(sgemm_kernel_L1_M1_40) -.Lsgemm_kernel_L1_M1_22: +L(sgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1520,30 +1520,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M1_22 + bgt L(sgemm_kernel_L1_M1_22) -.Lsgemm_kernel_L1_M1_40: +L(sgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M1_100 + ble L(sgemm_kernel_L1_M1_100) -.Lsgemm_kernel_L1_M1_42: +L(sgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M1_42 + bgt L(sgemm_kernel_L1_M1_42) -.Lsgemm_kernel_L1_M1_100: +L(sgemm_kernel_L1_M1_100): SAVE1x1 -.Lsgemm_kernel_L1_END: +L(sgemm_kernel_L1_END): -.Lsgemm_kernel_L999: +L(sgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/sgemm_kernel_8x8.S b/kernel/arm64/sgemm_kernel_8x8.S index 6ba64dd355..4d73fb1e81 100644 --- a/kernel/arm64/sgemm_kernel_8x8.S +++ b/kernel/arm64/sgemm_kernel_8x8.S @@ -1263,7 +1263,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE -.Lsgemm_kernel_begin: +L(sgemm_kernel_begin): .align 5 add sp, sp, #-(11 * 16) @@ -1291,12 +1291,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Lsgemm_kernel_L4_BEGIN + ble L(sgemm_kernel_L4_BEGIN) /******************************************************************************/ /******************************************************************************/ -.Lsgemm_kernel_L8_BEGIN: +L(sgemm_kernel_L8_BEGIN): mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #3 @@ -1304,156 +1304,156 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lsgemm_kernel_L8_M8_BEGIN: +L(sgemm_kernel_L8_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lsgemm_kernel_L8_M4_BEGIN + ble L(sgemm_kernel_L8_M4_BEGIN) -.Lsgemm_kernel_L8_M8_20: +L(sgemm_kernel_L8_M8_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L8_M8_32 + blt L(sgemm_kernel_L8_M8_32) KERNEL8x8_I // do one in the K KERNEL8x8_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L8_M8_22a + ble L(sgemm_kernel_L8_M8_22a) .align 5 -.Lsgemm_kernel_L8_M8_22: +L(sgemm_kernel_L8_M8_22): KERNEL8x8_M1 KERNEL8x8_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M8_22 + bgt L(sgemm_kernel_L8_M8_22) -.Lsgemm_kernel_L8_M8_22a: +L(sgemm_kernel_L8_M8_22a): KERNEL8x8_M1 KERNEL8x8_E - b .Lsgemm_kernel_L8_M8_44 + b L(sgemm_kernel_L8_M8_44) -.Lsgemm_kernel_L8_M8_32: +L(sgemm_kernel_L8_M8_32): tst counterL, #1 - ble .Lsgemm_kernel_L8_M8_40 + ble L(sgemm_kernel_L8_M8_40) KERNEL8x8_I KERNEL8x8_E - b .Lsgemm_kernel_L8_M8_44 + b L(sgemm_kernel_L8_M8_44) -.Lsgemm_kernel_L8_M8_40: +L(sgemm_kernel_L8_M8_40): INIT8x8 -.Lsgemm_kernel_L8_M8_44: +L(sgemm_kernel_L8_M8_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L8_M8_100 + ble L(sgemm_kernel_L8_M8_100) -.Lsgemm_kernel_L8_M8_46: +L(sgemm_kernel_L8_M8_46): KERNEL8x8_SUB -.Lsgemm_kernel_L8_M8_100: +L(sgemm_kernel_L8_M8_100): SAVE8x8 -.Lsgemm_kernel_L8_M8_END: +L(sgemm_kernel_L8_M8_END): subs counterI, counterI, #1 - bne .Lsgemm_kernel_L8_M8_20 + bne L(sgemm_kernel_L8_M8_20) /******************************************************************************/ -.Lsgemm_kernel_L8_M4_BEGIN: +L(sgemm_kernel_L8_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L8_END + ble L(sgemm_kernel_L8_END) tst counterI, #4 - ble .Lsgemm_kernel_L8_M2_BEGIN + ble L(sgemm_kernel_L8_M2_BEGIN) -.Lsgemm_kernel_L8_M4_20: +L(sgemm_kernel_L8_M4_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L8_M4_32 + blt L(sgemm_kernel_L8_M4_32) KERNEL4x8_I // do one in the K KERNEL4x8_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L8_M4_22a + ble L(sgemm_kernel_L8_M4_22a) .align 5 -.Lsgemm_kernel_L8_M4_22: +L(sgemm_kernel_L8_M4_22): KERNEL4x8_M1 KERNEL4x8_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M4_22 + bgt L(sgemm_kernel_L8_M4_22) -.Lsgemm_kernel_L8_M4_22a: +L(sgemm_kernel_L8_M4_22a): KERNEL4x8_M1 KERNEL4x8_E - b .Lsgemm_kernel_L8_M4_44 + b L(sgemm_kernel_L8_M4_44) -.Lsgemm_kernel_L8_M4_32: +L(sgemm_kernel_L8_M4_32): tst counterL, #1 - ble .Lsgemm_kernel_L8_M4_40 + ble L(sgemm_kernel_L8_M4_40) KERNEL4x8_I KERNEL4x8_E - b .Lsgemm_kernel_L8_M4_44 + b L(sgemm_kernel_L8_M4_44) -.Lsgemm_kernel_L8_M4_40: +L(sgemm_kernel_L8_M4_40): INIT4x8 -.Lsgemm_kernel_L8_M4_44: +L(sgemm_kernel_L8_M4_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L8_M4_100 + ble L(sgemm_kernel_L8_M4_100) -.Lsgemm_kernel_L8_M4_46: +L(sgemm_kernel_L8_M4_46): KERNEL4x8_SUB -.Lsgemm_kernel_L8_M4_100: +L(sgemm_kernel_L8_M4_100): SAVE4x8 -.Lsgemm_kernel_L8_M4_END: +L(sgemm_kernel_L8_M4_END): /******************************************************************************/ -.Lsgemm_kernel_L8_M2_BEGIN: +L(sgemm_kernel_L8_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L8_END + ble L(sgemm_kernel_L8_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L8_M1_BEGIN + ble L(sgemm_kernel_L8_M1_BEGIN) -.Lsgemm_kernel_L8_M2_20: +L(sgemm_kernel_L8_M2_20): INIT2x8 @@ -1461,9 +1461,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L8_M2_40 + ble L(sgemm_kernel_L8_M2_40) -.Lsgemm_kernel_L8_M2_22: +L(sgemm_kernel_L8_M2_22): KERNEL2x8_SUB KERNEL2x8_SUB @@ -1476,35 +1476,35 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M2_22 + bgt L(sgemm_kernel_L8_M2_22) -.Lsgemm_kernel_L8_M2_40: +L(sgemm_kernel_L8_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L8_M2_100 + ble L(sgemm_kernel_L8_M2_100) -.Lsgemm_kernel_L8_M2_42: +L(sgemm_kernel_L8_M2_42): KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M2_42 + bgt L(sgemm_kernel_L8_M2_42) -.Lsgemm_kernel_L8_M2_100: +L(sgemm_kernel_L8_M2_100): SAVE2x8 -.Lsgemm_kernel_L8_M2_END: +L(sgemm_kernel_L8_M2_END): /******************************************************************************/ -.Lsgemm_kernel_L8_M1_BEGIN: +L(sgemm_kernel_L8_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L8_END + ble L(sgemm_kernel_L8_END) -.Lsgemm_kernel_L8_M1_20: +L(sgemm_kernel_L8_M1_20): INIT1x8 @@ -1512,9 +1512,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L8_M1_40 + ble L(sgemm_kernel_L8_M1_40) -.Lsgemm_kernel_L8_M1_22: +L(sgemm_kernel_L8_M1_22): KERNEL1x8_SUB KERNEL1x8_SUB KERNEL1x8_SUB @@ -1526,43 +1526,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M1_22 + bgt L(sgemm_kernel_L8_M1_22) -.Lsgemm_kernel_L8_M1_40: +L(sgemm_kernel_L8_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L8_M1_100 + ble L(sgemm_kernel_L8_M1_100) -.Lsgemm_kernel_L8_M1_42: +L(sgemm_kernel_L8_M1_42): KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M1_42 + bgt L(sgemm_kernel_L8_M1_42) -.Lsgemm_kernel_L8_M1_100: +L(sgemm_kernel_L8_M1_100): SAVE1x8 -.Lsgemm_kernel_L8_END: +L(sgemm_kernel_L8_END): lsl temp, origK, #5 // B = B + K * 4 * 8 add origPB, origPB, temp subs counterJ, counterJ , #1 // j-- - bgt .Lsgemm_kernel_L8_BEGIN + bgt L(sgemm_kernel_L8_BEGIN) /******************************************************************************/ /******************************************************************************/ -.Lsgemm_kernel_L4_BEGIN: +L(sgemm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #7 - ble .Lsgemm_kernel_L999 + ble L(sgemm_kernel_L999) tst counterJ , #4 - ble .Lsgemm_kernel_L2_BEGIN + ble L(sgemm_kernel_L2_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1572,156 +1572,156 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lsgemm_kernel_L4_M8_BEGIN: +L(sgemm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lsgemm_kernel_L4_M4_BEGIN + ble L(sgemm_kernel_L4_M4_BEGIN) -.Lsgemm_kernel_L4_M8_20: +L(sgemm_kernel_L4_M8_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L4_M8_32 + blt L(sgemm_kernel_L4_M8_32) KERNEL8x4_I // do one in the K KERNEL8x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L4_M8_22a + ble L(sgemm_kernel_L4_M8_22a) .align 5 -.Lsgemm_kernel_L4_M8_22: +L(sgemm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M8_22 + bgt L(sgemm_kernel_L4_M8_22) -.Lsgemm_kernel_L4_M8_22a: +L(sgemm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_E - b .Lsgemm_kernel_L4_M8_44 + b L(sgemm_kernel_L4_M8_44) -.Lsgemm_kernel_L4_M8_32: +L(sgemm_kernel_L4_M8_32): tst counterL, #1 - ble .Lsgemm_kernel_L4_M8_40 + ble L(sgemm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_E - b .Lsgemm_kernel_L4_M8_44 + b L(sgemm_kernel_L4_M8_44) -.Lsgemm_kernel_L4_M8_40: +L(sgemm_kernel_L4_M8_40): INIT8x4 -.Lsgemm_kernel_L4_M8_44: +L(sgemm_kernel_L4_M8_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L4_M8_100 + ble L(sgemm_kernel_L4_M8_100) -.Lsgemm_kernel_L4_M8_46: +L(sgemm_kernel_L4_M8_46): KERNEL8x4_SUB -.Lsgemm_kernel_L4_M8_100: +L(sgemm_kernel_L4_M8_100): SAVE8x4 -.Lsgemm_kernel_L4_M8_END: +L(sgemm_kernel_L4_M8_END): subs counterI, counterI, #1 - bne .Lsgemm_kernel_L4_M8_20 + bne L(sgemm_kernel_L4_M8_20) /******************************************************************************/ -.Lsgemm_kernel_L4_M4_BEGIN: +L(sgemm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #4 - ble .Lsgemm_kernel_L4_M2_BEGIN + ble L(sgemm_kernel_L4_M2_BEGIN) -.Lsgemm_kernel_L4_M4_20: +L(sgemm_kernel_L4_M4_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L4_M4_32 + blt L(sgemm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L4_M4_22a + ble L(sgemm_kernel_L4_M4_22a) .align 5 -.Lsgemm_kernel_L4_M4_22: +L(sgemm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M4_22 + bgt L(sgemm_kernel_L4_M4_22) -.Lsgemm_kernel_L4_M4_22a: +L(sgemm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lsgemm_kernel_L4_M4_44 + b L(sgemm_kernel_L4_M4_44) -.Lsgemm_kernel_L4_M4_32: +L(sgemm_kernel_L4_M4_32): tst counterL, #1 - ble .Lsgemm_kernel_L4_M4_40 + ble L(sgemm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lsgemm_kernel_L4_M4_44 + b L(sgemm_kernel_L4_M4_44) -.Lsgemm_kernel_L4_M4_40: +L(sgemm_kernel_L4_M4_40): INIT4x4 -.Lsgemm_kernel_L4_M4_44: +L(sgemm_kernel_L4_M4_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L4_M4_100 + ble L(sgemm_kernel_L4_M4_100) -.Lsgemm_kernel_L4_M4_46: +L(sgemm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lsgemm_kernel_L4_M4_100: +L(sgemm_kernel_L4_M4_100): SAVE4x4 -.Lsgemm_kernel_L4_M4_END: +L(sgemm_kernel_L4_M4_END): /******************************************************************************/ -.Lsgemm_kernel_L4_M2_BEGIN: +L(sgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L4_M1_BEGIN + ble L(sgemm_kernel_L4_M1_BEGIN) -.Lsgemm_kernel_L4_M2_20: +L(sgemm_kernel_L4_M2_20): INIT2x4 @@ -1729,9 +1729,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L4_M2_40 + ble L(sgemm_kernel_L4_M2_40) -.Lsgemm_kernel_L4_M2_22: +L(sgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1744,35 +1744,35 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M2_22 + bgt L(sgemm_kernel_L4_M2_22) -.Lsgemm_kernel_L4_M2_40: +L(sgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M2_100 + ble L(sgemm_kernel_L4_M2_100) -.Lsgemm_kernel_L4_M2_42: +L(sgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M2_42 + bgt L(sgemm_kernel_L4_M2_42) -.Lsgemm_kernel_L4_M2_100: +L(sgemm_kernel_L4_M2_100): SAVE2x4 -.Lsgemm_kernel_L4_M2_END: +L(sgemm_kernel_L4_M2_END): /******************************************************************************/ -.Lsgemm_kernel_L4_M1_BEGIN: +L(sgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) -.Lsgemm_kernel_L4_M1_20: +L(sgemm_kernel_L4_M1_20): INIT1x4 @@ -1780,9 +1780,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L4_M1_40 + ble L(sgemm_kernel_L4_M1_40) -.Lsgemm_kernel_L4_M1_22: +L(sgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1794,39 +1794,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M1_22 + bgt L(sgemm_kernel_L4_M1_22) -.Lsgemm_kernel_L4_M1_40: +L(sgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M1_100 + ble L(sgemm_kernel_L4_M1_100) -.Lsgemm_kernel_L4_M1_42: +L(sgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M1_42 + bgt L(sgemm_kernel_L4_M1_42) -.Lsgemm_kernel_L4_M1_100: +L(sgemm_kernel_L4_M1_100): SAVE1x4 -.Lsgemm_kernel_L4_END: +L(sgemm_kernel_L4_END): add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4 /******************************************************************************/ /******************************************************************************/ -.Lsgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(sgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lsgemm_kernel_L999 + ble L(sgemm_kernel_L999) tst counterJ , #2 - ble .Lsgemm_kernel_L1_BEGIN + ble L(sgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1836,14 +1836,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lsgemm_kernel_L2_M8_BEGIN: +L(sgemm_kernel_L2_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI,#0 - ble .Lsgemm_kernel_L2_M4_BEGIN + ble L(sgemm_kernel_L2_M4_BEGIN) -.Lsgemm_kernel_L2_M8_20: +L(sgemm_kernel_L2_M8_20): INIT8x2 @@ -1851,10 +1851,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M8_40 + ble L(sgemm_kernel_L2_M8_40) .align 5 -.Lsgemm_kernel_L2_M8_22: +L(sgemm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB KERNEL8x2_SUB @@ -1866,42 +1866,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M8_22 + bgt L(sgemm_kernel_L2_M8_22) -.Lsgemm_kernel_L2_M8_40: +L(sgemm_kernel_L2_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M8_100 + ble L(sgemm_kernel_L2_M8_100) -.Lsgemm_kernel_L2_M8_42: +L(sgemm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M8_42 + bgt L(sgemm_kernel_L2_M8_42) -.Lsgemm_kernel_L2_M8_100: +L(sgemm_kernel_L2_M8_100): SAVE8x2 -.Lsgemm_kernel_L2_M8_END: +L(sgemm_kernel_L2_M8_END): subs counterI, counterI, #1 - bgt .Lsgemm_kernel_L2_M8_20 + bgt L(sgemm_kernel_L2_M8_20) /******************************************************************************/ -.Lsgemm_kernel_L2_M4_BEGIN: +L(sgemm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) tst counterI, #4 - ble .Lsgemm_kernel_L2_M2_BEGIN + ble L(sgemm_kernel_L2_M2_BEGIN) -.Lsgemm_kernel_L2_M4_20: +L(sgemm_kernel_L2_M4_20): INIT4x2 @@ -1909,10 +1909,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M4_40 + ble L(sgemm_kernel_L2_M4_40) .align 5 -.Lsgemm_kernel_L2_M4_22: +L(sgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1924,39 +1924,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M4_22 + bgt L(sgemm_kernel_L2_M4_22) -.Lsgemm_kernel_L2_M4_40: +L(sgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M4_100 + ble L(sgemm_kernel_L2_M4_100) -.Lsgemm_kernel_L2_M4_42: +L(sgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M4_42 + bgt L(sgemm_kernel_L2_M4_42) -.Lsgemm_kernel_L2_M4_100: +L(sgemm_kernel_L2_M4_100): SAVE4x2 -.Lsgemm_kernel_L2_M4_END: +L(sgemm_kernel_L2_M4_END): /******************************************************************************/ -.Lsgemm_kernel_L2_M2_BEGIN: +L(sgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L2_M1_BEGIN + ble L(sgemm_kernel_L2_M1_BEGIN) -.Lsgemm_kernel_L2_M2_20: +L(sgemm_kernel_L2_M2_20): INIT2x2 @@ -1964,9 +1964,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M2_40 + ble L(sgemm_kernel_L2_M2_40) -.Lsgemm_kernel_L2_M2_22: +L(sgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1979,35 +1979,35 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M2_22 + bgt L(sgemm_kernel_L2_M2_22) -.Lsgemm_kernel_L2_M2_40: +L(sgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M2_100 + ble L(sgemm_kernel_L2_M2_100) -.Lsgemm_kernel_L2_M2_42: +L(sgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M2_42 + bgt L(sgemm_kernel_L2_M2_42) -.Lsgemm_kernel_L2_M2_100: +L(sgemm_kernel_L2_M2_100): SAVE2x2 -.Lsgemm_kernel_L2_M2_END: +L(sgemm_kernel_L2_M2_END): /******************************************************************************/ -.Lsgemm_kernel_L2_M1_BEGIN: +L(sgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) -.Lsgemm_kernel_L2_M1_20: +L(sgemm_kernel_L2_M1_20): INIT1x2 @@ -2015,9 +2015,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lsgemm_kernel_L2_M1_40 + ble L(sgemm_kernel_L2_M1_40) -.Lsgemm_kernel_L2_M1_22: +L(sgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -2029,37 +2029,37 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M1_22 + bgt L(sgemm_kernel_L2_M1_22) -.Lsgemm_kernel_L2_M1_40: +L(sgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M1_100 + ble L(sgemm_kernel_L2_M1_100) -.Lsgemm_kernel_L2_M1_42: +L(sgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M1_42 + bgt L(sgemm_kernel_L2_M1_42) -.Lsgemm_kernel_L2_M1_100: +L(sgemm_kernel_L2_M1_100): SAVE1x2 -.Lsgemm_kernel_L2_END: +L(sgemm_kernel_L2_END): add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 /******************************************************************************/ /******************************************************************************/ -.Lsgemm_kernel_L1_BEGIN: +L(sgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lsgemm_kernel_L999 // done + ble L(sgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -2069,14 +2069,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lsgemm_kernel_L1_M8_BEGIN: +L(sgemm_kernel_L1_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 cmp counterI, #0 - ble .Lsgemm_kernel_L1_M4_BEGIN + ble L(sgemm_kernel_L1_M4_BEGIN) -.Lsgemm_kernel_L1_M8_20: +L(sgemm_kernel_L1_M8_20): INIT8x1 @@ -2084,10 +2084,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M8_40 + ble L(sgemm_kernel_L1_M8_40) .align 5 -.Lsgemm_kernel_L1_M8_22: +L(sgemm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -2099,42 +2099,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M8_22 + bgt L(sgemm_kernel_L1_M8_22) -.Lsgemm_kernel_L1_M8_40: +L(sgemm_kernel_L1_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M8_100 + ble L(sgemm_kernel_L1_M8_100) -.Lsgemm_kernel_L1_M8_42: +L(sgemm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M8_42 + bgt L(sgemm_kernel_L1_M8_42) -.Lsgemm_kernel_L1_M8_100: +L(sgemm_kernel_L1_M8_100): SAVE8x1 -.Lsgemm_kernel_L1_M8_END: +L(sgemm_kernel_L1_M8_END): subs counterI, counterI, #1 - bgt .Lsgemm_kernel_L1_M8_20 + bgt L(sgemm_kernel_L1_M8_20) /******************************************************************************/ -.Lsgemm_kernel_L1_M4_BEGIN: +L(sgemm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) tst counterI, #4 - ble .Lsgemm_kernel_L1_M2_BEGIN + ble L(sgemm_kernel_L1_M2_BEGIN) -.Lsgemm_kernel_L1_M4_20: +L(sgemm_kernel_L1_M4_20): INIT4x1 @@ -2142,10 +2142,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M4_40 + ble L(sgemm_kernel_L1_M4_40) .align 5 -.Lsgemm_kernel_L1_M4_22: +L(sgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -2157,39 +2157,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M4_22 + bgt L(sgemm_kernel_L1_M4_22) -.Lsgemm_kernel_L1_M4_40: +L(sgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M4_100 + ble L(sgemm_kernel_L1_M4_100) -.Lsgemm_kernel_L1_M4_42: +L(sgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M4_42 + bgt L(sgemm_kernel_L1_M4_42) -.Lsgemm_kernel_L1_M4_100: +L(sgemm_kernel_L1_M4_100): SAVE4x1 -.Lsgemm_kernel_L1_M4_END: +L(sgemm_kernel_L1_M4_END): /******************************************************************************/ -.Lsgemm_kernel_L1_M2_BEGIN: +L(sgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L1_M1_BEGIN + ble L(sgemm_kernel_L1_M1_BEGIN) -.Lsgemm_kernel_L1_M2_20: +L(sgemm_kernel_L1_M2_20): INIT2x1 @@ -2197,9 +2197,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M2_40 + ble L(sgemm_kernel_L1_M2_40) -.Lsgemm_kernel_L1_M2_22: +L(sgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -2212,35 +2212,35 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M2_22 + bgt L(sgemm_kernel_L1_M2_22) -.Lsgemm_kernel_L1_M2_40: +L(sgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M2_100 + ble L(sgemm_kernel_L1_M2_100) -.Lsgemm_kernel_L1_M2_42: +L(sgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M2_42 + bgt L(sgemm_kernel_L1_M2_42) -.Lsgemm_kernel_L1_M2_100: +L(sgemm_kernel_L1_M2_100): SAVE2x1 -.Lsgemm_kernel_L1_M2_END: +L(sgemm_kernel_L1_M2_END): /******************************************************************************/ -.Lsgemm_kernel_L1_M1_BEGIN: +L(sgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) -.Lsgemm_kernel_L1_M1_20: +L(sgemm_kernel_L1_M1_20): INIT1x1 @@ -2248,9 +2248,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M1_40 + ble L(sgemm_kernel_L1_M1_40) -.Lsgemm_kernel_L1_M1_22: +L(sgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -2262,30 +2262,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M1_22 + bgt L(sgemm_kernel_L1_M1_22) -.Lsgemm_kernel_L1_M1_40: +L(sgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M1_100 + ble L(sgemm_kernel_L1_M1_100) -.Lsgemm_kernel_L1_M1_42: +L(sgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M1_42 + bgt L(sgemm_kernel_L1_M1_42) -.Lsgemm_kernel_L1_M1_100: +L(sgemm_kernel_L1_M1_100): SAVE1x1 -.Lsgemm_kernel_L1_END: +L(sgemm_kernel_L1_END): /******************************************************************************/ -.Lsgemm_kernel_L999: +L(sgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/sgemm_kernel_8x8_cortexa53.S b/kernel/arm64/sgemm_kernel_8x8_cortexa53.S index 628a928ca7..68d933be64 100644 --- a/kernel/arm64/sgemm_kernel_8x8_cortexa53.S +++ b/kernel/arm64/sgemm_kernel_8x8_cortexa53.S @@ -1230,7 +1230,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE -.Lsgemm_kernel_begin: +L(sgemm_kernel_begin): .align 5 add sp, sp, #-(11 * 16) @@ -1258,12 +1258,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Lsgemm_kernel_L4_BEGIN + ble L(sgemm_kernel_L4_BEGIN) /******************************************************************************/ /******************************************************************************/ -.Lsgemm_kernel_L8_BEGIN: +L(sgemm_kernel_L8_BEGIN): mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #3 @@ -1271,20 +1271,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lsgemm_kernel_L8_M8_BEGIN: +L(sgemm_kernel_L8_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lsgemm_kernel_L8_M4_BEGIN + ble L(sgemm_kernel_L8_M4_BEGIN) -.Lsgemm_kernel_L8_M8_20: +L(sgemm_kernel_L8_M8_20): mov pB, origPB asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 16 to do? - blt .Lsgemm_kernel_L8_M8_32 + blt L(sgemm_kernel_L8_M8_32) KERNEL8x8_I // do one in the K KERNEL8x8_M2 // do another in the K @@ -1296,10 +1296,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x8_M2 subs counterL, counterL, #2 - ble .Lsgemm_kernel_L8_M8_22a + ble L(sgemm_kernel_L8_M8_22a) .align 5 -.Lsgemm_kernel_L8_M8_22: +L(sgemm_kernel_L8_M8_22): KERNEL8x8_M1 KERNEL8x8_M2 @@ -1311,9 +1311,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x8_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M8_22 + bgt L(sgemm_kernel_L8_M8_22) -.Lsgemm_kernel_L8_M8_22a: +L(sgemm_kernel_L8_M8_22a): KERNEL8x8_M1 KERNEL8x8_M2 @@ -1324,12 +1324,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x8_M1 KERNEL8x8_E - b .Lsgemm_kernel_L8_M8_44 + b L(sgemm_kernel_L8_M8_44) -.Lsgemm_kernel_L8_M8_32: +L(sgemm_kernel_L8_M8_32): tst counterL, #1 - ble .Lsgemm_kernel_L8_M8_40 + ble L(sgemm_kernel_L8_M8_40) KERNEL8x8_I KERNEL8x8_M2 @@ -1340,114 +1340,114 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x8_M1 KERNEL8x8_E - b .Lsgemm_kernel_L8_M8_44 + b L(sgemm_kernel_L8_M8_44) -.Lsgemm_kernel_L8_M8_40: +L(sgemm_kernel_L8_M8_40): INIT8x8 -.Lsgemm_kernel_L8_M8_44: +L(sgemm_kernel_L8_M8_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L8_M8_100 + ble L(sgemm_kernel_L8_M8_100) -.Lsgemm_kernel_L8_M8_46: +L(sgemm_kernel_L8_M8_46): KERNEL8x8_SUB subs counterL, counterL, 1 - bgt .Lsgemm_kernel_L8_M8_46 + bgt L(sgemm_kernel_L8_M8_46) -.Lsgemm_kernel_L8_M8_100: +L(sgemm_kernel_L8_M8_100): SAVE8x8 -.Lsgemm_kernel_L8_M8_END: +L(sgemm_kernel_L8_M8_END): subs counterI, counterI, #1 - bne .Lsgemm_kernel_L8_M8_20 + bne L(sgemm_kernel_L8_M8_20) /******************************************************************************/ -.Lsgemm_kernel_L8_M4_BEGIN: +L(sgemm_kernel_L8_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L8_END + ble L(sgemm_kernel_L8_END) tst counterI, #4 - ble .Lsgemm_kernel_L8_M2_BEGIN + ble L(sgemm_kernel_L8_M2_BEGIN) -.Lsgemm_kernel_L8_M4_20: +L(sgemm_kernel_L8_M4_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L8_M4_32 + blt L(sgemm_kernel_L8_M4_32) KERNEL4x8_I // do one in the K KERNEL4x8_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L8_M4_22a + ble L(sgemm_kernel_L8_M4_22a) .align 5 -.Lsgemm_kernel_L8_M4_22: +L(sgemm_kernel_L8_M4_22): KERNEL4x8_M1 KERNEL4x8_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M4_22 + bgt L(sgemm_kernel_L8_M4_22) -.Lsgemm_kernel_L8_M4_22a: +L(sgemm_kernel_L8_M4_22a): KERNEL4x8_M1 KERNEL4x8_E - b .Lsgemm_kernel_L8_M4_44 + b L(sgemm_kernel_L8_M4_44) -.Lsgemm_kernel_L8_M4_32: +L(sgemm_kernel_L8_M4_32): tst counterL, #1 - ble .Lsgemm_kernel_L8_M4_40 + ble L(sgemm_kernel_L8_M4_40) KERNEL4x8_I KERNEL4x8_E - b .Lsgemm_kernel_L8_M4_44 + b L(sgemm_kernel_L8_M4_44) -.Lsgemm_kernel_L8_M4_40: +L(sgemm_kernel_L8_M4_40): INIT4x8 -.Lsgemm_kernel_L8_M4_44: +L(sgemm_kernel_L8_M4_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L8_M4_100 + ble L(sgemm_kernel_L8_M4_100) -.Lsgemm_kernel_L8_M4_46: +L(sgemm_kernel_L8_M4_46): KERNEL4x8_SUB -.Lsgemm_kernel_L8_M4_100: +L(sgemm_kernel_L8_M4_100): SAVE4x8 -.Lsgemm_kernel_L8_M4_END: +L(sgemm_kernel_L8_M4_END): /******************************************************************************/ -.Lsgemm_kernel_L8_M2_BEGIN: +L(sgemm_kernel_L8_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L8_END + ble L(sgemm_kernel_L8_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L8_M1_BEGIN + ble L(sgemm_kernel_L8_M1_BEGIN) -.Lsgemm_kernel_L8_M2_20: +L(sgemm_kernel_L8_M2_20): INIT2x8 @@ -1455,9 +1455,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L8_M2_40 + ble L(sgemm_kernel_L8_M2_40) -.Lsgemm_kernel_L8_M2_22: +L(sgemm_kernel_L8_M2_22): KERNEL2x8_SUB KERNEL2x8_SUB @@ -1470,35 +1470,35 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M2_22 + bgt L(sgemm_kernel_L8_M2_22) -.Lsgemm_kernel_L8_M2_40: +L(sgemm_kernel_L8_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L8_M2_100 + ble L(sgemm_kernel_L8_M2_100) -.Lsgemm_kernel_L8_M2_42: +L(sgemm_kernel_L8_M2_42): KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M2_42 + bgt L(sgemm_kernel_L8_M2_42) -.Lsgemm_kernel_L8_M2_100: +L(sgemm_kernel_L8_M2_100): SAVE2x8 -.Lsgemm_kernel_L8_M2_END: +L(sgemm_kernel_L8_M2_END): /******************************************************************************/ -.Lsgemm_kernel_L8_M1_BEGIN: +L(sgemm_kernel_L8_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L8_END + ble L(sgemm_kernel_L8_END) -.Lsgemm_kernel_L8_M1_20: +L(sgemm_kernel_L8_M1_20): INIT1x8 @@ -1506,9 +1506,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L8_M1_40 + ble L(sgemm_kernel_L8_M1_40) -.Lsgemm_kernel_L8_M1_22: +L(sgemm_kernel_L8_M1_22): KERNEL1x8_SUB KERNEL1x8_SUB KERNEL1x8_SUB @@ -1520,43 +1520,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M1_22 + bgt L(sgemm_kernel_L8_M1_22) -.Lsgemm_kernel_L8_M1_40: +L(sgemm_kernel_L8_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L8_M1_100 + ble L(sgemm_kernel_L8_M1_100) -.Lsgemm_kernel_L8_M1_42: +L(sgemm_kernel_L8_M1_42): KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_M1_42 + bgt L(sgemm_kernel_L8_M1_42) -.Lsgemm_kernel_L8_M1_100: +L(sgemm_kernel_L8_M1_100): SAVE1x8 -.Lsgemm_kernel_L8_END: +L(sgemm_kernel_L8_END): lsl temp, origK, #5 // B = B + K * 4 * 8 add origPB, origPB, temp subs counterJ, counterJ , #1 // j-- - bgt .Lsgemm_kernel_L8_BEGIN + bgt L(sgemm_kernel_L8_BEGIN) /******************************************************************************/ /******************************************************************************/ -.Lsgemm_kernel_L4_BEGIN: +L(sgemm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #7 - ble .Lsgemm_kernel_L999 + ble L(sgemm_kernel_L999) tst counterJ , #4 - ble .Lsgemm_kernel_L2_BEGIN + ble L(sgemm_kernel_L2_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1566,156 +1566,156 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lsgemm_kernel_L4_M8_BEGIN: +L(sgemm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lsgemm_kernel_L4_M4_BEGIN + ble L(sgemm_kernel_L4_M4_BEGIN) -.Lsgemm_kernel_L4_M8_20: +L(sgemm_kernel_L4_M8_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L4_M8_32 + blt L(sgemm_kernel_L4_M8_32) KERNEL8x4_I // do one in the K KERNEL8x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L4_M8_22a + ble L(sgemm_kernel_L4_M8_22a) .align 5 -.Lsgemm_kernel_L4_M8_22: +L(sgemm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M8_22 + bgt L(sgemm_kernel_L4_M8_22) -.Lsgemm_kernel_L4_M8_22a: +L(sgemm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_E - b .Lsgemm_kernel_L4_M8_44 + b L(sgemm_kernel_L4_M8_44) -.Lsgemm_kernel_L4_M8_32: +L(sgemm_kernel_L4_M8_32): tst counterL, #1 - ble .Lsgemm_kernel_L4_M8_40 + ble L(sgemm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_E - b .Lsgemm_kernel_L4_M8_44 + b L(sgemm_kernel_L4_M8_44) -.Lsgemm_kernel_L4_M8_40: +L(sgemm_kernel_L4_M8_40): INIT8x4 -.Lsgemm_kernel_L4_M8_44: +L(sgemm_kernel_L4_M8_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L4_M8_100 + ble L(sgemm_kernel_L4_M8_100) -.Lsgemm_kernel_L4_M8_46: +L(sgemm_kernel_L4_M8_46): KERNEL8x4_SUB -.Lsgemm_kernel_L4_M8_100: +L(sgemm_kernel_L4_M8_100): SAVE8x4 -.Lsgemm_kernel_L4_M8_END: +L(sgemm_kernel_L4_M8_END): subs counterI, counterI, #1 - bne .Lsgemm_kernel_L4_M8_20 + bne L(sgemm_kernel_L4_M8_20) /******************************************************************************/ -.Lsgemm_kernel_L4_M4_BEGIN: +L(sgemm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #4 - ble .Lsgemm_kernel_L4_M2_BEGIN + ble L(sgemm_kernel_L4_M2_BEGIN) -.Lsgemm_kernel_L4_M4_20: +L(sgemm_kernel_L4_M4_20): mov pB, origPB asr counterL , origK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L4_M4_32 + blt L(sgemm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lsgemm_kernel_L4_M4_22a + ble L(sgemm_kernel_L4_M4_22a) .align 5 -.Lsgemm_kernel_L4_M4_22: +L(sgemm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M4_22 + bgt L(sgemm_kernel_L4_M4_22) -.Lsgemm_kernel_L4_M4_22a: +L(sgemm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lsgemm_kernel_L4_M4_44 + b L(sgemm_kernel_L4_M4_44) -.Lsgemm_kernel_L4_M4_32: +L(sgemm_kernel_L4_M4_32): tst counterL, #1 - ble .Lsgemm_kernel_L4_M4_40 + ble L(sgemm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lsgemm_kernel_L4_M4_44 + b L(sgemm_kernel_L4_M4_44) -.Lsgemm_kernel_L4_M4_40: +L(sgemm_kernel_L4_M4_40): INIT4x4 -.Lsgemm_kernel_L4_M4_44: +L(sgemm_kernel_L4_M4_44): ands counterL , origK, #1 - ble .Lsgemm_kernel_L4_M4_100 + ble L(sgemm_kernel_L4_M4_100) -.Lsgemm_kernel_L4_M4_46: +L(sgemm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lsgemm_kernel_L4_M4_100: +L(sgemm_kernel_L4_M4_100): SAVE4x4 -.Lsgemm_kernel_L4_M4_END: +L(sgemm_kernel_L4_M4_END): /******************************************************************************/ -.Lsgemm_kernel_L4_M2_BEGIN: +L(sgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L4_M1_BEGIN + ble L(sgemm_kernel_L4_M1_BEGIN) -.Lsgemm_kernel_L4_M2_20: +L(sgemm_kernel_L4_M2_20): INIT2x4 @@ -1723,9 +1723,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L4_M2_40 + ble L(sgemm_kernel_L4_M2_40) -.Lsgemm_kernel_L4_M2_22: +L(sgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1738,35 +1738,35 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M2_22 + bgt L(sgemm_kernel_L4_M2_22) -.Lsgemm_kernel_L4_M2_40: +L(sgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M2_100 + ble L(sgemm_kernel_L4_M2_100) -.Lsgemm_kernel_L4_M2_42: +L(sgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M2_42 + bgt L(sgemm_kernel_L4_M2_42) -.Lsgemm_kernel_L4_M2_100: +L(sgemm_kernel_L4_M2_100): SAVE2x4 -.Lsgemm_kernel_L4_M2_END: +L(sgemm_kernel_L4_M2_END): /******************************************************************************/ -.Lsgemm_kernel_L4_M1_BEGIN: +L(sgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L4_END + ble L(sgemm_kernel_L4_END) -.Lsgemm_kernel_L4_M1_20: +L(sgemm_kernel_L4_M1_20): INIT1x4 @@ -1774,9 +1774,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L4_M1_40 + ble L(sgemm_kernel_L4_M1_40) -.Lsgemm_kernel_L4_M1_22: +L(sgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1788,39 +1788,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M1_22 + bgt L(sgemm_kernel_L4_M1_22) -.Lsgemm_kernel_L4_M1_40: +L(sgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L4_M1_100 + ble L(sgemm_kernel_L4_M1_100) -.Lsgemm_kernel_L4_M1_42: +L(sgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_M1_42 + bgt L(sgemm_kernel_L4_M1_42) -.Lsgemm_kernel_L4_M1_100: +L(sgemm_kernel_L4_M1_100): SAVE1x4 -.Lsgemm_kernel_L4_END: +L(sgemm_kernel_L4_END): add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4 /******************************************************************************/ /******************************************************************************/ -.Lsgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(sgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lsgemm_kernel_L999 + ble L(sgemm_kernel_L999) tst counterJ , #2 - ble .Lsgemm_kernel_L1_BEGIN + ble L(sgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1830,14 +1830,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lsgemm_kernel_L2_M8_BEGIN: +L(sgemm_kernel_L2_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI,#0 - ble .Lsgemm_kernel_L2_M4_BEGIN + ble L(sgemm_kernel_L2_M4_BEGIN) -.Lsgemm_kernel_L2_M8_20: +L(sgemm_kernel_L2_M8_20): INIT8x2 @@ -1845,10 +1845,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M8_40 + ble L(sgemm_kernel_L2_M8_40) .align 5 -.Lsgemm_kernel_L2_M8_22: +L(sgemm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB KERNEL8x2_SUB @@ -1860,42 +1860,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M8_22 + bgt L(sgemm_kernel_L2_M8_22) -.Lsgemm_kernel_L2_M8_40: +L(sgemm_kernel_L2_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M8_100 + ble L(sgemm_kernel_L2_M8_100) -.Lsgemm_kernel_L2_M8_42: +L(sgemm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M8_42 + bgt L(sgemm_kernel_L2_M8_42) -.Lsgemm_kernel_L2_M8_100: +L(sgemm_kernel_L2_M8_100): SAVE8x2 -.Lsgemm_kernel_L2_M8_END: +L(sgemm_kernel_L2_M8_END): subs counterI, counterI, #1 - bgt .Lsgemm_kernel_L2_M8_20 + bgt L(sgemm_kernel_L2_M8_20) /******************************************************************************/ -.Lsgemm_kernel_L2_M4_BEGIN: +L(sgemm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) tst counterI, #4 - ble .Lsgemm_kernel_L2_M2_BEGIN + ble L(sgemm_kernel_L2_M2_BEGIN) -.Lsgemm_kernel_L2_M4_20: +L(sgemm_kernel_L2_M4_20): INIT4x2 @@ -1903,10 +1903,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M4_40 + ble L(sgemm_kernel_L2_M4_40) .align 5 -.Lsgemm_kernel_L2_M4_22: +L(sgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1918,39 +1918,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M4_22 + bgt L(sgemm_kernel_L2_M4_22) -.Lsgemm_kernel_L2_M4_40: +L(sgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M4_100 + ble L(sgemm_kernel_L2_M4_100) -.Lsgemm_kernel_L2_M4_42: +L(sgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M4_42 + bgt L(sgemm_kernel_L2_M4_42) -.Lsgemm_kernel_L2_M4_100: +L(sgemm_kernel_L2_M4_100): SAVE4x2 -.Lsgemm_kernel_L2_M4_END: +L(sgemm_kernel_L2_M4_END): /******************************************************************************/ -.Lsgemm_kernel_L2_M2_BEGIN: +L(sgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L2_M1_BEGIN + ble L(sgemm_kernel_L2_M1_BEGIN) -.Lsgemm_kernel_L2_M2_20: +L(sgemm_kernel_L2_M2_20): INIT2x2 @@ -1958,9 +1958,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lsgemm_kernel_L2_M2_40 + ble L(sgemm_kernel_L2_M2_40) -.Lsgemm_kernel_L2_M2_22: +L(sgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1973,35 +1973,35 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M2_22 + bgt L(sgemm_kernel_L2_M2_22) -.Lsgemm_kernel_L2_M2_40: +L(sgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M2_100 + ble L(sgemm_kernel_L2_M2_100) -.Lsgemm_kernel_L2_M2_42: +L(sgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M2_42 + bgt L(sgemm_kernel_L2_M2_42) -.Lsgemm_kernel_L2_M2_100: +L(sgemm_kernel_L2_M2_100): SAVE2x2 -.Lsgemm_kernel_L2_M2_END: +L(sgemm_kernel_L2_M2_END): /******************************************************************************/ -.Lsgemm_kernel_L2_M1_BEGIN: +L(sgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L2_END + ble L(sgemm_kernel_L2_END) -.Lsgemm_kernel_L2_M1_20: +L(sgemm_kernel_L2_M1_20): INIT1x2 @@ -2009,9 +2009,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lsgemm_kernel_L2_M1_40 + ble L(sgemm_kernel_L2_M1_40) -.Lsgemm_kernel_L2_M1_22: +L(sgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -2023,37 +2023,37 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M1_22 + bgt L(sgemm_kernel_L2_M1_22) -.Lsgemm_kernel_L2_M1_40: +L(sgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L2_M1_100 + ble L(sgemm_kernel_L2_M1_100) -.Lsgemm_kernel_L2_M1_42: +L(sgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_M1_42 + bgt L(sgemm_kernel_L2_M1_42) -.Lsgemm_kernel_L2_M1_100: +L(sgemm_kernel_L2_M1_100): SAVE1x2 -.Lsgemm_kernel_L2_END: +L(sgemm_kernel_L2_END): add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 /******************************************************************************/ /******************************************************************************/ -.Lsgemm_kernel_L1_BEGIN: +L(sgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lsgemm_kernel_L999 // done + ble L(sgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -2063,14 +2063,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lsgemm_kernel_L1_M8_BEGIN: +L(sgemm_kernel_L1_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 cmp counterI, #0 - ble .Lsgemm_kernel_L1_M4_BEGIN + ble L(sgemm_kernel_L1_M4_BEGIN) -.Lsgemm_kernel_L1_M8_20: +L(sgemm_kernel_L1_M8_20): INIT8x1 @@ -2078,10 +2078,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M8_40 + ble L(sgemm_kernel_L1_M8_40) .align 5 -.Lsgemm_kernel_L1_M8_22: +L(sgemm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -2093,42 +2093,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M8_22 + bgt L(sgemm_kernel_L1_M8_22) -.Lsgemm_kernel_L1_M8_40: +L(sgemm_kernel_L1_M8_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M8_100 + ble L(sgemm_kernel_L1_M8_100) -.Lsgemm_kernel_L1_M8_42: +L(sgemm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M8_42 + bgt L(sgemm_kernel_L1_M8_42) -.Lsgemm_kernel_L1_M8_100: +L(sgemm_kernel_L1_M8_100): SAVE8x1 -.Lsgemm_kernel_L1_M8_END: +L(sgemm_kernel_L1_M8_END): subs counterI, counterI, #1 - bgt .Lsgemm_kernel_L1_M8_20 + bgt L(sgemm_kernel_L1_M8_20) /******************************************************************************/ -.Lsgemm_kernel_L1_M4_BEGIN: +L(sgemm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) tst counterI, #4 - ble .Lsgemm_kernel_L1_M2_BEGIN + ble L(sgemm_kernel_L1_M2_BEGIN) -.Lsgemm_kernel_L1_M4_20: +L(sgemm_kernel_L1_M4_20): INIT4x1 @@ -2136,10 +2136,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M4_40 + ble L(sgemm_kernel_L1_M4_40) .align 5 -.Lsgemm_kernel_L1_M4_22: +L(sgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -2151,39 +2151,39 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M4_22 + bgt L(sgemm_kernel_L1_M4_22) -.Lsgemm_kernel_L1_M4_40: +L(sgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M4_100 + ble L(sgemm_kernel_L1_M4_100) -.Lsgemm_kernel_L1_M4_42: +L(sgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M4_42 + bgt L(sgemm_kernel_L1_M4_42) -.Lsgemm_kernel_L1_M4_100: +L(sgemm_kernel_L1_M4_100): SAVE4x1 -.Lsgemm_kernel_L1_M4_END: +L(sgemm_kernel_L1_M4_END): /******************************************************************************/ -.Lsgemm_kernel_L1_M2_BEGIN: +L(sgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lsgemm_kernel_L1_M1_BEGIN + ble L(sgemm_kernel_L1_M1_BEGIN) -.Lsgemm_kernel_L1_M2_20: +L(sgemm_kernel_L1_M2_20): INIT2x1 @@ -2191,9 +2191,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M2_40 + ble L(sgemm_kernel_L1_M2_40) -.Lsgemm_kernel_L1_M2_22: +L(sgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -2206,35 +2206,35 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M2_22 + bgt L(sgemm_kernel_L1_M2_22) -.Lsgemm_kernel_L1_M2_40: +L(sgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M2_100 + ble L(sgemm_kernel_L1_M2_100) -.Lsgemm_kernel_L1_M2_42: +L(sgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M2_42 + bgt L(sgemm_kernel_L1_M2_42) -.Lsgemm_kernel_L1_M2_100: +L(sgemm_kernel_L1_M2_100): SAVE2x1 -.Lsgemm_kernel_L1_M2_END: +L(sgemm_kernel_L1_M2_END): /******************************************************************************/ -.Lsgemm_kernel_L1_M1_BEGIN: +L(sgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lsgemm_kernel_L1_END + ble L(sgemm_kernel_L1_END) -.Lsgemm_kernel_L1_M1_20: +L(sgemm_kernel_L1_M1_20): INIT1x1 @@ -2242,9 +2242,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lsgemm_kernel_L1_M1_40 + ble L(sgemm_kernel_L1_M1_40) -.Lsgemm_kernel_L1_M1_22: +L(sgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -2256,30 +2256,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M1_22 + bgt L(sgemm_kernel_L1_M1_22) -.Lsgemm_kernel_L1_M1_40: +L(sgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lsgemm_kernel_L1_M1_100 + ble L(sgemm_kernel_L1_M1_100) -.Lsgemm_kernel_L1_M1_42: +L(sgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_M1_42 + bgt L(sgemm_kernel_L1_M1_42) -.Lsgemm_kernel_L1_M1_100: +L(sgemm_kernel_L1_M1_100): SAVE1x1 -.Lsgemm_kernel_L1_END: +L(sgemm_kernel_L1_END): /******************************************************************************/ -.Lsgemm_kernel_L999: +L(sgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/sgemm_kernel_inter_sve_v2x8.S b/kernel/arm64/sgemm_kernel_inter_sve_v2x8.S index f8c4aa61c1..247581c218 100644 --- a/kernel/arm64/sgemm_kernel_inter_sve_v2x8.S +++ b/kernel/arm64/sgemm_kernel_inter_sve_v2x8.S @@ -855,38 +855,38 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Lsgemm_kernel_L4_BEGIN + ble L(sgemm_kernel_L4_BEGIN) /******************************************************************************/ /* Repeat this as long as there are 8 left in N */ .align 5 -.Lsgemm_kernel_L8_BEGIN: +L(sgemm_kernel_L8_BEGIN): mov pCRow0, pC add pC, pC, LDC, lsl #3 // add 8 x LDC mov pA1, origPA // pA1 = start of A array -.Lsgemm_kernel_L8_Mv2_BEGIN: +L(sgemm_kernel_L8_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 // Check if M < 2*SVE_LEN - blt .Lsgemm_kernel_L8_Mv1_BEGIN + blt L(sgemm_kernel_L8_Mv1_BEGIN) mov counterI, origM /* Until we have at least 2*SVE_LEN iters left in M, we do them with V2*8 kernel */ .align 5 -.Lsgemm_kernel_L8_Mv2_20: +L(sgemm_kernel_L8_Mv2_20): mov pB, origPB INITv2x8 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L8_Mv2_32 + blt L(sgemm_kernel_L8_Mv2_32) KERNELv2x8_I KERNELv2x8_M2 @@ -898,10 +898,10 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lsgemm_kernel_L8_Mv2_22a + ble L(sgemm_kernel_L8_Mv2_22a) .align 5 -.Lsgemm_kernel_L8_Mv2_22: +L(sgemm_kernel_L8_Mv2_22): KERNELv2x8_M1 KERNELv2x8_M2 @@ -913,10 +913,10 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_Mv2_22 + bgt L(sgemm_kernel_L8_Mv2_22) .align 5 -.Lsgemm_kernel_L8_Mv2_22a: +L(sgemm_kernel_L8_Mv2_22a): KERNELv2x8_M1 KERNELv2x8_M2 @@ -927,13 +927,13 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M1 KERNELv2x8_E - b .Lsgemm_kernel_L8_Mv2_44 + b L(sgemm_kernel_L8_Mv2_44) .align 5 -.Lsgemm_kernel_L8_Mv2_32: +L(sgemm_kernel_L8_Mv2_32): tst counterL, #1 - ble .Lsgemm_kernel_L8_Mv2_40 + ble L(sgemm_kernel_L8_Mv2_40) KERNELv2x8_I KERNELv2x8_M2 @@ -945,54 +945,54 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_E - b .Lsgemm_kernel_L8_Mv2_44 + b L(sgemm_kernel_L8_Mv2_44) -.Lsgemm_kernel_L8_Mv2_40: +L(sgemm_kernel_L8_Mv2_40): INITv2x8 -.Lsgemm_kernel_L8_Mv2_44: +L(sgemm_kernel_L8_Mv2_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L8_Mv2_100 + ble L(sgemm_kernel_L8_Mv2_100) .align 5 -.Lsgemm_kernel_L8_Mv2_46: +L(sgemm_kernel_L8_Mv2_46): KERNELv2x8_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L8_Mv2_46 + bne L(sgemm_kernel_L8_Mv2_46) -.Lsgemm_kernel_L8_Mv2_100: +L(sgemm_kernel_L8_Mv2_100): SAVEv2x8 -.Lsgemm_kernel_L8_Mv2_END: +L(sgemm_kernel_L8_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Lsgemm_kernel_L8_Mv2_20 + bge L(sgemm_kernel_L8_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Lsgemm_kernel_L8_END + beq L(sgemm_kernel_L8_END) ////////////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x8 kernel. -.Lsgemm_kernel_L8_Mv1_BEGIN: +L(sgemm_kernel_L8_Mv1_BEGIN): whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension .align 5 -.Lsgemm_kernel_L8_Mv1_20: +L(sgemm_kernel_L8_Mv1_20): mov pB, origPB INITv1x8 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L8_Mv1_32 + blt L(sgemm_kernel_L8_Mv1_32) KERNELv1x8_I KERNELv1x8_M2 @@ -1004,10 +1004,10 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lsgemm_kernel_L8_Mv1_22a + ble L(sgemm_kernel_L8_Mv1_22a) .align 5 -.Lsgemm_kernel_L8_Mv1_22: +L(sgemm_kernel_L8_Mv1_22): KERNELv1x8_M1 KERNELv1x8_M2 @@ -1019,10 +1019,10 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_Mv1_22 + bgt L(sgemm_kernel_L8_Mv1_22) .align 5 -.Lsgemm_kernel_L8_Mv1_22a: +L(sgemm_kernel_L8_Mv1_22a): KERNELv1x8_M1 KERNELv1x8_M2 @@ -1033,13 +1033,13 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M1 KERNELv1x8_E - b .Lsgemm_kernel_L8_Mv1_44 + b L(sgemm_kernel_L8_Mv1_44) .align 5 -.Lsgemm_kernel_L8_Mv1_32: +L(sgemm_kernel_L8_Mv1_32): tst counterL, #1 - ble .Lsgemm_kernel_L8_Mv1_40 + ble L(sgemm_kernel_L8_Mv1_40) KERNELv1x8_I KERNELv1x8_M2 @@ -1051,53 +1051,53 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_E - b .Lsgemm_kernel_L8_Mv1_44 + b L(sgemm_kernel_L8_Mv1_44) -.Lsgemm_kernel_L8_Mv1_40: +L(sgemm_kernel_L8_Mv1_40): INITv1x8 -.Lsgemm_kernel_L8_Mv1_44: +L(sgemm_kernel_L8_Mv1_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L8_Mv1_100 + ble L(sgemm_kernel_L8_Mv1_100) .align 5 -.Lsgemm_kernel_L8_Mv1_46: +L(sgemm_kernel_L8_Mv1_46): KERNELv1x8_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L8_Mv1_46 + bne L(sgemm_kernel_L8_Mv1_46) -.Lsgemm_kernel_L8_Mv1_100: +L(sgemm_kernel_L8_Mv1_100): SAVEv1x8 -.Lsgemm_kernel_L8_Mv1_END: +L(sgemm_kernel_L8_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension - b.any .Lsgemm_kernel_L8_Mv1_20 + b.any L(sgemm_kernel_L8_Mv1_20) -.Lsgemm_kernel_L8_END: +L(sgemm_kernel_L8_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 8 * 4 subs counterJ, counterJ , #1 // j-- - bgt .Lsgemm_kernel_L8_BEGIN + bgt L(sgemm_kernel_L8_BEGIN) /******************************************************************************/ /* Repeat the same thing if 4 left in N */ .align 5 -.Lsgemm_kernel_L4_BEGIN: +L(sgemm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #4 - ble .Lsgemm_kernel_L2_BEGIN + ble L(sgemm_kernel_L2_BEGIN) mov pCRow0, pC @@ -1106,26 +1106,26 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Lsgemm_kernel_L4_Mv2_BEGIN: +L(sgemm_kernel_L4_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Lsgemm_kernel_L4_Mv1_BEGIN + blt L(sgemm_kernel_L4_Mv1_BEGIN) mov counterI, origM .align 5 -.Lsgemm_kernel_L4_Mv2_20: +L(sgemm_kernel_L4_Mv2_20): mov pB, origPB INITv2x4 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Lsgemm_kernel_L4_Mv2_44 + ble L(sgemm_kernel_L4_Mv2_44) .align 5 -.Lsgemm_kernel_L4_Mv2_22: +L(sgemm_kernel_L4_Mv2_22): KERNELv2x4_SUB KERNELv2x4_SUB @@ -1137,53 +1137,53 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_Mv2_22 + bgt L(sgemm_kernel_L4_Mv2_22) -.Lsgemm_kernel_L4_Mv2_44: +L(sgemm_kernel_L4_Mv2_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L4_Mv2_100 + ble L(sgemm_kernel_L4_Mv2_100) .align 5 -.Lsgemm_kernel_L4_Mv2_46: +L(sgemm_kernel_L4_Mv2_46): KERNELv2x4_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L4_Mv2_46 + bne L(sgemm_kernel_L4_Mv2_46) -.Lsgemm_kernel_L4_Mv2_100: +L(sgemm_kernel_L4_Mv2_100): SAVEv2x4 -.Lsgemm_kernel_L4_Mv2_END: +L(sgemm_kernel_L4_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Lsgemm_kernel_L4_Mv2_20 + bge L(sgemm_kernel_L4_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Lsgemm_kernel_L4_END + beq L(sgemm_kernel_L4_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x4 kernel. -.Lsgemm_kernel_L4_Mv1_BEGIN: +L(sgemm_kernel_L4_Mv1_BEGIN): whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension .align 5 -.Lsgemm_kernel_L4_Mv1_20: +L(sgemm_kernel_L4_Mv1_20): mov pB, origPB INITv1x4 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Lsgemm_kernel_L4_Mv1_44 + ble L(sgemm_kernel_L4_Mv1_44) .align 5 -.Lsgemm_kernel_L4_Mv1_22: +L(sgemm_kernel_L4_Mv1_22): KERNELv1x4_SUB KERNELv1x4_SUB @@ -1195,34 +1195,34 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_Mv1_22 + bgt L(sgemm_kernel_L4_Mv1_22) -.Lsgemm_kernel_L4_Mv1_44: +L(sgemm_kernel_L4_Mv1_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L4_Mv1_100 + ble L(sgemm_kernel_L4_Mv1_100) .align 5 -.Lsgemm_kernel_L4_Mv1_46: +L(sgemm_kernel_L4_Mv1_46): KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L4_Mv1_46 + bne L(sgemm_kernel_L4_Mv1_46) -.Lsgemm_kernel_L4_Mv1_100: +L(sgemm_kernel_L4_Mv1_100): SAVEv1x4 -.Lsgemm_kernel_L4_Mv1_END: +L(sgemm_kernel_L4_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lsgemm_kernel_L4_Mv1_20 + b.any L(sgemm_kernel_L4_Mv1_20) -.Lsgemm_kernel_L4_END: +L(sgemm_kernel_L4_END): lsl temp, origK, #4 add origPB, origPB, temp // B = B + K * 4 * 4 @@ -1230,11 +1230,11 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ /* Repeat the same thing if 2 left in N */ .align 5 -.Lsgemm_kernel_L2_BEGIN: +L(sgemm_kernel_L2_BEGIN): mov counterJ , origN tst counterJ , #2 - ble .Lsgemm_kernel_L1_BEGIN + ble L(sgemm_kernel_L1_BEGIN) mov pCRow0, pC @@ -1242,26 +1242,26 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Lsgemm_kernel_L2_Mv2_BEGIN: +L(sgemm_kernel_L2_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Lsgemm_kernel_L2_Mv1_BEGIN + blt L(sgemm_kernel_L2_Mv1_BEGIN) mov counterI, origM .align 5 -.Lsgemm_kernel_L2_Mv2_20: +L(sgemm_kernel_L2_Mv2_20): mov pB, origPB INITv2x2 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Lsgemm_kernel_L2_Mv2_44 + ble L(sgemm_kernel_L2_Mv2_44) .align 5 -.Lsgemm_kernel_L2_Mv2_22: +L(sgemm_kernel_L2_Mv2_22): KERNELv2x2_SUB KERNELv2x2_SUB @@ -1273,54 +1273,54 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_Mv2_22 + bgt L(sgemm_kernel_L2_Mv2_22) -.Lsgemm_kernel_L2_Mv2_44: +L(sgemm_kernel_L2_Mv2_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L2_Mv2_100 + ble L(sgemm_kernel_L2_Mv2_100) .align 5 -.Lsgemm_kernel_L2_Mv2_46: +L(sgemm_kernel_L2_Mv2_46): KERNELv2x2_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L2_Mv2_46 + bne L(sgemm_kernel_L2_Mv2_46) -.Lsgemm_kernel_L2_Mv2_100: +L(sgemm_kernel_L2_Mv2_100): SAVEv2x2 -.Lsgemm_kernel_L2_Mv2_END: +L(sgemm_kernel_L2_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Lsgemm_kernel_L2_Mv2_20 + bge L(sgemm_kernel_L2_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Lsgemm_kernel_L2_END + beq L(sgemm_kernel_L2_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x2 kernel. -.Lsgemm_kernel_L2_Mv1_BEGIN: +L(sgemm_kernel_L2_Mv1_BEGIN): whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s .align 5 -.Lsgemm_kernel_L2_Mv1_20: +L(sgemm_kernel_L2_Mv1_20): mov pB, origPB INITv1x2 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Lsgemm_kernel_L2_Mv1_44 + ble L(sgemm_kernel_L2_Mv1_44) .align 5 -.Lsgemm_kernel_L2_Mv1_22: +L(sgemm_kernel_L2_Mv1_22): KERNELv1x2_SUB KERNELv1x2_SUB @@ -1332,45 +1332,45 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_Mv1_22 + bgt L(sgemm_kernel_L2_Mv1_22) -.Lsgemm_kernel_L2_Mv1_44: +L(sgemm_kernel_L2_Mv1_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L2_Mv1_100 + ble L(sgemm_kernel_L2_Mv1_100) .align 5 -.Lsgemm_kernel_L2_Mv1_46: +L(sgemm_kernel_L2_Mv1_46): KERNELv1x2_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L2_Mv1_46 + bne L(sgemm_kernel_L2_Mv1_46) -.Lsgemm_kernel_L2_Mv1_100: +L(sgemm_kernel_L2_Mv1_100): SAVEv1x2 -.Lsgemm_kernel_L2_Mv1_END: +L(sgemm_kernel_L2_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lsgemm_kernel_L2_Mv1_20 + b.any L(sgemm_kernel_L2_Mv1_20) -.Lsgemm_kernel_L2_END: +L(sgemm_kernel_L2_END): add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 /******************************************************************************/ /* Repeat the same thing if 1 left in N */ .align 5 -.Lsgemm_kernel_L1_BEGIN: +L(sgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lsgemm_kernel_L999 // done + ble L(sgemm_kernel_L999) // done mov pCRow0, pC @@ -1378,26 +1378,26 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Lsgemm_kernel_L1_Mv2_BEGIN: +L(sgemm_kernel_L1_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Lsgemm_kernel_L1_Mv1_BEGIN + blt L(sgemm_kernel_L1_Mv1_BEGIN) mov counterI, origM .align 5 -.Lsgemm_kernel_L1_Mv2_20: +L(sgemm_kernel_L1_Mv2_20): mov pB, origPB INITv2x1 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Lsgemm_kernel_L1_Mv2_44 + ble L(sgemm_kernel_L1_Mv2_44) .align 5 -.Lsgemm_kernel_L1_Mv2_22: +L(sgemm_kernel_L1_Mv2_22): KERNELv2x1_SUB KERNELv2x1_SUB @@ -1409,54 +1409,54 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_Mv2_22 + bgt L(sgemm_kernel_L1_Mv2_22) -.Lsgemm_kernel_L1_Mv2_44: +L(sgemm_kernel_L1_Mv2_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L1_Mv2_100 + ble L(sgemm_kernel_L1_Mv2_100) .align 5 -.Lsgemm_kernel_L1_Mv2_46: +L(sgemm_kernel_L1_Mv2_46): KERNELv2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_Mv2_46 + bgt L(sgemm_kernel_L1_Mv2_46) -.Lsgemm_kernel_L1_Mv2_100: +L(sgemm_kernel_L1_Mv2_100): SAVEv2x1 -.Lsgemm_kernel_L1_Mv2_END: +L(sgemm_kernel_L1_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Lsgemm_kernel_L1_Mv2_20 + bge L(sgemm_kernel_L1_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Lsgemm_kernel_L1_END + beq L(sgemm_kernel_L1_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x1 kernel. -.Lsgemm_kernel_L1_Mv1_BEGIN: +L(sgemm_kernel_L1_Mv1_BEGIN): whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s .align 5 -.Lsgemm_kernel_L1_Mv1_20: +L(sgemm_kernel_L1_Mv1_20): mov pB, origPB INITv1x1 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Lsgemm_kernel_L1_Mv1_44 + ble L(sgemm_kernel_L1_Mv1_44) .align 5 -.Lsgemm_kernel_L1_Mv1_22: +L(sgemm_kernel_L1_Mv1_22): KERNELv1x1_SUB KERNELv1x1_SUB @@ -1468,38 +1468,38 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_Mv1_22 + bgt L(sgemm_kernel_L1_Mv1_22) -.Lsgemm_kernel_L1_Mv1_44: +L(sgemm_kernel_L1_Mv1_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L1_Mv1_100 + ble L(sgemm_kernel_L1_Mv1_100) .align 5 -.Lsgemm_kernel_L1_Mv1_46: +L(sgemm_kernel_L1_Mv1_46): KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_Mv1_46 + bgt L(sgemm_kernel_L1_Mv1_46) -.Lsgemm_kernel_L1_Mv1_100: +L(sgemm_kernel_L1_Mv1_100): SAVEv1x1 -.Lsgemm_kernel_L1_Mv1_END: +L(sgemm_kernel_L1_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lsgemm_kernel_L1_Mv1_20 + b.any L(sgemm_kernel_L1_Mv1_20) -.Lsgemm_kernel_L1_END: +L(sgemm_kernel_L1_END): /******************************************************************************/ -.Lsgemm_kernel_L999: +L(sgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/sgemm_kernel_sve_v1x8.S b/kernel/arm64/sgemm_kernel_sve_v1x8.S index 88c74bc0f5..b3d5e21560 100644 --- a/kernel/arm64/sgemm_kernel_sve_v1x8.S +++ b/kernel/arm64/sgemm_kernel_sve_v1x8.S @@ -479,20 +479,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Ldgemm_kernel_L4_BEGIN + ble L(dgemm_kernel_L4_BEGIN) /******************************************************************************/ /* Repeat this as long as there are 8 left in N */ .align 5 -.Ldgemm_kernel_L8_BEGIN: +L(dgemm_kernel_L8_BEGIN): mov pCRow0, pC add pC, pC, LDC, lsl #3 // add 8 x LDC mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L8_Mv1_BEGIN: +L(dgemm_kernel_L8_Mv1_BEGIN): /* Loop over M is done in an SVE fashion. This has the benefit of the last M%SVE_LEN iterations being done in a single sweep */ mov counterI, #0 @@ -500,14 +500,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension .align 5 -.Ldgemm_kernel_L8_Mv1_20: +L(dgemm_kernel_L8_Mv1_20): mov pB, origPB INITv1x8 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Ldgemm_kernel_L8_Mv1_32 + blt L(dgemm_kernel_L8_Mv1_32) KERNELv1x8_I KERNELv1x8_M2 @@ -519,10 +519,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Ldgemm_kernel_L8_Mv1_22a + ble L(dgemm_kernel_L8_Mv1_22a) .align 5 -.Ldgemm_kernel_L8_Mv1_22: +L(dgemm_kernel_L8_Mv1_22): KERNELv1x8_M1 KERNELv1x8_M2 @@ -534,10 +534,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M2 subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L8_Mv1_22 + bgt L(dgemm_kernel_L8_Mv1_22) .align 5 -.Ldgemm_kernel_L8_Mv1_22a: +L(dgemm_kernel_L8_Mv1_22a): KERNELv1x8_M1 KERNELv1x8_M2 @@ -548,13 +548,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M1 KERNELv1x8_E - b .Ldgemm_kernel_L8_Mv1_44 + b L(dgemm_kernel_L8_Mv1_44) .align 5 -.Ldgemm_kernel_L8_Mv1_32: +L(dgemm_kernel_L8_Mv1_32): tst counterL, #1 - ble .Ldgemm_kernel_L8_Mv1_40 + ble L(dgemm_kernel_L8_Mv1_40) KERNELv1x8_I KERNELv1x8_M2 @@ -566,56 +566,56 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_E - b .Ldgemm_kernel_L8_Mv1_44 + b L(dgemm_kernel_L8_Mv1_44) -.Ldgemm_kernel_L8_Mv1_40: +L(dgemm_kernel_L8_Mv1_40): INITv1x8 -.Ldgemm_kernel_L8_Mv1_44: +L(dgemm_kernel_L8_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L8_Mv1_100 + ble L(dgemm_kernel_L8_Mv1_100) .align 5 -.Ldgemm_kernel_L8_Mv1_46: +L(dgemm_kernel_L8_Mv1_46): KERNELv1x8_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L8_Mv1_46 + bne L(dgemm_kernel_L8_Mv1_46) -.Ldgemm_kernel_L8_Mv1_100: +L(dgemm_kernel_L8_Mv1_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVEv1x8 -.Ldgemm_kernel_L8_Mv1_END: +L(dgemm_kernel_L8_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension - b.any .Ldgemm_kernel_L8_Mv1_20 + b.any L(dgemm_kernel_L8_Mv1_20) -.Ldgemm_kernel_L8_END: +L(dgemm_kernel_L8_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 8 * 4 subs counterJ, counterJ , #1 // j-- - bgt .Ldgemm_kernel_L8_BEGIN + bgt L(dgemm_kernel_L8_BEGIN) /******************************************************************************/ /* Repeat the same thing if 4 left in N */ .align 5 -.Ldgemm_kernel_L4_BEGIN: +L(dgemm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #4 - ble .Ldgemm_kernel_L2_BEGIN + ble L(dgemm_kernel_L2_BEGIN) mov pCRow0, pC @@ -624,24 +624,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L4_Mv1_BEGIN: +L(dgemm_kernel_L4_Mv1_BEGIN): mov counterI, #0 whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s .align 5 -.Ldgemm_kernel_L4_Mv1_20: +L(dgemm_kernel_L4_Mv1_20): mov pB, origPB INITv1x4 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L4_Mv1_44 + ble L(dgemm_kernel_L4_Mv1_44) .align 5 -.Ldgemm_kernel_L4_Mv1_22: +L(dgemm_kernel_L4_Mv1_22): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x4_SUB @@ -657,38 +657,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L4_Mv1_22 + bgt L(dgemm_kernel_L4_Mv1_22) -.Ldgemm_kernel_L4_Mv1_44: +L(dgemm_kernel_L4_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L4_Mv1_100 + ble L(dgemm_kernel_L4_Mv1_100) .align 5 -.Ldgemm_kernel_L4_Mv1_46: +L(dgemm_kernel_L4_Mv1_46): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L4_Mv1_46 + bne L(dgemm_kernel_L4_Mv1_46) -.Ldgemm_kernel_L4_Mv1_100: +L(dgemm_kernel_L4_Mv1_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVEv1x4 -.Ldgemm_kernel_L4_Mv1_END: +L(dgemm_kernel_L4_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Ldgemm_kernel_L4_Mv1_20 + b.any L(dgemm_kernel_L4_Mv1_20) -.Ldgemm_kernel_L4_END: +L(dgemm_kernel_L4_END): lsl temp, origK, #4 add origPB, origPB, temp // B = B + K * 4 * 4 @@ -696,11 +696,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /* Repeat the same thing if 2 left in N */ .align 5 -.Ldgemm_kernel_L2_BEGIN: +L(dgemm_kernel_L2_BEGIN): mov counterJ , origN tst counterJ , #2 - ble .Ldgemm_kernel_L1_BEGIN + ble L(dgemm_kernel_L1_BEGIN) mov pCRow0, pC @@ -708,24 +708,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L2_Mv1_BEGIN: +L(dgemm_kernel_L2_Mv1_BEGIN): mov counterI, #0 whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s .align 5 -.Ldgemm_kernel_L2_Mv1_20: +L(dgemm_kernel_L2_Mv1_20): mov pB, origPB INITv1x2 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Ldgemm_kernel_L2_Mv1_44 + ble L(dgemm_kernel_L2_Mv1_44) .align 5 -.Ldgemm_kernel_L2_Mv1_22: +L(dgemm_kernel_L2_Mv1_22): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x2_SUB @@ -739,49 +739,49 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L2_Mv1_22 + bgt L(dgemm_kernel_L2_Mv1_22) -.Ldgemm_kernel_L2_Mv1_44: +L(dgemm_kernel_L2_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L2_Mv1_100 + ble L(dgemm_kernel_L2_Mv1_100) .align 5 -.Ldgemm_kernel_L2_Mv1_46: +L(dgemm_kernel_L2_Mv1_46): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x2_SUB subs counterL, counterL, #1 - bne .Ldgemm_kernel_L2_Mv1_46 + bne L(dgemm_kernel_L2_Mv1_46) -.Ldgemm_kernel_L2_Mv1_100: +L(dgemm_kernel_L2_Mv1_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVEv1x2 -.Ldgemm_kernel_L2_Mv1_END: +L(dgemm_kernel_L2_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Ldgemm_kernel_L2_Mv1_20 + b.any L(dgemm_kernel_L2_Mv1_20) -.Ldgemm_kernel_L2_END: +L(dgemm_kernel_L2_END): add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 /******************************************************************************/ /* Repeat the same thing if 1 left in N */ .align 5 -.Ldgemm_kernel_L1_BEGIN: +L(dgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Ldgemm_kernel_L999 // done + ble L(dgemm_kernel_L999) // done mov pCRow0, pC @@ -789,24 +789,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Ldgemm_kernel_L1_Mv1_BEGIN: +L(dgemm_kernel_L1_Mv1_BEGIN): mov counterI, #0 whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s .align 5 -.Ldgemm_kernel_L1_Mv1_20: +L(dgemm_kernel_L1_Mv1_20): mov pB, origPB INITv1x1 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Ldgemm_kernel_L1_Mv1_44 + ble L(dgemm_kernel_L1_Mv1_44) .align 5 -.Ldgemm_kernel_L1_Mv1_22: +L(dgemm_kernel_L1_Mv1_22): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x1_SUB @@ -819,42 +819,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv1_22 + bgt L(dgemm_kernel_L1_Mv1_22) -.Ldgemm_kernel_L1_Mv1_44: +L(dgemm_kernel_L1_Mv1_44): ands counterL , origK, #7 - ble .Ldgemm_kernel_L1_Mv1_100 + ble L(dgemm_kernel_L1_Mv1_100) .align 5 -.Ldgemm_kernel_L1_Mv1_46: +L(dgemm_kernel_L1_Mv1_46): prfm PLDL1KEEP, [pB, #B_PRE_SIZE] KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Ldgemm_kernel_L1_Mv1_46 + bgt L(dgemm_kernel_L1_Mv1_46) -.Ldgemm_kernel_L1_Mv1_100: +L(dgemm_kernel_L1_Mv1_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVEv1x1 -.Ldgemm_kernel_L1_Mv1_END: +L(dgemm_kernel_L1_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Ldgemm_kernel_L1_Mv1_20 + b.any L(dgemm_kernel_L1_Mv1_20) -.Ldgemm_kernel_L1_END: +L(dgemm_kernel_L1_END): /******************************************************************************/ -.Ldgemm_kernel_L999: +L(dgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/sgemm_kernel_sve_v2x8.S b/kernel/arm64/sgemm_kernel_sve_v2x8.S index c969ed4db4..a073987db0 100644 --- a/kernel/arm64/sgemm_kernel_sve_v2x8.S +++ b/kernel/arm64/sgemm_kernel_sve_v2x8.S @@ -882,24 +882,24 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Lsgemm_kernel_L4_BEGIN + ble L(sgemm_kernel_L4_BEGIN) /******************************************************************************/ /* Repeat this as long as there are 8 left in N */ .align 5 -.Lsgemm_kernel_L8_BEGIN: +L(sgemm_kernel_L8_BEGIN): mov pCRow0, pC add pC, pC, LDC, lsl #3 // add 8 x LDC mov pA1, origPA // pA1 = start of A array -.Lsgemm_kernel_L8_Mv2_BEGIN: +L(sgemm_kernel_L8_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 // Check if M < 2*SVE_LEN - blt .Lsgemm_kernel_L8_Mv1_BEGIN + blt L(sgemm_kernel_L8_Mv1_BEGIN) mov counterI, origM @@ -908,14 +908,14 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ add pA2, pA1, temp, lsl #2 // pA1 = start of A array .align 5 -.Lsgemm_kernel_L8_Mv2_20: +L(sgemm_kernel_L8_Mv2_20): mov pB, origPB INITv2x8 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L8_Mv2_32 + blt L(sgemm_kernel_L8_Mv2_32) KERNELv2x8_I KERNELv2x8_M2 @@ -927,10 +927,10 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lsgemm_kernel_L8_Mv2_22a + ble L(sgemm_kernel_L8_Mv2_22a) .align 5 -.Lsgemm_kernel_L8_Mv2_22: +L(sgemm_kernel_L8_Mv2_22): KERNELv2x8_M1 KERNELv2x8_M2 @@ -942,10 +942,10 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_Mv2_22 + bgt L(sgemm_kernel_L8_Mv2_22) .align 5 -.Lsgemm_kernel_L8_Mv2_22a: +L(sgemm_kernel_L8_Mv2_22a): KERNELv2x8_M1 KERNELv2x8_M2 @@ -956,13 +956,13 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_M1 KERNELv2x8_E - b .Lsgemm_kernel_L8_Mv2_44 + b L(sgemm_kernel_L8_Mv2_44) .align 5 -.Lsgemm_kernel_L8_Mv2_32: +L(sgemm_kernel_L8_Mv2_32): tst counterL, #1 - ble .Lsgemm_kernel_L8_Mv2_40 + ble L(sgemm_kernel_L8_Mv2_40) KERNELv2x8_I KERNELv2x8_M2 @@ -974,57 +974,57 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x8_E - b .Lsgemm_kernel_L8_Mv2_44 + b L(sgemm_kernel_L8_Mv2_44) -.Lsgemm_kernel_L8_Mv2_40: +L(sgemm_kernel_L8_Mv2_40): INITv2x8 -.Lsgemm_kernel_L8_Mv2_44: +L(sgemm_kernel_L8_Mv2_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L8_Mv2_100 + ble L(sgemm_kernel_L8_Mv2_100) .align 5 -.Lsgemm_kernel_L8_Mv2_46: +L(sgemm_kernel_L8_Mv2_46): KERNELv2x8_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L8_Mv2_46 + bne L(sgemm_kernel_L8_Mv2_46) -.Lsgemm_kernel_L8_Mv2_100: +L(sgemm_kernel_L8_Mv2_100): SAVEv2x8 mov pA1, pA2 // pA1 = pA2 mul temp, vec_len, origK // generate address of pA2 add pA2, pA1, temp, lsl #2 // -.Lsgemm_kernel_L8_Mv2_END: +L(sgemm_kernel_L8_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Lsgemm_kernel_L8_Mv2_20 + bge L(sgemm_kernel_L8_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Lsgemm_kernel_L8_END + beq L(sgemm_kernel_L8_END) ////////////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x8 kernel. -.Lsgemm_kernel_L8_Mv1_BEGIN: +L(sgemm_kernel_L8_Mv1_BEGIN): whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension .align 5 -.Lsgemm_kernel_L8_Mv1_20: +L(sgemm_kernel_L8_Mv1_20): mov pB, origPB INITv1x8 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Lsgemm_kernel_L8_Mv1_32 + blt L(sgemm_kernel_L8_Mv1_32) KERNELv1x8_I KERNELv1x8_M2 @@ -1036,10 +1036,10 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lsgemm_kernel_L8_Mv1_22a + ble L(sgemm_kernel_L8_Mv1_22a) .align 5 -.Lsgemm_kernel_L8_Mv1_22: +L(sgemm_kernel_L8_Mv1_22): KERNELv1x8_M1 KERNELv1x8_M2 @@ -1051,10 +1051,10 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M2 subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L8_Mv1_22 + bgt L(sgemm_kernel_L8_Mv1_22) .align 5 -.Lsgemm_kernel_L8_Mv1_22a: +L(sgemm_kernel_L8_Mv1_22a): KERNELv1x8_M1 KERNELv1x8_M2 @@ -1065,13 +1065,13 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_M1 KERNELv1x8_E - b .Lsgemm_kernel_L8_Mv1_44 + b L(sgemm_kernel_L8_Mv1_44) .align 5 -.Lsgemm_kernel_L8_Mv1_32: +L(sgemm_kernel_L8_Mv1_32): tst counterL, #1 - ble .Lsgemm_kernel_L8_Mv1_40 + ble L(sgemm_kernel_L8_Mv1_40) KERNELv1x8_I KERNELv1x8_M2 @@ -1083,53 +1083,53 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x8_E - b .Lsgemm_kernel_L8_Mv1_44 + b L(sgemm_kernel_L8_Mv1_44) -.Lsgemm_kernel_L8_Mv1_40: +L(sgemm_kernel_L8_Mv1_40): INITv1x8 -.Lsgemm_kernel_L8_Mv1_44: +L(sgemm_kernel_L8_Mv1_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L8_Mv1_100 + ble L(sgemm_kernel_L8_Mv1_100) .align 5 -.Lsgemm_kernel_L8_Mv1_46: +L(sgemm_kernel_L8_Mv1_46): KERNELv1x8_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L8_Mv1_46 + bne L(sgemm_kernel_L8_Mv1_46) -.Lsgemm_kernel_L8_Mv1_100: +L(sgemm_kernel_L8_Mv1_100): SAVEv1x8 -.Lsgemm_kernel_L8_Mv1_END: +L(sgemm_kernel_L8_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension - b.any .Lsgemm_kernel_L8_Mv1_20 + b.any L(sgemm_kernel_L8_Mv1_20) -.Lsgemm_kernel_L8_END: +L(sgemm_kernel_L8_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 8 * 4 subs counterJ, counterJ , #1 // j-- - bgt .Lsgemm_kernel_L8_BEGIN + bgt L(sgemm_kernel_L8_BEGIN) /******************************************************************************/ /* Repeat the same thing if 4 left in N */ .align 5 -.Lsgemm_kernel_L4_BEGIN: +L(sgemm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #4 - ble .Lsgemm_kernel_L2_BEGIN + ble L(sgemm_kernel_L2_BEGIN) mov pCRow0, pC @@ -1138,11 +1138,11 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Lsgemm_kernel_L4_Mv2_BEGIN: +L(sgemm_kernel_L4_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Lsgemm_kernel_L4_Mv1_BEGIN + blt L(sgemm_kernel_L4_Mv1_BEGIN) mov counterI, origM @@ -1150,17 +1150,17 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ add pA2, pA1, temp, lsl #2 // pA1 = start of A array .align 5 -.Lsgemm_kernel_L4_Mv2_20: +L(sgemm_kernel_L4_Mv2_20): mov pB, origPB INITv2x4 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Lsgemm_kernel_L4_Mv2_44 + ble L(sgemm_kernel_L4_Mv2_44) .align 5 -.Lsgemm_kernel_L4_Mv2_22: +L(sgemm_kernel_L4_Mv2_22): KERNELv2x4_SUB KERNELv2x4_SUB @@ -1172,56 +1172,56 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_Mv2_22 + bgt L(sgemm_kernel_L4_Mv2_22) -.Lsgemm_kernel_L4_Mv2_44: +L(sgemm_kernel_L4_Mv2_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L4_Mv2_100 + ble L(sgemm_kernel_L4_Mv2_100) .align 5 -.Lsgemm_kernel_L4_Mv2_46: +L(sgemm_kernel_L4_Mv2_46): KERNELv2x4_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L4_Mv2_46 + bne L(sgemm_kernel_L4_Mv2_46) -.Lsgemm_kernel_L4_Mv2_100: +L(sgemm_kernel_L4_Mv2_100): SAVEv2x4 mov pA1, pA2 // pA1 = pA2 mul temp, vec_len, origK // generate address of pA2 add pA2, pA1, temp, lsl #2 // -.Lsgemm_kernel_L4_Mv2_END: +L(sgemm_kernel_L4_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Lsgemm_kernel_L4_Mv2_20 + bge L(sgemm_kernel_L4_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Lsgemm_kernel_L4_END + beq L(sgemm_kernel_L4_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x4 kernel. -.Lsgemm_kernel_L4_Mv1_BEGIN: +L(sgemm_kernel_L4_Mv1_BEGIN): whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension .align 5 -.Lsgemm_kernel_L4_Mv1_20: +L(sgemm_kernel_L4_Mv1_20): mov pB, origPB INITv1x4 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Lsgemm_kernel_L4_Mv1_44 + ble L(sgemm_kernel_L4_Mv1_44) .align 5 -.Lsgemm_kernel_L4_Mv1_22: +L(sgemm_kernel_L4_Mv1_22): KERNELv1x4_SUB KERNELv1x4_SUB @@ -1233,34 +1233,34 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x4_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L4_Mv1_22 + bgt L(sgemm_kernel_L4_Mv1_22) -.Lsgemm_kernel_L4_Mv1_44: +L(sgemm_kernel_L4_Mv1_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L4_Mv1_100 + ble L(sgemm_kernel_L4_Mv1_100) .align 5 -.Lsgemm_kernel_L4_Mv1_46: +L(sgemm_kernel_L4_Mv1_46): KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L4_Mv1_46 + bne L(sgemm_kernel_L4_Mv1_46) -.Lsgemm_kernel_L4_Mv1_100: +L(sgemm_kernel_L4_Mv1_100): SAVEv1x4 -.Lsgemm_kernel_L4_Mv1_END: +L(sgemm_kernel_L4_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lsgemm_kernel_L4_Mv1_20 + b.any L(sgemm_kernel_L4_Mv1_20) -.Lsgemm_kernel_L4_END: +L(sgemm_kernel_L4_END): lsl temp, origK, #4 add origPB, origPB, temp // B = B + K * 4 * 4 @@ -1268,11 +1268,11 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ /* Repeat the same thing if 2 left in N */ .align 5 -.Lsgemm_kernel_L2_BEGIN: +L(sgemm_kernel_L2_BEGIN): mov counterJ , origN tst counterJ , #2 - ble .Lsgemm_kernel_L1_BEGIN + ble L(sgemm_kernel_L1_BEGIN) mov pCRow0, pC @@ -1280,11 +1280,11 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Lsgemm_kernel_L2_Mv2_BEGIN: +L(sgemm_kernel_L2_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Lsgemm_kernel_L2_Mv1_BEGIN + blt L(sgemm_kernel_L2_Mv1_BEGIN) mov counterI, origM @@ -1292,17 +1292,17 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ add pA2, pA1, temp, lsl #2 // pA1 = start of A array .align 5 -.Lsgemm_kernel_L2_Mv2_20: +L(sgemm_kernel_L2_Mv2_20): mov pB, origPB INITv2x2 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Lsgemm_kernel_L2_Mv2_44 + ble L(sgemm_kernel_L2_Mv2_44) .align 5 -.Lsgemm_kernel_L2_Mv2_22: +L(sgemm_kernel_L2_Mv2_22): KERNELv2x2_SUB KERNELv2x2_SUB @@ -1314,57 +1314,57 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_Mv2_22 + bgt L(sgemm_kernel_L2_Mv2_22) -.Lsgemm_kernel_L2_Mv2_44: +L(sgemm_kernel_L2_Mv2_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L2_Mv2_100 + ble L(sgemm_kernel_L2_Mv2_100) .align 5 -.Lsgemm_kernel_L2_Mv2_46: +L(sgemm_kernel_L2_Mv2_46): KERNELv2x2_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L2_Mv2_46 + bne L(sgemm_kernel_L2_Mv2_46) -.Lsgemm_kernel_L2_Mv2_100: +L(sgemm_kernel_L2_Mv2_100): SAVEv2x2 mov pA1, pA2 // pA1 = pA2 mul temp, vec_len, origK // generate address of pA2 add pA2, pA1, temp, lsl #2 // -.Lsgemm_kernel_L2_Mv2_END: +L(sgemm_kernel_L2_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Lsgemm_kernel_L2_Mv2_20 + bge L(sgemm_kernel_L2_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Lsgemm_kernel_L2_END + beq L(sgemm_kernel_L2_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x2 kernel. -.Lsgemm_kernel_L2_Mv1_BEGIN: +L(sgemm_kernel_L2_Mv1_BEGIN): whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s .align 5 -.Lsgemm_kernel_L2_Mv1_20: +L(sgemm_kernel_L2_Mv1_20): mov pB, origPB INITv1x2 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Lsgemm_kernel_L2_Mv1_44 + ble L(sgemm_kernel_L2_Mv1_44) .align 5 -.Lsgemm_kernel_L2_Mv1_22: +L(sgemm_kernel_L2_Mv1_22): KERNELv1x2_SUB KERNELv1x2_SUB @@ -1376,45 +1376,45 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L2_Mv1_22 + bgt L(sgemm_kernel_L2_Mv1_22) -.Lsgemm_kernel_L2_Mv1_44: +L(sgemm_kernel_L2_Mv1_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L2_Mv1_100 + ble L(sgemm_kernel_L2_Mv1_100) .align 5 -.Lsgemm_kernel_L2_Mv1_46: +L(sgemm_kernel_L2_Mv1_46): KERNELv1x2_SUB subs counterL, counterL, #1 - bne .Lsgemm_kernel_L2_Mv1_46 + bne L(sgemm_kernel_L2_Mv1_46) -.Lsgemm_kernel_L2_Mv1_100: +L(sgemm_kernel_L2_Mv1_100): SAVEv1x2 -.Lsgemm_kernel_L2_Mv1_END: +L(sgemm_kernel_L2_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lsgemm_kernel_L2_Mv1_20 + b.any L(sgemm_kernel_L2_Mv1_20) -.Lsgemm_kernel_L2_END: +L(sgemm_kernel_L2_END): add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 /******************************************************************************/ /* Repeat the same thing if 1 left in N */ .align 5 -.Lsgemm_kernel_L1_BEGIN: +L(sgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lsgemm_kernel_L999 // done + ble L(sgemm_kernel_L999) // done mov pCRow0, pC @@ -1422,11 +1422,11 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ mov pA1, origPA // pA1 = start of A array -.Lsgemm_kernel_L1_Mv2_BEGIN: +L(sgemm_kernel_L1_Mv2_BEGIN): mov counterI, #0 cmp origM, vec_lenx2 - blt .Lsgemm_kernel_L1_Mv1_BEGIN + blt L(sgemm_kernel_L1_Mv1_BEGIN) mov counterI, origM @@ -1435,17 +1435,17 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ .align 5 -.Lsgemm_kernel_L1_Mv2_20: +L(sgemm_kernel_L1_Mv2_20): mov pB, origPB INITv2x1 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Lsgemm_kernel_L1_Mv2_44 + ble L(sgemm_kernel_L1_Mv2_44) .align 5 -.Lsgemm_kernel_L1_Mv2_22: +L(sgemm_kernel_L1_Mv2_22): KERNELv2x1_SUB KERNELv2x1_SUB @@ -1457,57 +1457,57 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_Mv2_22 + bgt L(sgemm_kernel_L1_Mv2_22) -.Lsgemm_kernel_L1_Mv2_44: +L(sgemm_kernel_L1_Mv2_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L1_Mv2_100 + ble L(sgemm_kernel_L1_Mv2_100) .align 5 -.Lsgemm_kernel_L1_Mv2_46: +L(sgemm_kernel_L1_Mv2_46): KERNELv2x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_Mv2_46 + bgt L(sgemm_kernel_L1_Mv2_46) -.Lsgemm_kernel_L1_Mv2_100: +L(sgemm_kernel_L1_Mv2_100): SAVEv2x1 mov pA1, pA2 // pA1 = pA2 mul temp, vec_len, origK // generate address of pA2 add pA2, pA1, temp, lsl #2 // -.Lsgemm_kernel_L1_Mv2_END: +L(sgemm_kernel_L1_Mv2_END): sub counterI, counterI, vec_lenx2 cmp counterI, vec_lenx2 - bge .Lsgemm_kernel_L1_Mv2_20 + bge L(sgemm_kernel_L1_Mv2_20) sub counterI, origM, counterI cmp counterI, origM - beq .Lsgemm_kernel_L1_END + beq L(sgemm_kernel_L1_END) ////////////////////////////////// // We have less than 2*SVE_LEN left. We do this with V1x1 kernel. -.Lsgemm_kernel_L1_Mv1_BEGIN: +L(sgemm_kernel_L1_Mv1_BEGIN): whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s .align 5 -.Lsgemm_kernel_L1_Mv1_20: +L(sgemm_kernel_L1_Mv1_20): mov pB, origPB INITv1x1 // fill with zeros asr counterL , origK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Lsgemm_kernel_L1_Mv1_44 + ble L(sgemm_kernel_L1_Mv1_44) .align 5 -.Lsgemm_kernel_L1_Mv1_22: +L(sgemm_kernel_L1_Mv1_22): KERNELv1x1_SUB KERNELv1x1_SUB @@ -1519,38 +1519,38 @@ With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_Mv1_22 + bgt L(sgemm_kernel_L1_Mv1_22) -.Lsgemm_kernel_L1_Mv1_44: +L(sgemm_kernel_L1_Mv1_44): ands counterL , origK, #7 - ble .Lsgemm_kernel_L1_Mv1_100 + ble L(sgemm_kernel_L1_Mv1_100) .align 5 -.Lsgemm_kernel_L1_Mv1_46: +L(sgemm_kernel_L1_Mv1_46): KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lsgemm_kernel_L1_Mv1_46 + bgt L(sgemm_kernel_L1_Mv1_46) -.Lsgemm_kernel_L1_Mv1_100: +L(sgemm_kernel_L1_Mv1_100): SAVEv1x1 -.Lsgemm_kernel_L1_Mv1_END: +L(sgemm_kernel_L1_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lsgemm_kernel_L1_Mv1_20 + b.any L(sgemm_kernel_L1_Mv1_20) -.Lsgemm_kernel_L1_END: +L(sgemm_kernel_L1_END): /******************************************************************************/ -.Lsgemm_kernel_L999: +L(sgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/sgemm_ncopy_4.S b/kernel/arm64/sgemm_ncopy_4.S index de8c8eca6f..5570bb1459 100644 --- a/kernel/arm64/sgemm_ncopy_4.S +++ b/kernel/arm64/sgemm_ncopy_4.S @@ -182,14 +182,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. lsl LDA, LDA, #2 // LDA = LDA * SIZE -.Ldgemm_ncopy_L4_BEGIN: +L(dgemm_ncopy_L4_BEGIN): asr J, N, #2 // J = N / 4 cmp J, #0 - ble .Ldgemm_ncopy_L2_BEGIN + ble L(dgemm_ncopy_L2_BEGIN) .align 5 -.Ldgemm_ncopy_L4_M4_BEGIN: +L(dgemm_ncopy_L4_M4_BEGIN): mov A01, A00 add A02, A01, LDA @@ -199,121 +199,121 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #2 // I = M / 4 cmp I, #0 - ble .Ldgemm_ncopy_L4_M4_40 + ble L(dgemm_ncopy_L4_M4_40) .align 5 -.Ldgemm_ncopy_L4_M4_20: +L(dgemm_ncopy_L4_M4_20): COPY4x4 subs I , I , #1 - bne .Ldgemm_ncopy_L4_M4_20 + bne L(dgemm_ncopy_L4_M4_20) -.Ldgemm_ncopy_L4_M4_40: +L(dgemm_ncopy_L4_M4_40): and I, M , #3 cmp I, #0 - ble .Ldgemm_ncopy_L4_M4_END + ble L(dgemm_ncopy_L4_M4_END) .align 5 -.Ldgemm_ncopy_L4_M4_60: +L(dgemm_ncopy_L4_M4_60): COPY1x4 subs I , I , #1 - bne .Ldgemm_ncopy_L4_M4_60 + bne L(dgemm_ncopy_L4_M4_60) -.Ldgemm_ncopy_L4_M4_END: +L(dgemm_ncopy_L4_M4_END): subs J , J, #1 // j-- - bne .Ldgemm_ncopy_L4_M4_BEGIN + bne L(dgemm_ncopy_L4_M4_BEGIN) /*********************************************************************************************/ -.Ldgemm_ncopy_L2_BEGIN: +L(dgemm_ncopy_L2_BEGIN): tst N, #3 - ble .Ldgemm_ncopy_L999 + ble L(dgemm_ncopy_L999) tst N, #2 - ble .Ldgemm_ncopy_L1_BEGIN + ble L(dgemm_ncopy_L1_BEGIN) -.Ldgemm_ncopy_L2_M4_BEGIN: +L(dgemm_ncopy_L2_M4_BEGIN): mov A01, A00 add A02, A01, LDA add A00, A02, LDA asr I, M, #2 // I = M / 4 cmp I, #0 - ble .Ldgemm_ncopy_L2_M4_40 + ble L(dgemm_ncopy_L2_M4_40) .align 5 -.Ldgemm_ncopy_L2_M4_20: +L(dgemm_ncopy_L2_M4_20): COPY4x2 subs I , I , #1 - bne .Ldgemm_ncopy_L2_M4_20 + bne L(dgemm_ncopy_L2_M4_20) -.Ldgemm_ncopy_L2_M4_40: +L(dgemm_ncopy_L2_M4_40): and I, M , #3 cmp I, #0 - ble .Ldgemm_ncopy_L2_M4_END + ble L(dgemm_ncopy_L2_M4_END) .align 5 -.Ldgemm_ncopy_L2_M4_60: +L(dgemm_ncopy_L2_M4_60): COPY1x2 subs I , I , #1 - bne .Ldgemm_ncopy_L2_M4_60 + bne L(dgemm_ncopy_L2_M4_60) -.Ldgemm_ncopy_L2_M4_END: +L(dgemm_ncopy_L2_M4_END): /*********************************************************************************************/ -.Ldgemm_ncopy_L1_BEGIN: +L(dgemm_ncopy_L1_BEGIN): tst N, #1 - ble .Ldgemm_ncopy_L999 + ble L(dgemm_ncopy_L999) -.Ldgemm_ncopy_L1_M4_BEGIN: +L(dgemm_ncopy_L1_M4_BEGIN): mov A01, A00 asr I, M, #2 // I = M / 4 cmp I, #0 - ble .Ldgemm_ncopy_L1_M4_40 + ble L(dgemm_ncopy_L1_M4_40) .align 5 -.Ldgemm_ncopy_L1_M4_20: +L(dgemm_ncopy_L1_M4_20): COPY4x1 subs I , I , #1 - bne .Ldgemm_ncopy_L1_M4_20 + bne L(dgemm_ncopy_L1_M4_20) -.Ldgemm_ncopy_L1_M4_40: +L(dgemm_ncopy_L1_M4_40): and I, M , #3 cmp I, #0 - ble .Ldgemm_ncopy_L1_M4_END + ble L(dgemm_ncopy_L1_M4_END) .align 5 -.Ldgemm_ncopy_L1_M4_60: +L(dgemm_ncopy_L1_M4_60): COPY1x1 subs I , I , #1 - bne .Ldgemm_ncopy_L1_M4_60 + bne L(dgemm_ncopy_L1_M4_60) -.Ldgemm_ncopy_L1_M4_END: +L(dgemm_ncopy_L1_M4_END): -.Ldgemm_ncopy_L999: +L(dgemm_ncopy_L999): mov x0, #0 RESTORE_REGS diff --git a/kernel/arm64/sgemm_ncopy_8.S b/kernel/arm64/sgemm_ncopy_8.S index d941eb3eb4..11b07c0844 100644 --- a/kernel/arm64/sgemm_ncopy_8.S +++ b/kernel/arm64/sgemm_ncopy_8.S @@ -267,14 +267,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. lsl LDA, LDA, #2 // LDA = LDA * SIZE -.Lsgemm_ncopy_L8_BEGIN: +L(sgemm_ncopy_L8_BEGIN): asr J, N, #3 // J = N / 8 cmp J, #0 - ble .Lsgemm_ncopy_L4_BEGIN + ble L(sgemm_ncopy_L4_BEGIN) .align 5 -.Lsgemm_ncopy_L8_M4_BEGIN: +L(sgemm_ncopy_L8_M4_BEGIN): mov A01, A00 add A02, A01, LDA @@ -288,136 +288,136 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #2 // I = M / 4 cmp I, #0 - ble .Lsgemm_ncopy_L8_M4_40 + ble L(sgemm_ncopy_L8_M4_40) asr K, M, #4 // K = M / 16(cacheline) mov TEMP1, A01 .align 5 -.Lsgemm_tcopy_L8_warnup_1: +L(sgemm_tcopy_L8_warnup_1): ldr s0, [TEMP1], #64 subs K, K, #1 - bgt .Lsgemm_tcopy_L8_warnup_1 + bgt L(sgemm_tcopy_L8_warnup_1) asr K, M, #4 // K = M / 16(cacheline) mov TEMP1, A02 .align 5 -.Lsgemm_tcopy_L8_warnup_2: +L(sgemm_tcopy_L8_warnup_2): ldr s0, [TEMP1], #64 subs K, K, #1 - bgt .Lsgemm_tcopy_L8_warnup_2 + bgt L(sgemm_tcopy_L8_warnup_2) asr K, M, #4 // K = M / 16(cacheline) mov TEMP1, A03 .align 5 -.Lsgemm_tcopy_L8_warnup_3: +L(sgemm_tcopy_L8_warnup_3): ldr s0, [TEMP1], #64 subs K, K, #1 - bgt .Lsgemm_tcopy_L8_warnup_3 + bgt L(sgemm_tcopy_L8_warnup_3) asr K, M, #4 // K = M / 16(cacheline) mov TEMP1, A04 .align 5 -.Lsgemm_tcopy_L8_warnup_4: +L(sgemm_tcopy_L8_warnup_4): ldr s0, [TEMP1], #64 subs K, K, #1 - bgt .Lsgemm_tcopy_L8_warnup_4 + bgt L(sgemm_tcopy_L8_warnup_4) asr K, M, #4 // K = M / 16(cacheline) mov TEMP1, A05 .align 5 -.Lsgemm_tcopy_L8_warnup_5: +L(sgemm_tcopy_L8_warnup_5): ldr s0, [TEMP1], #64 subs K, K, #1 - bgt .Lsgemm_tcopy_L8_warnup_5 + bgt L(sgemm_tcopy_L8_warnup_5) asr K, M, #4 // K = M / 16(cacheline) mov TEMP1, A06 .align 5 -.Lsgemm_tcopy_L8_warnup_6: +L(sgemm_tcopy_L8_warnup_6): ldr s0, [TEMP1], #64 subs K, K, #1 - bgt .Lsgemm_tcopy_L8_warnup_6 + bgt L(sgemm_tcopy_L8_warnup_6) asr K, M, #4 // K = M / 16(cacheline) mov TEMP1, A07 .align 5 -.Lsgemm_tcopy_L8_warnup_7: +L(sgemm_tcopy_L8_warnup_7): ldr s0, [TEMP1], #64 subs K, K, #1 - bgt .Lsgemm_tcopy_L8_warnup_7 + bgt L(sgemm_tcopy_L8_warnup_7) asr K, M, #4 // K = M / 16(cacheline) mov TEMP1, A08 .align 5 -.Lsgemm_tcopy_L8_warnup_8: +L(sgemm_tcopy_L8_warnup_8): ldr s0, [TEMP1], #64 subs K, K, #1 - bgt .Lsgemm_tcopy_L8_warnup_8 + bgt L(sgemm_tcopy_L8_warnup_8) .align 5 -.Lsgemm_ncopy_L8_M4_20: +L(sgemm_ncopy_L8_M4_20): COPY4x8 subs I, I, #1 - bne .Lsgemm_ncopy_L8_M4_20 + bne L(sgemm_ncopy_L8_M4_20) -.Lsgemm_ncopy_L8_M4_40: +L(sgemm_ncopy_L8_M4_40): and I, M, #2 cmp I, #0 - ble .Lsgemm_ncopy_L8_M4_60 + ble L(sgemm_ncopy_L8_M4_60) COPY2x8 -.Lsgemm_ncopy_L8_M4_60: +L(sgemm_ncopy_L8_M4_60): and I, M, #1 cmp I, #0 - ble .Lsgemm_ncopy_L8_M4_END + ble L(sgemm_ncopy_L8_M4_END) COPY1x8 -.Lsgemm_ncopy_L8_M4_END: +L(sgemm_ncopy_L8_M4_END): subs J , J, #1 // j-- - bne .Lsgemm_ncopy_L8_M4_BEGIN + bne L(sgemm_ncopy_L8_M4_BEGIN) /*********************************************************************************************/ -.Lsgemm_ncopy_L4_BEGIN: +L(sgemm_ncopy_L4_BEGIN): tst N, #7 - ble .Lsgemm_ncopy_L999 + ble L(sgemm_ncopy_L999) tst N, #4 - ble .Lsgemm_ncopy_L2_BEGIN + ble L(sgemm_ncopy_L2_BEGIN) -.Lsgemm_ncopy_L4_M4_BEGIN: +L(sgemm_ncopy_L4_M4_BEGIN): mov A01, A00 add A02, A01, LDA add A03, A02, LDA @@ -426,43 +426,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #2 // I = M / 4 cmp I, #0 - ble .Lsgemm_ncopy_L4_M4_40 + ble L(sgemm_ncopy_L4_M4_40) .align 5 -.Lsgemm_ncopy_L4_M4_20: +L(sgemm_ncopy_L4_M4_20): COPY4x4 subs I, I, #1 - bne .Lsgemm_ncopy_L4_M4_20 + bne L(sgemm_ncopy_L4_M4_20) -.Lsgemm_ncopy_L4_M4_40: +L(sgemm_ncopy_L4_M4_40): and I, M, #2 cmp I, #0 - ble .Lsgemm_ncopy_L4_M4_60 + ble L(sgemm_ncopy_L4_M4_60) COPY2x4 -.Lsgemm_ncopy_L4_M4_60: +L(sgemm_ncopy_L4_M4_60): and I, M, #1 cmp I, #0 - ble .Lsgemm_ncopy_L4_M4_END + ble L(sgemm_ncopy_L4_M4_END) COPY1x4 -.Lsgemm_ncopy_L4_M4_END: +L(sgemm_ncopy_L4_M4_END): /*********************************************************************************************/ -.Lsgemm_ncopy_L2_BEGIN: +L(sgemm_ncopy_L2_BEGIN): tst N, #2 - ble .Lsgemm_ncopy_L1_BEGIN + ble L(sgemm_ncopy_L1_BEGIN) -.Lsgemm_ncopy_L2_M4_BEGIN: +L(sgemm_ncopy_L2_M4_BEGIN): mov A01, A00 add A02, A01, LDA @@ -470,59 +470,59 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #2 // I = M / 4 cmp I, #0 - ble .Lsgemm_ncopy_L2_M4_40 + ble L(sgemm_ncopy_L2_M4_40) .align 5 -.Lsgemm_ncopy_L2_M4_20: +L(sgemm_ncopy_L2_M4_20): COPY4x2 subs I , I , #1 - bne .Lsgemm_ncopy_L2_M4_20 + bne L(sgemm_ncopy_L2_M4_20) -.Lsgemm_ncopy_L2_M4_40: +L(sgemm_ncopy_L2_M4_40): and I, M, #2 cmp I, #0 - ble .Lsgemm_ncopy_L2_M4_60 + ble L(sgemm_ncopy_L2_M4_60) COPY2x2 -.Lsgemm_ncopy_L2_M4_60: +L(sgemm_ncopy_L2_M4_60): and I, M, #1 cmp I, #0 - ble .Lsgemm_ncopy_L2_M4_END + ble L(sgemm_ncopy_L2_M4_END) COPY1x2 -.Lsgemm_ncopy_L2_M4_END: +L(sgemm_ncopy_L2_M4_END): -.Lsgemm_ncopy_L1_BEGIN: +L(sgemm_ncopy_L1_BEGIN): tst N, #1 - ble .Lsgemm_ncopy_L999 + ble L(sgemm_ncopy_L999) -.Lsgemm_ncopy_L1_M1_BEGIN: +L(sgemm_ncopy_L1_M1_BEGIN): mov A01, A00 mov I, M cmp I, #0 - ble .Lsgemm_ncopy_L1_M1_END + ble L(sgemm_ncopy_L1_M1_END) .align 5 -.Lsgemm_ncopy_L1_M1_20: +L(sgemm_ncopy_L1_M1_20): COPY1x1 subs I, I, #1 - bne .Lsgemm_ncopy_L1_M1_20 + bne L(sgemm_ncopy_L1_M1_20) -.Lsgemm_ncopy_L1_M1_END: +L(sgemm_ncopy_L1_M1_END): -.Lsgemm_ncopy_L999: +L(sgemm_ncopy_L999): mov x0, #0 RESTORE_REGS diff --git a/kernel/arm64/sgemm_tcopy_16.S b/kernel/arm64/sgemm_tcopy_16.S index 3066421bb9..934fe933eb 100644 --- a/kernel/arm64/sgemm_tcopy_16.S +++ b/kernel/arm64/sgemm_tcopy_16.S @@ -566,13 +566,13 @@ All rights reserved. lsl M8, M, #6 // M8 = M * 16 * SIZE -.Lsgemm_tcopy_L8_BEGIN: +L(sgemm_tcopy_L8_BEGIN): asr J, M, #3 // J = M / 8 cmp J, #0 - ble .Lsgemm_tcopy_L4_BEGIN + ble L(sgemm_tcopy_L4_BEGIN) .align 5 -.Lsgemm_tcopy_L8_M16_BEGIN: +L(sgemm_tcopy_L8_M16_BEGIN): mov A01, A add A02, A01, LDA @@ -589,57 +589,57 @@ All rights reserved. asr I, N, #4 // I = N / 16 cmp I, #0 - ble .Lsgemm_tcopy_L8_M16_40 + ble L(sgemm_tcopy_L8_M16_40) .align 5 -.Lsgemm_tcopy_L8_M16_20: +L(sgemm_tcopy_L8_M16_20): COPY16x8 subs I , I , #1 - bne .Lsgemm_tcopy_L8_M16_20 + bne L(sgemm_tcopy_L8_M16_20) -.Lsgemm_tcopy_L8_M16_40: +L(sgemm_tcopy_L8_M16_40): tst N , #8 - ble .Lsgemm_tcopy_L8_M16_60 + ble L(sgemm_tcopy_L8_M16_60) COPY8x8 -.Lsgemm_tcopy_L8_M16_60: +L(sgemm_tcopy_L8_M16_60): tst N , #4 - ble .Lsgemm_tcopy_L8_M16_80 + ble L(sgemm_tcopy_L8_M16_80) COPY4x8 -.Lsgemm_tcopy_L8_M16_80: +L(sgemm_tcopy_L8_M16_80): tst N , #2 - ble .Lsgemm_tcopy_L8_M16_100 + ble L(sgemm_tcopy_L8_M16_100) COPY2x8 -.Lsgemm_tcopy_L8_M16_100: +L(sgemm_tcopy_L8_M16_100): tst N, #1 - ble .Lsgemm_tcopy_L8_M16_END + ble L(sgemm_tcopy_L8_M16_END) COPY1x8 -.Lsgemm_tcopy_L8_M16_END: +L(sgemm_tcopy_L8_M16_END): subs J , J, #1 // j-- - bne .Lsgemm_tcopy_L8_M16_BEGIN + bne L(sgemm_tcopy_L8_M16_BEGIN) /*********************************************************************************************/ -.Lsgemm_tcopy_L4_BEGIN: +L(sgemm_tcopy_L4_BEGIN): tst M, #7 - ble .Lsgemm_tcopy_L999 + ble L(sgemm_tcopy_L999) tst M, #4 - ble .Lsgemm_tcopy_L2_BEGIN + ble L(sgemm_tcopy_L2_BEGIN) -.Lsgemm_tcopy_L4_M16_BEGIN: +L(sgemm_tcopy_L4_M16_BEGIN): mov A01, A add A02, A01, LDA @@ -652,57 +652,57 @@ All rights reserved. asr I, N, #4 // I = N / 16 cmp I, #0 - ble .Lsgemm_tcopy_L4_M16_40 + ble L(sgemm_tcopy_L4_M16_40) .align 5 -.Lsgemm_tcopy_L4_M16_20: +L(sgemm_tcopy_L4_M16_20): COPY16x4 subs I , I , #1 - bne .Lsgemm_tcopy_L4_M16_20 + bne L(sgemm_tcopy_L4_M16_20) -.Lsgemm_tcopy_L4_M16_40: +L(sgemm_tcopy_L4_M16_40): tst N , #8 - ble .Lsgemm_tcopy_L4_M16_60 + ble L(sgemm_tcopy_L4_M16_60) COPY8x4 -.Lsgemm_tcopy_L4_M16_60: +L(sgemm_tcopy_L4_M16_60): tst N , #4 - ble .Lsgemm_tcopy_L4_M16_80 + ble L(sgemm_tcopy_L4_M16_80) COPY4x4 -.Lsgemm_tcopy_L4_M16_80: +L(sgemm_tcopy_L4_M16_80): tst N , #2 - ble .Lsgemm_tcopy_L4_M16_100 + ble L(sgemm_tcopy_L4_M16_100) COPY2x4 -.Lsgemm_tcopy_L4_M16_100: +L(sgemm_tcopy_L4_M16_100): tst N, #1 - ble .Lsgemm_tcopy_L4_M16_END + ble L(sgemm_tcopy_L4_M16_END) COPY1x4 -.Lsgemm_tcopy_L4_M16_END: +L(sgemm_tcopy_L4_M16_END): /*********************************************************************************************/ -.Lsgemm_tcopy_L2_BEGIN: +L(sgemm_tcopy_L2_BEGIN): tst M, #3 - ble .Lsgemm_tcopy_L999 + ble L(sgemm_tcopy_L999) tst M, #2 - ble .Lsgemm_tcopy_L1_BEGIN + ble L(sgemm_tcopy_L1_BEGIN) -.Lsgemm_tcopy_L2_M16_BEGIN: +L(sgemm_tcopy_L2_M16_BEGIN): mov A01, A add A02, A01, LDA add A, A02, LDA @@ -712,99 +712,99 @@ All rights reserved. asr I, N, #4 // I = N / 16 cmp I, #0 - ble .Lsgemm_tcopy_L2_M16_40 + ble L(sgemm_tcopy_L2_M16_40) .align 5 -.Lsgemm_tcopy_L2_M16_20: +L(sgemm_tcopy_L2_M16_20): COPY16x2 subs I , I , #1 - bne .Lsgemm_tcopy_L2_M16_20 + bne L(sgemm_tcopy_L2_M16_20) -.Lsgemm_tcopy_L2_M16_40: +L(sgemm_tcopy_L2_M16_40): tst N , #8 - ble .Lsgemm_tcopy_L2_M16_60 + ble L(sgemm_tcopy_L2_M16_60) COPY8x2 -.Lsgemm_tcopy_L2_M16_60: +L(sgemm_tcopy_L2_M16_60): tst N , #4 - ble .Lsgemm_tcopy_L2_M16_80 + ble L(sgemm_tcopy_L2_M16_80) COPY4x2 -.Lsgemm_tcopy_L2_M16_80: +L(sgemm_tcopy_L2_M16_80): tst N , #2 - ble .Lsgemm_tcopy_L2_M16_100 + ble L(sgemm_tcopy_L2_M16_100) COPY2x2 -.Lsgemm_tcopy_L2_M16_100: +L(sgemm_tcopy_L2_M16_100): tst N , #1 - ble .Lsgemm_tcopy_L2_M16_END + ble L(sgemm_tcopy_L2_M16_END) COPY1x2 -.Lsgemm_tcopy_L2_M16_END: +L(sgemm_tcopy_L2_M16_END): /*********************************************************************************************/ -.Lsgemm_tcopy_L1_BEGIN: +L(sgemm_tcopy_L1_BEGIN): tst M, #1 - ble .Lsgemm_tcopy_L999 + ble L(sgemm_tcopy_L999) -.Lsgemm_tcopy_L1_M16_BEGIN: +L(sgemm_tcopy_L1_M16_BEGIN): mov A01, A // A01 = A mov B00, B asr I, N, #4 // I = M / 16 cmp I, #0 - ble .Lsgemm_tcopy_L1_M16_40 + ble L(sgemm_tcopy_L1_M16_40) .align 5 -.Lsgemm_tcopy_L1_M16_20: +L(sgemm_tcopy_L1_M16_20): COPY16x1 subs I , I , #1 - bne .Lsgemm_tcopy_L1_M16_20 + bne L(sgemm_tcopy_L1_M16_20) -.Lsgemm_tcopy_L1_M16_40: +L(sgemm_tcopy_L1_M16_40): tst N , #8 - ble .Lsgemm_tcopy_L1_M16_60 + ble L(sgemm_tcopy_L1_M16_60) COPY8x1 -.Lsgemm_tcopy_L1_M16_60: +L(sgemm_tcopy_L1_M16_60): tst N , #4 - ble .Lsgemm_tcopy_L1_M16_80 + ble L(sgemm_tcopy_L1_M16_80) COPY4x1 -.Lsgemm_tcopy_L1_M16_80: +L(sgemm_tcopy_L1_M16_80): tst N , #2 - ble .Lsgemm_tcopy_L1_M16_100 + ble L(sgemm_tcopy_L1_M16_100) COPY2x1 -.Lsgemm_tcopy_L1_M16_100: +L(sgemm_tcopy_L1_M16_100): tst N , #1 - ble .Lsgemm_tcopy_L1_M16_END + ble L(sgemm_tcopy_L1_M16_END) COPY1x1 -.Lsgemm_tcopy_L1_M16_END: +L(sgemm_tcopy_L1_M16_END): -.Lsgemm_tcopy_L999: +L(sgemm_tcopy_L999): mov x0, #0 // set return value RESTORE_REGS ret diff --git a/kernel/arm64/sgemm_tcopy_8.S b/kernel/arm64/sgemm_tcopy_8.S index 7d81ba2667..ef52b98b75 100644 --- a/kernel/arm64/sgemm_tcopy_8.S +++ b/kernel/arm64/sgemm_tcopy_8.S @@ -478,14 +478,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. lsl M8, M, #5 // M8 = M * 8 * SIZE -.Lsgemm_tcopy_L8_BEGIN: +L(sgemm_tcopy_L8_BEGIN): asr J, M, #3 // J = M / 8 cmp J, #0 - ble .Lsgemm_tcopy_L4_BEGIN + ble L(sgemm_tcopy_L4_BEGIN) .align 5 -.Lsgemm_tcopy_L8_M8_BEGIN: +L(sgemm_tcopy_L8_M8_BEGIN): mov A01, A add A02, A01, LDA @@ -502,53 +502,53 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, N, #3 // I = N / 8 cmp I, #0 - ble .Lsgemm_tcopy_L8_M8_40 + ble L(sgemm_tcopy_L8_M8_40) .align 5 -.Lsgemm_tcopy_L8_M8_20: +L(sgemm_tcopy_L8_M8_20): COPY8x8 subs I , I , #1 - bne .Lsgemm_tcopy_L8_M8_20 + bne L(sgemm_tcopy_L8_M8_20) -.Lsgemm_tcopy_L8_M8_40: +L(sgemm_tcopy_L8_M8_40): tst N , #4 - ble .Lsgemm_tcopy_L8_M8_60 + ble L(sgemm_tcopy_L8_M8_60) COPY4x8 -.Lsgemm_tcopy_L8_M8_60: +L(sgemm_tcopy_L8_M8_60): tst N , #2 - ble .Lsgemm_tcopy_L8_M8_80 + ble L(sgemm_tcopy_L8_M8_80) COPY2x8 -.Lsgemm_tcopy_L8_M8_80: +L(sgemm_tcopy_L8_M8_80): tst N, #1 - ble .Lsgemm_tcopy_L8_M8_END + ble L(sgemm_tcopy_L8_M8_END) COPY1x8 -.Lsgemm_tcopy_L8_M8_END: +L(sgemm_tcopy_L8_M8_END): subs J, J, #1 // j-- - bne .Lsgemm_tcopy_L8_M8_BEGIN + bne L(sgemm_tcopy_L8_M8_BEGIN) /*********************************************************************************************/ -.Lsgemm_tcopy_L4_BEGIN: +L(sgemm_tcopy_L4_BEGIN): tst M, #7 - ble .Lsgemm_tcopy_L999 + ble L(sgemm_tcopy_L999) tst M, #4 - ble .Lsgemm_tcopy_L2_BEGIN + ble L(sgemm_tcopy_L2_BEGIN) -.Lsgemm_tcopy_L4_M8_BEGIN: +L(sgemm_tcopy_L4_M8_BEGIN): mov A01, A add A02, A01, LDA @@ -561,51 +561,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, N, #3 // I = N / 8 cmp I, #0 - ble .Lsgemm_tcopy_L4_M8_40 + ble L(sgemm_tcopy_L4_M8_40) .align 5 -.Lsgemm_tcopy_L4_M8_20: +L(sgemm_tcopy_L4_M8_20): COPY8x4 subs I , I , #1 - bne .Lsgemm_tcopy_L4_M8_20 + bne L(sgemm_tcopy_L4_M8_20) -.Lsgemm_tcopy_L4_M8_40: +L(sgemm_tcopy_L4_M8_40): tst N , #4 - ble .Lsgemm_tcopy_L4_M8_60 + ble L(sgemm_tcopy_L4_M8_60) COPY4x4 -.Lsgemm_tcopy_L4_M8_60: +L(sgemm_tcopy_L4_M8_60): tst N , #2 - ble .Lsgemm_tcopy_L4_M8_80 + ble L(sgemm_tcopy_L4_M8_80) COPY2x4 -.Lsgemm_tcopy_L4_M8_80: +L(sgemm_tcopy_L4_M8_80): tst N , #1 - ble .Lsgemm_tcopy_L4_M8_END + ble L(sgemm_tcopy_L4_M8_END) COPY1x4 -.Lsgemm_tcopy_L4_M8_END: +L(sgemm_tcopy_L4_M8_END): /*********************************************************************************************/ -.Lsgemm_tcopy_L2_BEGIN: +L(sgemm_tcopy_L2_BEGIN): tst M, #3 - ble .Lsgemm_tcopy_L999 + ble L(sgemm_tcopy_L999) tst M, #2 - ble .Lsgemm_tcopy_L1_BEGIN + ble L(sgemm_tcopy_L1_BEGIN) -.Lsgemm_tcopy_L2_M16_BEGIN: +L(sgemm_tcopy_L2_M16_BEGIN): mov A01, A add A02, A01, LDA @@ -616,89 +616,89 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, N, #3 // I = N / 8 cmp I, #0 - ble .Lsgemm_tcopy_L2_M8_40 + ble L(sgemm_tcopy_L2_M8_40) .align 5 -.Lsgemm_tcopy_L2_M8_20: +L(sgemm_tcopy_L2_M8_20): COPY8x2 subs I , I , #1 - bne .Lsgemm_tcopy_L2_M8_20 + bne L(sgemm_tcopy_L2_M8_20) -.Lsgemm_tcopy_L2_M8_40: +L(sgemm_tcopy_L2_M8_40): tst N , #4 - ble .Lsgemm_tcopy_L2_M8_60 + ble L(sgemm_tcopy_L2_M8_60) COPY4x2 -.Lsgemm_tcopy_L2_M8_60: +L(sgemm_tcopy_L2_M8_60): tst N , #2 - ble .Lsgemm_tcopy_L2_M8_80 + ble L(sgemm_tcopy_L2_M8_80) COPY2x2 -.Lsgemm_tcopy_L2_M8_80: +L(sgemm_tcopy_L2_M8_80): tst N , #1 - ble .Lsgemm_tcopy_L2_M8_END + ble L(sgemm_tcopy_L2_M8_END) COPY1x2 -.Lsgemm_tcopy_L2_M8_END: +L(sgemm_tcopy_L2_M8_END): /*********************************************************************************************/ -.Lsgemm_tcopy_L1_BEGIN: +L(sgemm_tcopy_L1_BEGIN): tst M, #1 - ble .Lsgemm_tcopy_L999 + ble L(sgemm_tcopy_L999) -.Lsgemm_tcopy_L1_M16_BEGIN: +L(sgemm_tcopy_L1_M16_BEGIN): mov A01, A // A01 = A mov B00, B asr I, N, #3 // I = M / 8 cmp I, #0 - ble .Lsgemm_tcopy_L1_M8_40 + ble L(sgemm_tcopy_L1_M8_40) .align 5 -.Lsgemm_tcopy_L1_M8_20: +L(sgemm_tcopy_L1_M8_20): COPY8x1 subs I , I , #1 - bne .Lsgemm_tcopy_L1_M8_20 + bne L(sgemm_tcopy_L1_M8_20) -.Lsgemm_tcopy_L1_M8_40: +L(sgemm_tcopy_L1_M8_40): tst N , #4 - ble .Lsgemm_tcopy_L1_M8_60 + ble L(sgemm_tcopy_L1_M8_60) COPY4x1 -.Lsgemm_tcopy_L1_M8_60: +L(sgemm_tcopy_L1_M8_60): tst N , #2 - ble .Lsgemm_tcopy_L1_M8_80 + ble L(sgemm_tcopy_L1_M8_80) COPY2x1 -.Lsgemm_tcopy_L1_M8_80: +L(sgemm_tcopy_L1_M8_80): tst N , #1 - ble .Lsgemm_tcopy_L1_M8_END + ble L(sgemm_tcopy_L1_M8_END) COPY1x1 -.Lsgemm_tcopy_L1_M8_END: +L(sgemm_tcopy_L1_M8_END): -.Lsgemm_tcopy_L999: +L(sgemm_tcopy_L999): mov x0, #0 // set return value RESTORE_REGS diff --git a/kernel/arm64/strmm_kernel_16x4.S b/kernel/arm64/strmm_kernel_16x4.S index a44326aeb2..c81cf018af 100644 --- a/kernel/arm64/strmm_kernel_16x4.S +++ b/kernel/arm64/strmm_kernel_16x4.S @@ -1036,7 +1036,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE -.Lstrmm_kernel_begin: +L(strmm_kernel_begin): .align 5 add sp, sp, #-(11 * 16) @@ -1067,11 +1067,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lstrmm_kernel_L2_BEGIN + ble L(strmm_kernel_L2_BEGIN) /******************************************************************************/ -.Lstrmm_kernel_L4_BEGIN: +L(strmm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -1085,15 +1085,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = start of A array -.Lstrmm_kernel_L4_M16_BEGIN: +L(strmm_kernel_L4_M16_BEGIN): mov counterI, origM asr counterI, counterI, #4 // counterI = counterI / 16 cmp counterI, #0 - ble .Lstrmm_kernel_L4_M8_BEGIN + ble L(strmm_kernel_L4_M8_BEGIN) .align 5 -.Lstrmm_kernel_L4_M16_20: +L(strmm_kernel_L4_M16_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1115,7 +1115,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 cmp counterL , #2 - blt .Lstrmm_kernel_L4_M16_32 + blt L(strmm_kernel_L4_M16_32) KERNEL16x4_I KERNEL16x4_M2 @@ -1127,10 +1127,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x4_M2 subs counterL, counterL, #2 - ble .Lstrmm_kernel_L4_M16_22a + ble L(strmm_kernel_L4_M16_22a) .align 5 -.Lstrmm_kernel_L4_M16_22: +L(strmm_kernel_L4_M16_22): KERNEL16x4_M1 KERNEL16x4_M2 @@ -1142,10 +1142,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x4_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M16_22 + bgt L(strmm_kernel_L4_M16_22) .align 5 -.Lstrmm_kernel_L4_M16_22a: +L(strmm_kernel_L4_M16_22a): KERNEL16x4_M1 KERNEL16x4_M2 @@ -1156,13 +1156,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x4_M1 KERNEL16x4_E - b .Lstrmm_kernel_L4_M16_44 + b L(strmm_kernel_L4_M16_44) .align 5 -.Lstrmm_kernel_L4_M16_32: +L(strmm_kernel_L4_M16_32): tst counterL, #1 - ble .Lstrmm_kernel_L4_M16_40 + ble L(strmm_kernel_L4_M16_40) KERNEL16x4_I KERNEL16x4_M2 @@ -1173,25 +1173,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x4_M1 KERNEL16x4_E - b .Lstrmm_kernel_L4_M16_44 + b L(strmm_kernel_L4_M16_44) -.Lstrmm_kernel_L4_M16_40: +L(strmm_kernel_L4_M16_40): INIT16x4 -.Lstrmm_kernel_L4_M16_44: +L(strmm_kernel_L4_M16_44): ands counterL , tempK, #7 - ble .Lstrmm_kernel_L4_M16_100 + ble L(strmm_kernel_L4_M16_100) .align 5 -.Lstrmm_kernel_L4_M16_46: +L(strmm_kernel_L4_M16_46): KERNEL16x4_SUB subs counterL, counterL, #1 - bne .Lstrmm_kernel_L4_M16_46 + bne L(strmm_kernel_L4_M16_46) -.Lstrmm_kernel_L4_M16_100: +L(strmm_kernel_L4_M16_100): SAVE16x4 @@ -1214,22 +1214,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] -.Lstrmm_kernel_L4_M16_END: +L(strmm_kernel_L4_M16_END): subs counterI, counterI, #1 - bne .Lstrmm_kernel_L4_M16_20 + bne L(strmm_kernel_L4_M16_20) //------------------------------------------------------------------------------ -.Lstrmm_kernel_L4_M8_BEGIN: +L(strmm_kernel_L4_M8_BEGIN): mov counterI, origM tst counterI , #15 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) tst counterI, #8 - ble .Lstrmm_kernel_L4_M4_BEGIN + ble L(strmm_kernel_L4_M4_BEGIN) -.Lstrmm_kernel_L4_M8_20: +L(strmm_kernel_L4_M8_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1251,54 +1251,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lstrmm_kernel_L4_M8_32 + blt L(strmm_kernel_L4_M8_32) KERNEL8x4_I // do one in the K KERNEL8x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lstrmm_kernel_L4_M8_22a + ble L(strmm_kernel_L4_M8_22a) .align 5 -.Lstrmm_kernel_L4_M8_22: +L(strmm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M8_22 + bgt L(strmm_kernel_L4_M8_22) -.Lstrmm_kernel_L4_M8_22a: +L(strmm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_E - b .Lstrmm_kernel_L4_M8_44 + b L(strmm_kernel_L4_M8_44) -.Lstrmm_kernel_L4_M8_32: +L(strmm_kernel_L4_M8_32): tst counterL, #1 - ble .Lstrmm_kernel_L4_M8_40 + ble L(strmm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_E - b .Lstrmm_kernel_L4_M8_44 + b L(strmm_kernel_L4_M8_44) -.Lstrmm_kernel_L4_M8_40: +L(strmm_kernel_L4_M8_40): INIT8x4 -.Lstrmm_kernel_L4_M8_44: +L(strmm_kernel_L4_M8_44): ands counterL , tempK, #1 - ble .Lstrmm_kernel_L4_M8_100 + ble L(strmm_kernel_L4_M8_100) -.Lstrmm_kernel_L4_M8_46: +L(strmm_kernel_L4_M8_46): KERNEL8x4_SUB -.Lstrmm_kernel_L4_M8_100: +L(strmm_kernel_L4_M8_100): SAVE8x4 @@ -1318,20 +1318,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #8 #endif -.Lstrmm_kernel_L4_M8_END: +L(strmm_kernel_L4_M8_END): //------------------------------------------------------------------------------ -.Lstrmm_kernel_L4_M4_BEGIN: +L(strmm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) tst counterI, #4 - ble .Lstrmm_kernel_L4_M2_BEGIN + ble L(strmm_kernel_L4_M2_BEGIN) -.Lstrmm_kernel_L4_M4_20: +L(strmm_kernel_L4_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1351,54 +1351,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lstrmm_kernel_L4_M4_32 + blt L(strmm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lstrmm_kernel_L4_M4_22a + ble L(strmm_kernel_L4_M4_22a) .align 5 -.Lstrmm_kernel_L4_M4_22: +L(strmm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M4_22 + bgt L(strmm_kernel_L4_M4_22) -.Lstrmm_kernel_L4_M4_22a: +L(strmm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lstrmm_kernel_L4_M4_44 + b L(strmm_kernel_L4_M4_44) -.Lstrmm_kernel_L4_M4_32: +L(strmm_kernel_L4_M4_32): tst counterL, #1 - ble .Lstrmm_kernel_L4_M4_40 + ble L(strmm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lstrmm_kernel_L4_M4_44 + b L(strmm_kernel_L4_M4_44) -.Lstrmm_kernel_L4_M4_40: +L(strmm_kernel_L4_M4_40): INIT4x4 -.Lstrmm_kernel_L4_M4_44: +L(strmm_kernel_L4_M4_44): ands counterL , tempK, #1 - ble .Lstrmm_kernel_L4_M4_100 + ble L(strmm_kernel_L4_M4_100) -.Lstrmm_kernel_L4_M4_46: +L(strmm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lstrmm_kernel_L4_M4_100: +L(strmm_kernel_L4_M4_100): SAVE4x4 @@ -1416,20 +1416,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L4_M4_END: +L(strmm_kernel_L4_M4_END): //------------------------------------------------------------------------------ -.Lstrmm_kernel_L4_M2_BEGIN: +L(strmm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L4_M1_BEGIN + ble L(strmm_kernel_L4_M1_BEGIN) -.Lstrmm_kernel_L4_M2_20: +L(strmm_kernel_L4_M2_20): INIT2x4 @@ -1452,9 +1452,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L4_M2_40 + ble L(strmm_kernel_L4_M2_40) -.Lstrmm_kernel_L4_M2_22: +L(strmm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1467,22 +1467,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M2_22 + bgt L(strmm_kernel_L4_M2_22) -.Lstrmm_kernel_L4_M2_40: +L(strmm_kernel_L4_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L4_M2_100 + ble L(strmm_kernel_L4_M2_100) -.Lstrmm_kernel_L4_M2_42: +L(strmm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M2_42 + bgt L(strmm_kernel_L4_M2_42) -.Lstrmm_kernel_L4_M2_100: +L(strmm_kernel_L4_M2_100): SAVE2x4 @@ -1501,15 +1501,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L4_M2_END: +L(strmm_kernel_L4_M2_END): -.Lstrmm_kernel_L4_M1_BEGIN: +L(strmm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) -.Lstrmm_kernel_L4_M1_20: +L(strmm_kernel_L4_M1_20): INIT1x4 @@ -1532,9 +1532,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L4_M1_40 + ble L(strmm_kernel_L4_M1_40) -.Lstrmm_kernel_L4_M1_22: +L(strmm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1546,22 +1546,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M1_22 + bgt L(strmm_kernel_L4_M1_22) -.Lstrmm_kernel_L4_M1_40: +L(strmm_kernel_L4_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L4_M1_100 + ble L(strmm_kernel_L4_M1_100) -.Lstrmm_kernel_L4_M1_42: +L(strmm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M1_42 + bgt L(strmm_kernel_L4_M1_42) -.Lstrmm_kernel_L4_M1_100: +L(strmm_kernel_L4_M1_100): SAVE1x4 @@ -1580,26 +1580,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #1 #endif -.Lstrmm_kernel_L4_END: +L(strmm_kernel_L4_END): add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4 #if !defined(LEFT) add tempOffset, tempOffset, #4 #endif subs counterJ, counterJ , #1 // j-- - bgt .Lstrmm_kernel_L4_BEGIN + bgt L(strmm_kernel_L4_BEGIN) /******************************************************************************/ -.Lstrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(strmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lstrmm_kernel_L999 + ble L(strmm_kernel_L999) tst counterJ , #2 - ble .Lstrmm_kernel_L1_BEGIN + ble L(strmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1610,14 +1610,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = A -.Lstrmm_kernel_L2_M16_BEGIN: +L(strmm_kernel_L2_M16_BEGIN): mov counterI, origM asr counterI, counterI, #4 // counterI = counterI / 16 cmp counterI,#0 - ble .Lstrmm_kernel_L2_M8_BEGIN + ble L(strmm_kernel_L2_M8_BEGIN) -.Lstrmm_kernel_L2_M16_20: +L(strmm_kernel_L2_M16_20): INIT16x2 @@ -1641,10 +1641,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M16_40 + ble L(strmm_kernel_L2_M16_40) .align 5 -.Lstrmm_kernel_L2_M16_22: +L(strmm_kernel_L2_M16_22): KERNEL16x2_SUB KERNEL16x2_SUB KERNEL16x2_SUB @@ -1656,22 +1656,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M16_22 + bgt L(strmm_kernel_L2_M16_22) -.Lstrmm_kernel_L2_M16_40: +L(strmm_kernel_L2_M16_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M16_100 + ble L(strmm_kernel_L2_M16_100) -.Lstrmm_kernel_L2_M16_42: +L(strmm_kernel_L2_M16_42): KERNEL16x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M16_42 + bgt L(strmm_kernel_L2_M16_42) -.Lstrmm_kernel_L2_M16_100: +L(strmm_kernel_L2_M16_100): SAVE16x2 @@ -1691,22 +1691,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #16 #endif -.Lstrmm_kernel_L2_M16_END: +L(strmm_kernel_L2_M16_END): subs counterI, counterI, #1 - bgt .Lstrmm_kernel_L2_M16_20 + bgt L(strmm_kernel_L2_M16_20) //------------------------------------------------------------------------------ -.Lstrmm_kernel_L2_M8_BEGIN: +L(strmm_kernel_L2_M8_BEGIN): mov counterI, origM tst counterI , #15 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) tst counterI, #8 - ble .Lstrmm_kernel_L2_M4_BEGIN + ble L(strmm_kernel_L2_M4_BEGIN) -.Lstrmm_kernel_L2_M8_20: +L(strmm_kernel_L2_M8_20): INIT8x2 @@ -1730,10 +1730,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M8_40 + ble L(strmm_kernel_L2_M8_40) .align 5 -.Lstrmm_kernel_L2_M8_22: +L(strmm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB KERNEL8x2_SUB @@ -1745,22 +1745,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M8_22 + bgt L(strmm_kernel_L2_M8_22) -.Lstrmm_kernel_L2_M8_40: +L(strmm_kernel_L2_M8_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M8_100 + ble L(strmm_kernel_L2_M8_100) -.Lstrmm_kernel_L2_M8_42: +L(strmm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M8_42 + bgt L(strmm_kernel_L2_M8_42) -.Lstrmm_kernel_L2_M8_100: +L(strmm_kernel_L2_M8_100): SAVE8x2 @@ -1780,19 +1780,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #8 #endif -.Lstrmm_kernel_L2_M8_END: +L(strmm_kernel_L2_M8_END): //------------------------------------------------------------------------------ -.Lstrmm_kernel_L2_M4_BEGIN: +L(strmm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) tst counterI, #4 - ble .Lstrmm_kernel_L2_M2_BEGIN + ble L(strmm_kernel_L2_M2_BEGIN) -.Lstrmm_kernel_L2_M4_20: +L(strmm_kernel_L2_M4_20): INIT4x2 @@ -1815,10 +1815,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M4_40 + ble L(strmm_kernel_L2_M4_40) .align 5 -.Lstrmm_kernel_L2_M4_22: +L(strmm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1830,22 +1830,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M4_22 + bgt L(strmm_kernel_L2_M4_22) -.Lstrmm_kernel_L2_M4_40: +L(strmm_kernel_L2_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M4_100 + ble L(strmm_kernel_L2_M4_100) -.Lstrmm_kernel_L2_M4_42: +L(strmm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M4_42 + bgt L(strmm_kernel_L2_M4_42) -.Lstrmm_kernel_L2_M4_100: +L(strmm_kernel_L2_M4_100): SAVE4x2 @@ -1864,21 +1864,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L2_M4_END: +L(strmm_kernel_L2_M4_END): //------------------------------------------------------------------------------ -.Lstrmm_kernel_L2_M2_BEGIN: +L(strmm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L2_M1_BEGIN + ble L(strmm_kernel_L2_M1_BEGIN) -.Lstrmm_kernel_L2_M2_20: +L(strmm_kernel_L2_M2_20): INIT2x2 @@ -1901,9 +1901,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M2_40 + ble L(strmm_kernel_L2_M2_40) -.Lstrmm_kernel_L2_M2_22: +L(strmm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1916,22 +1916,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M2_22 + bgt L(strmm_kernel_L2_M2_22) -.Lstrmm_kernel_L2_M2_40: +L(strmm_kernel_L2_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M2_100 + ble L(strmm_kernel_L2_M2_100) -.Lstrmm_kernel_L2_M2_42: +L(strmm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M2_42 + bgt L(strmm_kernel_L2_M2_42) -.Lstrmm_kernel_L2_M2_100: +L(strmm_kernel_L2_M2_100): SAVE2x2 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) @@ -1950,15 +1950,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L2_M2_END: +L(strmm_kernel_L2_M2_END): -.Lstrmm_kernel_L2_M1_BEGIN: +L(strmm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) -.Lstrmm_kernel_L2_M1_20: +L(strmm_kernel_L2_M1_20): INIT1x2 @@ -1981,9 +1981,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lstrmm_kernel_L2_M1_40 + ble L(strmm_kernel_L2_M1_40) -.Lstrmm_kernel_L2_M1_22: +L(strmm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1995,22 +1995,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M1_22 + bgt L(strmm_kernel_L2_M1_22) -.Lstrmm_kernel_L2_M1_40: +L(strmm_kernel_L2_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M1_100 + ble L(strmm_kernel_L2_M1_100) -.Lstrmm_kernel_L2_M1_42: +L(strmm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M1_42 + bgt L(strmm_kernel_L2_M1_42) -.Lstrmm_kernel_L2_M1_100: +L(strmm_kernel_L2_M1_100): SAVE1x2 @@ -2029,7 +2029,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #1 #endif -.Lstrmm_kernel_L2_END: +L(strmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -2037,11 +2037,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lstrmm_kernel_L1_BEGIN: +L(strmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lstrmm_kernel_L999 // done + ble L(strmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -2052,14 +2052,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = A -.Lstrmm_kernel_L1_M16_BEGIN: +L(strmm_kernel_L1_M16_BEGIN): mov counterI, origM asr counterI, counterI, #4 // counterI = counterI / 16 cmp counterI, #0 - ble .Lstrmm_kernel_L1_M8_BEGIN + ble L(strmm_kernel_L1_M8_BEGIN) -.Lstrmm_kernel_L1_M16_20: +L(strmm_kernel_L1_M16_20): INIT16x1 @@ -2083,10 +2083,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M16_40 + ble L(strmm_kernel_L1_M16_40) .align 5 -.Lstrmm_kernel_L1_M16_22: +L(strmm_kernel_L1_M16_22): KERNEL16x1_SUB KERNEL16x1_SUB KERNEL16x1_SUB @@ -2098,22 +2098,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL16x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M16_22 + bgt L(strmm_kernel_L1_M16_22) -.Lstrmm_kernel_L1_M16_40: +L(strmm_kernel_L1_M16_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M16_100 + ble L(strmm_kernel_L1_M16_100) -.Lstrmm_kernel_L1_M16_42: +L(strmm_kernel_L1_M16_42): KERNEL16x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M16_42 + bgt L(strmm_kernel_L1_M16_42) -.Lstrmm_kernel_L1_M16_100: +L(strmm_kernel_L1_M16_100): SAVE16x1 @@ -2133,23 +2133,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #16 #endif -.Lstrmm_kernel_L1_M16_END: +L(strmm_kernel_L1_M16_END): subs counterI, counterI, #1 - bgt .Lstrmm_kernel_L1_M16_20 + bgt L(strmm_kernel_L1_M16_20) //------------------------------------------------------------------------------ -.Lstrmm_kernel_L1_M8_BEGIN: +L(strmm_kernel_L1_M8_BEGIN): mov counterI, origM tst counterI , #15 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) tst counterI, #8 - ble .Lstrmm_kernel_L1_M4_BEGIN + ble L(strmm_kernel_L1_M4_BEGIN) -.Lstrmm_kernel_L1_M8_20: +L(strmm_kernel_L1_M8_20): INIT8x1 @@ -2173,10 +2173,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M8_40 + ble L(strmm_kernel_L1_M8_40) .align 5 -.Lstrmm_kernel_L1_M8_22: +L(strmm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -2188,22 +2188,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M8_22 + bgt L(strmm_kernel_L1_M8_22) -.Lstrmm_kernel_L1_M8_40: +L(strmm_kernel_L1_M8_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M8_100 + ble L(strmm_kernel_L1_M8_100) -.Lstrmm_kernel_L1_M8_42: +L(strmm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M8_42 + bgt L(strmm_kernel_L1_M8_42) -.Lstrmm_kernel_L1_M8_100: +L(strmm_kernel_L1_M8_100): SAVE8x1 @@ -2223,19 +2223,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #8 #endif -.Lstrmm_kernel_L1_M8_END: +L(strmm_kernel_L1_M8_END): //------------------------------------------------------------------------------ -.Lstrmm_kernel_L1_M4_BEGIN: +L(strmm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) tst counterI, #4 - ble .Lstrmm_kernel_L1_M2_BEGIN + ble L(strmm_kernel_L1_M2_BEGIN) -.Lstrmm_kernel_L1_M4_20: +L(strmm_kernel_L1_M4_20): INIT4x1 @@ -2258,10 +2258,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M4_40 + ble L(strmm_kernel_L1_M4_40) .align 5 -.Lstrmm_kernel_L1_M4_22: +L(strmm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -2273,22 +2273,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M4_22 + bgt L(strmm_kernel_L1_M4_22) -.Lstrmm_kernel_L1_M4_40: +L(strmm_kernel_L1_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M4_100 + ble L(strmm_kernel_L1_M4_100) -.Lstrmm_kernel_L1_M4_42: +L(strmm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M4_42 + bgt L(strmm_kernel_L1_M4_42) -.Lstrmm_kernel_L1_M4_100: +L(strmm_kernel_L1_M4_100): SAVE4x1 @@ -2307,20 +2307,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L1_M4_END: +L(strmm_kernel_L1_M4_END): //------------------------------------------------------------------------------ -.Lstrmm_kernel_L1_M2_BEGIN: +L(strmm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L1_M1_BEGIN + ble L(strmm_kernel_L1_M1_BEGIN) -.Lstrmm_kernel_L1_M2_20: +L(strmm_kernel_L1_M2_20): INIT2x1 @@ -2343,9 +2343,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M2_40 + ble L(strmm_kernel_L1_M2_40) -.Lstrmm_kernel_L1_M2_22: +L(strmm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -2358,22 +2358,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M2_22 + bgt L(strmm_kernel_L1_M2_22) -.Lstrmm_kernel_L1_M2_40: +L(strmm_kernel_L1_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M2_100 + ble L(strmm_kernel_L1_M2_100) -.Lstrmm_kernel_L1_M2_42: +L(strmm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M2_42 + bgt L(strmm_kernel_L1_M2_42) -.Lstrmm_kernel_L1_M2_100: +L(strmm_kernel_L1_M2_100): SAVE2x1 @@ -2392,15 +2392,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L1_M2_END: +L(strmm_kernel_L1_M2_END): -.Lstrmm_kernel_L1_M1_BEGIN: +L(strmm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) -.Lstrmm_kernel_L1_M1_20: +L(strmm_kernel_L1_M1_20): INIT1x1 @@ -2423,9 +2423,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M1_40 + ble L(strmm_kernel_L1_M1_40) -.Lstrmm_kernel_L1_M1_22: +L(strmm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -2437,28 +2437,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M1_22 + bgt L(strmm_kernel_L1_M1_22) -.Lstrmm_kernel_L1_M1_40: +L(strmm_kernel_L1_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M1_100 + ble L(strmm_kernel_L1_M1_100) -.Lstrmm_kernel_L1_M1_42: +L(strmm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M1_42 + bgt L(strmm_kernel_L1_M1_42) -.Lstrmm_kernel_L1_M1_100: +L(strmm_kernel_L1_M1_100): SAVE1x1 -.Lstrmm_kernel_L1_END: +L(strmm_kernel_L1_END): -.Lstrmm_kernel_L999: +L(strmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/strmm_kernel_4x4.S b/kernel/arm64/strmm_kernel_4x4.S index 5f7818c40a..d0cd0129fd 100644 --- a/kernel/arm64/strmm_kernel_4x4.S +++ b/kernel/arm64/strmm_kernel_4x4.S @@ -507,7 +507,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE -.Lstrmm_kernel_begin: +L(strmm_kernel_begin): .align 5 add sp, sp, #-(11 * 16) @@ -539,11 +539,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lstrmm_kernel_L2_BEGIN + ble L(strmm_kernel_L2_BEGIN) /******************************************************************************/ -.Lstrmm_kernel_L4_BEGIN: +L(strmm_kernel_L4_BEGIN): mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #2 @@ -553,14 +553,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lstrmm_kernel_L4_M4_BEGIN: +L(strmm_kernel_L4_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lstrmm_kernel_L4_M2_BEGIN + ble L(strmm_kernel_L4_M2_BEGIN) -.Lstrmm_kernel_L4_M4_20: +L(strmm_kernel_L4_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -581,54 +581,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lstrmm_kernel_L4_M4_32 + blt L(strmm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lstrmm_kernel_L4_M4_22a + ble L(strmm_kernel_L4_M4_22a) .align 5 -.Lstrmm_kernel_L4_M4_22: +L(strmm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M4_22 + bgt L(strmm_kernel_L4_M4_22) -.Lstrmm_kernel_L4_M4_22a: +L(strmm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lstrmm_kernel_L4_M4_44 + b L(strmm_kernel_L4_M4_44) -.Lstrmm_kernel_L4_M4_32: +L(strmm_kernel_L4_M4_32): tst counterL, #1 - ble .Lstrmm_kernel_L4_M4_40 + ble L(strmm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lstrmm_kernel_L4_M4_44 + b L(strmm_kernel_L4_M4_44) -.Lstrmm_kernel_L4_M4_40: +L(strmm_kernel_L4_M4_40): INIT4x4 -.Lstrmm_kernel_L4_M4_44: +L(strmm_kernel_L4_M4_44): ands counterL , tempK, #1 - ble .Lstrmm_kernel_L4_M4_100 + ble L(strmm_kernel_L4_M4_100) -.Lstrmm_kernel_L4_M4_46: +L(strmm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lstrmm_kernel_L4_M4_100: +L(strmm_kernel_L4_M4_100): SAVE4x4 @@ -647,20 +647,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L4_M4_END: +L(strmm_kernel_L4_M4_END): subs counterI, counterI, #1 - bne .Lstrmm_kernel_L4_M4_20 + bne L(strmm_kernel_L4_M4_20) -.Lstrmm_kernel_L4_M2_BEGIN: +L(strmm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L4_M1_BEGIN + ble L(strmm_kernel_L4_M1_BEGIN) -.Lstrmm_kernel_L4_M2_20: +L(strmm_kernel_L4_M2_20): INIT2x4 @@ -684,9 +684,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L4_M2_40 + ble L(strmm_kernel_L4_M2_40) -.Lstrmm_kernel_L4_M2_22: +L(strmm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -699,22 +699,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M2_22 + bgt L(strmm_kernel_L4_M2_22) -.Lstrmm_kernel_L4_M2_40: +L(strmm_kernel_L4_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L4_M2_100 + ble L(strmm_kernel_L4_M2_100) -.Lstrmm_kernel_L4_M2_42: +L(strmm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M2_42 + bgt L(strmm_kernel_L4_M2_42) -.Lstrmm_kernel_L4_M2_100: +L(strmm_kernel_L4_M2_100): SAVE2x4 @@ -735,15 +735,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.Lstrmm_kernel_L4_M2_END: +L(strmm_kernel_L4_M2_END): -.Lstrmm_kernel_L4_M1_BEGIN: +L(strmm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) -.Lstrmm_kernel_L4_M1_20: +L(strmm_kernel_L4_M1_20): INIT1x4 @@ -767,9 +767,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L4_M1_40 + ble L(strmm_kernel_L4_M1_40) -.Lstrmm_kernel_L4_M1_22: +L(strmm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -781,22 +781,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M1_22 + bgt L(strmm_kernel_L4_M1_22) -.Lstrmm_kernel_L4_M1_40: +L(strmm_kernel_L4_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L4_M1_100 + ble L(strmm_kernel_L4_M1_100) -.Lstrmm_kernel_L4_M1_42: +L(strmm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M1_42 + bgt L(strmm_kernel_L4_M1_42) -.Lstrmm_kernel_L4_M1_100: +L(strmm_kernel_L4_M1_100): SAVE1x4 @@ -817,7 +817,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.Lstrmm_kernel_L4_END: +L(strmm_kernel_L4_END): add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4 #if !defined(LEFT) @@ -825,19 +825,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Lstrmm_kernel_L4_BEGIN + bgt L(strmm_kernel_L4_BEGIN) /******************************************************************************/ -.Lstrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(strmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lstrmm_kernel_L999 + ble L(strmm_kernel_L999) tst counterJ , #2 - ble .Lstrmm_kernel_L1_BEGIN + ble L(strmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -849,14 +849,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lstrmm_kernel_L2_M4_BEGIN: +L(strmm_kernel_L2_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI,#0 - ble .Lstrmm_kernel_L2_M2_BEGIN + ble L(strmm_kernel_L2_M2_BEGIN) -.Lstrmm_kernel_L2_M4_20: +L(strmm_kernel_L2_M4_20): INIT4x2 @@ -880,10 +880,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M4_40 + ble L(strmm_kernel_L2_M4_40) .align 5 -.Lstrmm_kernel_L2_M4_22: +L(strmm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -895,22 +895,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M4_22 + bgt L(strmm_kernel_L2_M4_22) -.Lstrmm_kernel_L2_M4_40: +L(strmm_kernel_L2_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M4_100 + ble L(strmm_kernel_L2_M4_100) -.Lstrmm_kernel_L2_M4_42: +L(strmm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M4_42 + bgt L(strmm_kernel_L2_M4_42) -.Lstrmm_kernel_L2_M4_100: +L(strmm_kernel_L2_M4_100): SAVE4x2 @@ -930,22 +930,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L2_M4_END: +L(strmm_kernel_L2_M4_END): subs counterI, counterI, #1 - bgt .Lstrmm_kernel_L2_M4_20 + bgt L(strmm_kernel_L2_M4_20) -.Lstrmm_kernel_L2_M2_BEGIN: +L(strmm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L2_M1_BEGIN + ble L(strmm_kernel_L2_M1_BEGIN) -.Lstrmm_kernel_L2_M2_20: +L(strmm_kernel_L2_M2_20): INIT2x2 @@ -969,9 +969,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M2_40 + ble L(strmm_kernel_L2_M2_40) -.Lstrmm_kernel_L2_M2_22: +L(strmm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -984,22 +984,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M2_22 + bgt L(strmm_kernel_L2_M2_22) -.Lstrmm_kernel_L2_M2_40: +L(strmm_kernel_L2_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M2_100 + ble L(strmm_kernel_L2_M2_100) -.Lstrmm_kernel_L2_M2_42: +L(strmm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M2_42 + bgt L(strmm_kernel_L2_M2_42) -.Lstrmm_kernel_L2_M2_100: +L(strmm_kernel_L2_M2_100): SAVE2x2 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) @@ -1018,15 +1018,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L2_M2_END: +L(strmm_kernel_L2_M2_END): -.Lstrmm_kernel_L2_M1_BEGIN: +L(strmm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) -.Lstrmm_kernel_L2_M1_20: +L(strmm_kernel_L2_M1_20): INIT1x2 @@ -1050,9 +1050,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lstrmm_kernel_L2_M1_40 + ble L(strmm_kernel_L2_M1_40) -.Lstrmm_kernel_L2_M1_22: +L(strmm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1064,22 +1064,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M1_22 + bgt L(strmm_kernel_L2_M1_22) -.Lstrmm_kernel_L2_M1_40: +L(strmm_kernel_L2_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M1_100 + ble L(strmm_kernel_L2_M1_100) -.Lstrmm_kernel_L2_M1_42: +L(strmm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M1_42 + bgt L(strmm_kernel_L2_M1_42) -.Lstrmm_kernel_L2_M1_100: +L(strmm_kernel_L2_M1_100): SAVE1x2 @@ -1099,7 +1099,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Lstrmm_kernel_L2_END: +L(strmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -1107,11 +1107,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lstrmm_kernel_L1_BEGIN: +L(strmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lstrmm_kernel_L999 // done + ble L(strmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1123,14 +1123,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lstrmm_kernel_L1_M4_BEGIN: +L(strmm_kernel_L1_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lstrmm_kernel_L1_M2_BEGIN + ble L(strmm_kernel_L1_M2_BEGIN) -.Lstrmm_kernel_L1_M4_20: +L(strmm_kernel_L1_M4_20): INIT4x1 @@ -1154,10 +1154,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M4_40 + ble L(strmm_kernel_L1_M4_40) .align 5 -.Lstrmm_kernel_L1_M4_22: +L(strmm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1169,22 +1169,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M4_22 + bgt L(strmm_kernel_L1_M4_22) -.Lstrmm_kernel_L1_M4_40: +L(strmm_kernel_L1_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M4_100 + ble L(strmm_kernel_L1_M4_100) -.Lstrmm_kernel_L1_M4_42: +L(strmm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M4_42 + bgt L(strmm_kernel_L1_M4_42) -.Lstrmm_kernel_L1_M4_100: +L(strmm_kernel_L1_M4_100): SAVE4x1 @@ -1204,22 +1204,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L1_M4_END: +L(strmm_kernel_L1_M4_END): subs counterI, counterI, #1 - bgt .Lstrmm_kernel_L1_M4_20 + bgt L(strmm_kernel_L1_M4_20) -.Lstrmm_kernel_L1_M2_BEGIN: +L(strmm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L1_M1_BEGIN + ble L(strmm_kernel_L1_M1_BEGIN) -.Lstrmm_kernel_L1_M2_20: +L(strmm_kernel_L1_M2_20): INIT2x1 @@ -1243,9 +1243,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M2_40 + ble L(strmm_kernel_L1_M2_40) -.Lstrmm_kernel_L1_M2_22: +L(strmm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1258,22 +1258,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M2_22 + bgt L(strmm_kernel_L1_M2_22) -.Lstrmm_kernel_L1_M2_40: +L(strmm_kernel_L1_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M2_100 + ble L(strmm_kernel_L1_M2_100) -.Lstrmm_kernel_L1_M2_42: +L(strmm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M2_42 + bgt L(strmm_kernel_L1_M2_42) -.Lstrmm_kernel_L1_M2_100: +L(strmm_kernel_L1_M2_100): SAVE2x1 @@ -1294,15 +1294,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.Lstrmm_kernel_L1_M2_END: +L(strmm_kernel_L1_M2_END): -.Lstrmm_kernel_L1_M1_BEGIN: +L(strmm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) -.Lstrmm_kernel_L1_M1_20: +L(strmm_kernel_L1_M1_20): INIT1x1 @@ -1326,9 +1326,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M1_40 + ble L(strmm_kernel_L1_M1_40) -.Lstrmm_kernel_L1_M1_22: +L(strmm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1340,22 +1340,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M1_22 + bgt L(strmm_kernel_L1_M1_22) -.Lstrmm_kernel_L1_M1_40: +L(strmm_kernel_L1_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M1_100 + ble L(strmm_kernel_L1_M1_100) -.Lstrmm_kernel_L1_M1_42: +L(strmm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M1_42 + bgt L(strmm_kernel_L1_M1_42) -.Lstrmm_kernel_L1_M1_100: +L(strmm_kernel_L1_M1_100): SAVE1x1 @@ -1377,7 +1377,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif #endif -.Lstrmm_kernel_L1_END: +L(strmm_kernel_L1_END): #if 0 #if !defined(LEFT) @@ -1385,7 +1385,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif #endif -.Lstrmm_kernel_L999: +L(strmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/strmm_kernel_8x8.S b/kernel/arm64/strmm_kernel_8x8.S index cd18e6847d..761bea99da 100644 --- a/kernel/arm64/strmm_kernel_8x8.S +++ b/kernel/arm64/strmm_kernel_8x8.S @@ -1257,7 +1257,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE -.Lstrmm_kernel_begin: +L(strmm_kernel_begin): .align 5 add sp, sp, #-(11 * 16) @@ -1288,12 +1288,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Lstrmm_kernel_L4_BEGIN + ble L(strmm_kernel_L4_BEGIN) /******************************************************************************/ /******************************************************************************/ -.Lstrmm_kernel_L8_BEGIN: +L(strmm_kernel_L8_BEGIN): mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #3 @@ -1305,14 +1305,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lstrmm_kernel_L8_M8_BEGIN: +L(strmm_kernel_L8_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lstrmm_kernel_L8_M4_BEGIN + ble L(strmm_kernel_L8_M4_BEGIN) -.Lstrmm_kernel_L8_M8_20: +L(strmm_kernel_L8_M8_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1333,54 +1333,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lstrmm_kernel_L8_M8_32 + blt L(strmm_kernel_L8_M8_32) KERNEL8x8_I // do one in the K KERNEL8x8_M2 // do another in the K subs counterL, counterL, #2 - ble .Lstrmm_kernel_L8_M8_22a + ble L(strmm_kernel_L8_M8_22a) .align 5 -.Lstrmm_kernel_L8_M8_22: +L(strmm_kernel_L8_M8_22): KERNEL8x8_M1 KERNEL8x8_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M8_22 + bgt L(strmm_kernel_L8_M8_22) -.Lstrmm_kernel_L8_M8_22a: +L(strmm_kernel_L8_M8_22a): KERNEL8x8_M1 KERNEL8x8_E - b .Lstrmm_kernel_L8_M8_44 + b L(strmm_kernel_L8_M8_44) -.Lstrmm_kernel_L8_M8_32: +L(strmm_kernel_L8_M8_32): tst counterL, #1 - ble .Lstrmm_kernel_L8_M8_40 + ble L(strmm_kernel_L8_M8_40) KERNEL8x8_I KERNEL8x8_E - b .Lstrmm_kernel_L8_M8_44 + b L(strmm_kernel_L8_M8_44) -.Lstrmm_kernel_L8_M8_40: +L(strmm_kernel_L8_M8_40): INIT8x8 -.Lstrmm_kernel_L8_M8_44: +L(strmm_kernel_L8_M8_44): ands counterL , tempK, #1 - ble .Lstrmm_kernel_L8_M8_100 + ble L(strmm_kernel_L8_M8_100) -.Lstrmm_kernel_L8_M8_46: +L(strmm_kernel_L8_M8_46): KERNEL8x8_SUB -.Lstrmm_kernel_L8_M8_100: +L(strmm_kernel_L8_M8_100): SAVE8x8 @@ -1399,22 +1399,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #8 #endif -.Lstrmm_kernel_L8_M8_END: +L(strmm_kernel_L8_M8_END): subs counterI, counterI, #1 - bne .Lstrmm_kernel_L8_M8_20 + bne L(strmm_kernel_L8_M8_20) /******************************************************************************/ -.Lstrmm_kernel_L8_M4_BEGIN: +L(strmm_kernel_L8_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lstrmm_kernel_L8_END + ble L(strmm_kernel_L8_END) tst counterI, #4 - ble .Lstrmm_kernel_L8_M2_BEGIN + ble L(strmm_kernel_L8_M2_BEGIN) -.Lstrmm_kernel_L8_M4_20: +L(strmm_kernel_L8_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1436,54 +1436,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lstrmm_kernel_L8_M4_32 + blt L(strmm_kernel_L8_M4_32) KERNEL4x8_I // do one in the K KERNEL4x8_M2 // do another in the K subs counterL, counterL, #2 - ble .Lstrmm_kernel_L8_M4_22a + ble L(strmm_kernel_L8_M4_22a) .align 5 -.Lstrmm_kernel_L8_M4_22: +L(strmm_kernel_L8_M4_22): KERNEL4x8_M1 KERNEL4x8_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M4_22 + bgt L(strmm_kernel_L8_M4_22) -.Lstrmm_kernel_L8_M4_22a: +L(strmm_kernel_L8_M4_22a): KERNEL4x8_M1 KERNEL4x8_E - b .Lstrmm_kernel_L8_M4_44 + b L(strmm_kernel_L8_M4_44) -.Lstrmm_kernel_L8_M4_32: +L(strmm_kernel_L8_M4_32): tst counterL, #1 - ble .Lstrmm_kernel_L8_M4_40 + ble L(strmm_kernel_L8_M4_40) KERNEL4x8_I KERNEL4x8_E - b .Lstrmm_kernel_L8_M4_44 + b L(strmm_kernel_L8_M4_44) -.Lstrmm_kernel_L8_M4_40: +L(strmm_kernel_L8_M4_40): INIT4x8 -.Lstrmm_kernel_L8_M4_44: +L(strmm_kernel_L8_M4_44): ands counterL , tempK, #1 - ble .Lstrmm_kernel_L8_M4_100 + ble L(strmm_kernel_L8_M4_100) -.Lstrmm_kernel_L8_M4_46: +L(strmm_kernel_L8_M4_46): KERNEL4x8_SUB -.Lstrmm_kernel_L8_M4_100: +L(strmm_kernel_L8_M4_100): SAVE4x8 @@ -1503,20 +1503,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L8_M4_END: +L(strmm_kernel_L8_M4_END): /******************************************************************************/ -.Lstrmm_kernel_L8_M2_BEGIN: +L(strmm_kernel_L8_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L8_END + ble L(strmm_kernel_L8_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L8_M1_BEGIN + ble L(strmm_kernel_L8_M1_BEGIN) -.Lstrmm_kernel_L8_M2_20: +L(strmm_kernel_L8_M2_20): INIT2x8 @@ -1540,9 +1540,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L8_M2_40 + ble L(strmm_kernel_L8_M2_40) -.Lstrmm_kernel_L8_M2_22: +L(strmm_kernel_L8_M2_22): KERNEL2x8_SUB KERNEL2x8_SUB @@ -1555,22 +1555,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M2_22 + bgt L(strmm_kernel_L8_M2_22) -.Lstrmm_kernel_L8_M2_40: +L(strmm_kernel_L8_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L8_M2_100 + ble L(strmm_kernel_L8_M2_100) -.Lstrmm_kernel_L8_M2_42: +L(strmm_kernel_L8_M2_42): KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M2_42 + bgt L(strmm_kernel_L8_M2_42) -.Lstrmm_kernel_L8_M2_100: +L(strmm_kernel_L8_M2_100): SAVE2x8 @@ -1590,16 +1590,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L8_M2_END: +L(strmm_kernel_L8_M2_END): /******************************************************************************/ -.Lstrmm_kernel_L8_M1_BEGIN: +L(strmm_kernel_L8_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L8_END + ble L(strmm_kernel_L8_END) -.Lstrmm_kernel_L8_M1_20: +L(strmm_kernel_L8_M1_20): INIT1x8 @@ -1623,9 +1623,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L8_M1_40 + ble L(strmm_kernel_L8_M1_40) -.Lstrmm_kernel_L8_M1_22: +L(strmm_kernel_L8_M1_22): KERNEL1x8_SUB KERNEL1x8_SUB KERNEL1x8_SUB @@ -1637,22 +1637,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M1_22 + bgt L(strmm_kernel_L8_M1_22) -.Lstrmm_kernel_L8_M1_40: +L(strmm_kernel_L8_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L8_M1_100 + ble L(strmm_kernel_L8_M1_100) -.Lstrmm_kernel_L8_M1_42: +L(strmm_kernel_L8_M1_42): KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M1_42 + bgt L(strmm_kernel_L8_M1_42) -.Lstrmm_kernel_L8_M1_100: +L(strmm_kernel_L8_M1_100): SAVE1x8 @@ -1672,7 +1672,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Lstrmm_kernel_L8_END: +L(strmm_kernel_L8_END): lsl temp, origK, #5 // B = B + K * 4 * 8 add origPB, origPB, temp @@ -1681,19 +1681,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Lstrmm_kernel_L8_BEGIN + bgt L(strmm_kernel_L8_BEGIN) /******************************************************************************/ /******************************************************************************/ -.Lstrmm_kernel_L4_BEGIN: +L(strmm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #7 - ble .Lstrmm_kernel_L999 + ble L(strmm_kernel_L999) tst counterJ , #4 - ble .Lstrmm_kernel_L2_BEGIN + ble L(strmm_kernel_L2_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1707,14 +1707,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lstrmm_kernel_L4_M8_BEGIN: +L(strmm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lstrmm_kernel_L4_M4_BEGIN + ble L(strmm_kernel_L4_M4_BEGIN) -.Lstrmm_kernel_L4_M8_20: +L(strmm_kernel_L4_M8_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1736,54 +1736,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lstrmm_kernel_L4_M8_32 + blt L(strmm_kernel_L4_M8_32) KERNEL8x4_I // do one in the K KERNEL8x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lstrmm_kernel_L4_M8_22a + ble L(strmm_kernel_L4_M8_22a) .align 5 -.Lstrmm_kernel_L4_M8_22: +L(strmm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M8_22 + bgt L(strmm_kernel_L4_M8_22) -.Lstrmm_kernel_L4_M8_22a: +L(strmm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_E - b .Lstrmm_kernel_L4_M8_44 + b L(strmm_kernel_L4_M8_44) -.Lstrmm_kernel_L4_M8_32: +L(strmm_kernel_L4_M8_32): tst counterL, #1 - ble .Lstrmm_kernel_L4_M8_40 + ble L(strmm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_E - b .Lstrmm_kernel_L4_M8_44 + b L(strmm_kernel_L4_M8_44) -.Lstrmm_kernel_L4_M8_40: +L(strmm_kernel_L4_M8_40): INIT8x4 -.Lstrmm_kernel_L4_M8_44: +L(strmm_kernel_L4_M8_44): ands counterL , tempK, #1 - ble .Lstrmm_kernel_L4_M8_100 + ble L(strmm_kernel_L4_M8_100) -.Lstrmm_kernel_L4_M8_46: +L(strmm_kernel_L4_M8_46): KERNEL8x4_SUB -.Lstrmm_kernel_L4_M8_100: +L(strmm_kernel_L4_M8_100): SAVE8x4 @@ -1802,22 +1802,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #8 #endif -.Lstrmm_kernel_L4_M8_END: +L(strmm_kernel_L4_M8_END): subs counterI, counterI, #1 - bne .Lstrmm_kernel_L4_M8_20 + bne L(strmm_kernel_L4_M8_20) /******************************************************************************/ -.Lstrmm_kernel_L4_M4_BEGIN: +L(strmm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) tst counterI, #4 - ble .Lstrmm_kernel_L4_M2_BEGIN + ble L(strmm_kernel_L4_M2_BEGIN) -.Lstrmm_kernel_L4_M4_20: +L(strmm_kernel_L4_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1837,54 +1837,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lstrmm_kernel_L4_M4_32 + blt L(strmm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lstrmm_kernel_L4_M4_22a + ble L(strmm_kernel_L4_M4_22a) .align 5 -.Lstrmm_kernel_L4_M4_22: +L(strmm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M4_22 + bgt L(strmm_kernel_L4_M4_22) -.Lstrmm_kernel_L4_M4_22a: +L(strmm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lstrmm_kernel_L4_M4_44 + b L(strmm_kernel_L4_M4_44) -.Lstrmm_kernel_L4_M4_32: +L(strmm_kernel_L4_M4_32): tst counterL, #1 - ble .Lstrmm_kernel_L4_M4_40 + ble L(strmm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lstrmm_kernel_L4_M4_44 + b L(strmm_kernel_L4_M4_44) -.Lstrmm_kernel_L4_M4_40: +L(strmm_kernel_L4_M4_40): INIT4x4 -.Lstrmm_kernel_L4_M4_44: +L(strmm_kernel_L4_M4_44): ands counterL , tempK, #1 - ble .Lstrmm_kernel_L4_M4_100 + ble L(strmm_kernel_L4_M4_100) -.Lstrmm_kernel_L4_M4_46: +L(strmm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lstrmm_kernel_L4_M4_100: +L(strmm_kernel_L4_M4_100): SAVE4x4 @@ -1902,20 +1902,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L4_M4_END: +L(strmm_kernel_L4_M4_END): /******************************************************************************/ -.Lstrmm_kernel_L4_M2_BEGIN: +L(strmm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L4_M1_BEGIN + ble L(strmm_kernel_L4_M1_BEGIN) -.Lstrmm_kernel_L4_M2_20: +L(strmm_kernel_L4_M2_20): INIT2x4 @@ -1938,9 +1938,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L4_M2_40 + ble L(strmm_kernel_L4_M2_40) -.Lstrmm_kernel_L4_M2_22: +L(strmm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1953,22 +1953,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M2_22 + bgt L(strmm_kernel_L4_M2_22) -.Lstrmm_kernel_L4_M2_40: +L(strmm_kernel_L4_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L4_M2_100 + ble L(strmm_kernel_L4_M2_100) -.Lstrmm_kernel_L4_M2_42: +L(strmm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M2_42 + bgt L(strmm_kernel_L4_M2_42) -.Lstrmm_kernel_L4_M2_100: +L(strmm_kernel_L4_M2_100): SAVE2x4 @@ -1987,16 +1987,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L4_M2_END: +L(strmm_kernel_L4_M2_END): /******************************************************************************/ -.Lstrmm_kernel_L4_M1_BEGIN: +L(strmm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) -.Lstrmm_kernel_L4_M1_20: +L(strmm_kernel_L4_M1_20): INIT1x4 @@ -2019,9 +2019,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L4_M1_40 + ble L(strmm_kernel_L4_M1_40) -.Lstrmm_kernel_L4_M1_22: +L(strmm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -2033,22 +2033,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M1_22 + bgt L(strmm_kernel_L4_M1_22) -.Lstrmm_kernel_L4_M1_40: +L(strmm_kernel_L4_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L4_M1_100 + ble L(strmm_kernel_L4_M1_100) -.Lstrmm_kernel_L4_M1_42: +L(strmm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M1_42 + bgt L(strmm_kernel_L4_M1_42) -.Lstrmm_kernel_L4_M1_100: +L(strmm_kernel_L4_M1_100): SAVE1x4 @@ -2067,7 +2067,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #1 #endif -.Lstrmm_kernel_L4_END: +L(strmm_kernel_L4_END): add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4 #if !defined(LEFT) add tempOffset, tempOffset, #4 @@ -2076,14 +2076,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ /******************************************************************************/ -.Lstrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(strmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lstrmm_kernel_L999 + ble L(strmm_kernel_L999) tst counterJ , #2 - ble .Lstrmm_kernel_L1_BEGIN + ble L(strmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -2096,14 +2096,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lstrmm_kernel_L2_M8_BEGIN: +L(strmm_kernel_L2_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI,#0 - ble .Lstrmm_kernel_L2_M4_BEGIN + ble L(strmm_kernel_L2_M4_BEGIN) -.Lstrmm_kernel_L2_M8_20: +L(strmm_kernel_L2_M8_20): INIT8x2 @@ -2126,10 +2126,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M8_40 + ble L(strmm_kernel_L2_M8_40) .align 5 -.Lstrmm_kernel_L2_M8_22: +L(strmm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB KERNEL8x2_SUB @@ -2141,22 +2141,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M8_22 + bgt L(strmm_kernel_L2_M8_22) -.Lstrmm_kernel_L2_M8_40: +L(strmm_kernel_L2_M8_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M8_100 + ble L(strmm_kernel_L2_M8_100) -.Lstrmm_kernel_L2_M8_42: +L(strmm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M8_42 + bgt L(strmm_kernel_L2_M8_42) -.Lstrmm_kernel_L2_M8_100: +L(strmm_kernel_L2_M8_100): SAVE8x2 @@ -2175,23 +2175,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #8 #endif -.Lstrmm_kernel_L2_M8_END: +L(strmm_kernel_L2_M8_END): subs counterI, counterI, #1 - bgt .Lstrmm_kernel_L2_M8_20 + bgt L(strmm_kernel_L2_M8_20) /******************************************************************************/ -.Lstrmm_kernel_L2_M4_BEGIN: +L(strmm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) tst counterI, #4 - ble .Lstrmm_kernel_L2_M2_BEGIN + ble L(strmm_kernel_L2_M2_BEGIN) -.Lstrmm_kernel_L2_M4_20: +L(strmm_kernel_L2_M4_20): INIT4x2 @@ -2214,10 +2214,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M4_40 + ble L(strmm_kernel_L2_M4_40) .align 5 -.Lstrmm_kernel_L2_M4_22: +L(strmm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -2229,22 +2229,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M4_22 + bgt L(strmm_kernel_L2_M4_22) -.Lstrmm_kernel_L2_M4_40: +L(strmm_kernel_L2_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M4_100 + ble L(strmm_kernel_L2_M4_100) -.Lstrmm_kernel_L2_M4_42: +L(strmm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M4_42 + bgt L(strmm_kernel_L2_M4_42) -.Lstrmm_kernel_L2_M4_100: +L(strmm_kernel_L2_M4_100): SAVE4x2 @@ -2263,20 +2263,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L2_M4_END: +L(strmm_kernel_L2_M4_END): /******************************************************************************/ -.Lstrmm_kernel_L2_M2_BEGIN: +L(strmm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L2_M1_BEGIN + ble L(strmm_kernel_L2_M1_BEGIN) -.Lstrmm_kernel_L2_M2_20: +L(strmm_kernel_L2_M2_20): INIT2x2 @@ -2299,9 +2299,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M2_40 + ble L(strmm_kernel_L2_M2_40) -.Lstrmm_kernel_L2_M2_22: +L(strmm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -2314,22 +2314,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M2_22 + bgt L(strmm_kernel_L2_M2_22) -.Lstrmm_kernel_L2_M2_40: +L(strmm_kernel_L2_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M2_100 + ble L(strmm_kernel_L2_M2_100) -.Lstrmm_kernel_L2_M2_42: +L(strmm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M2_42 + bgt L(strmm_kernel_L2_M2_42) -.Lstrmm_kernel_L2_M2_100: +L(strmm_kernel_L2_M2_100): SAVE2x2 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) @@ -2348,16 +2348,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L2_M2_END: +L(strmm_kernel_L2_M2_END): /******************************************************************************/ -.Lstrmm_kernel_L2_M1_BEGIN: +L(strmm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) -.Lstrmm_kernel_L2_M1_20: +L(strmm_kernel_L2_M1_20): INIT1x2 @@ -2380,9 +2380,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lstrmm_kernel_L2_M1_40 + ble L(strmm_kernel_L2_M1_40) -.Lstrmm_kernel_L2_M1_22: +L(strmm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -2394,22 +2394,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M1_22 + bgt L(strmm_kernel_L2_M1_22) -.Lstrmm_kernel_L2_M1_40: +L(strmm_kernel_L2_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M1_100 + ble L(strmm_kernel_L2_M1_100) -.Lstrmm_kernel_L2_M1_42: +L(strmm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M1_42 + bgt L(strmm_kernel_L2_M1_42) -.Lstrmm_kernel_L2_M1_100: +L(strmm_kernel_L2_M1_100): SAVE1x2 @@ -2428,7 +2428,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #1 #endif -.Lstrmm_kernel_L2_END: +L(strmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -2437,11 +2437,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ /******************************************************************************/ -.Lstrmm_kernel_L1_BEGIN: +L(strmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lstrmm_kernel_L999 // done + ble L(strmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -2454,14 +2454,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lstrmm_kernel_L1_M8_BEGIN: +L(strmm_kernel_L1_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 cmp counterI, #0 - ble .Lstrmm_kernel_L1_M4_BEGIN + ble L(strmm_kernel_L1_M4_BEGIN) -.Lstrmm_kernel_L1_M8_20: +L(strmm_kernel_L1_M8_20): INIT8x1 @@ -2484,10 +2484,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M8_40 + ble L(strmm_kernel_L1_M8_40) .align 5 -.Lstrmm_kernel_L1_M8_22: +L(strmm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -2499,22 +2499,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M8_22 + bgt L(strmm_kernel_L1_M8_22) -.Lstrmm_kernel_L1_M8_40: +L(strmm_kernel_L1_M8_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M8_100 + ble L(strmm_kernel_L1_M8_100) -.Lstrmm_kernel_L1_M8_42: +L(strmm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M8_42 + bgt L(strmm_kernel_L1_M8_42) -.Lstrmm_kernel_L1_M8_100: +L(strmm_kernel_L1_M8_100): SAVE8x1 @@ -2533,23 +2533,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #8 #endif -.Lstrmm_kernel_L1_M8_END: +L(strmm_kernel_L1_M8_END): subs counterI, counterI, #1 - bgt .Lstrmm_kernel_L1_M8_20 + bgt L(strmm_kernel_L1_M8_20) /******************************************************************************/ -.Lstrmm_kernel_L1_M4_BEGIN: +L(strmm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) tst counterI, #4 - ble .Lstrmm_kernel_L1_M2_BEGIN + ble L(strmm_kernel_L1_M2_BEGIN) -.Lstrmm_kernel_L1_M4_20: +L(strmm_kernel_L1_M4_20): INIT4x1 @@ -2572,10 +2572,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M4_40 + ble L(strmm_kernel_L1_M4_40) .align 5 -.Lstrmm_kernel_L1_M4_22: +L(strmm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -2587,22 +2587,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M4_22 + bgt L(strmm_kernel_L1_M4_22) -.Lstrmm_kernel_L1_M4_40: +L(strmm_kernel_L1_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M4_100 + ble L(strmm_kernel_L1_M4_100) -.Lstrmm_kernel_L1_M4_42: +L(strmm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M4_42 + bgt L(strmm_kernel_L1_M4_42) -.Lstrmm_kernel_L1_M4_100: +L(strmm_kernel_L1_M4_100): SAVE4x1 @@ -2621,20 +2621,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L1_M4_END: +L(strmm_kernel_L1_M4_END): /******************************************************************************/ -.Lstrmm_kernel_L1_M2_BEGIN: +L(strmm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L1_M1_BEGIN + ble L(strmm_kernel_L1_M1_BEGIN) -.Lstrmm_kernel_L1_M2_20: +L(strmm_kernel_L1_M2_20): INIT2x1 @@ -2657,9 +2657,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M2_40 + ble L(strmm_kernel_L1_M2_40) -.Lstrmm_kernel_L1_M2_22: +L(strmm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -2672,22 +2672,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M2_22 + bgt L(strmm_kernel_L1_M2_22) -.Lstrmm_kernel_L1_M2_40: +L(strmm_kernel_L1_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M2_100 + ble L(strmm_kernel_L1_M2_100) -.Lstrmm_kernel_L1_M2_42: +L(strmm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M2_42 + bgt L(strmm_kernel_L1_M2_42) -.Lstrmm_kernel_L1_M2_100: +L(strmm_kernel_L1_M2_100): SAVE2x1 @@ -2706,16 +2706,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L1_M2_END: +L(strmm_kernel_L1_M2_END): /******************************************************************************/ -.Lstrmm_kernel_L1_M1_BEGIN: +L(strmm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) -.Lstrmm_kernel_L1_M1_20: +L(strmm_kernel_L1_M1_20): INIT1x1 @@ -2738,9 +2738,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M1_40 + ble L(strmm_kernel_L1_M1_40) -.Lstrmm_kernel_L1_M1_22: +L(strmm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -2752,30 +2752,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M1_22 + bgt L(strmm_kernel_L1_M1_22) -.Lstrmm_kernel_L1_M1_40: +L(strmm_kernel_L1_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M1_100 + ble L(strmm_kernel_L1_M1_100) -.Lstrmm_kernel_L1_M1_42: +L(strmm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M1_42 + bgt L(strmm_kernel_L1_M1_42) -.Lstrmm_kernel_L1_M1_100: +L(strmm_kernel_L1_M1_100): SAVE1x1 -.Lstrmm_kernel_L1_END: +L(strmm_kernel_L1_END): /******************************************************************************/ -.Lstrmm_kernel_L999: +L(strmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/strmm_kernel_8x8_cortexa53.S b/kernel/arm64/strmm_kernel_8x8_cortexa53.S index 4b84623f32..e65a2b9981 100644 --- a/kernel/arm64/strmm_kernel_8x8_cortexa53.S +++ b/kernel/arm64/strmm_kernel_8x8_cortexa53.S @@ -1258,7 +1258,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE -.Lstrmm_kernel_begin: +L(strmm_kernel_begin): .align 5 add sp, sp, #-(11 * 16) @@ -1289,12 +1289,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Lstrmm_kernel_L4_BEGIN + ble L(strmm_kernel_L4_BEGIN) /******************************************************************************/ /******************************************************************************/ -.Lstrmm_kernel_L8_BEGIN: +L(strmm_kernel_L8_BEGIN): mov pCRow0, pC // pCRow0 = C add pC, pC, LDC, lsl #3 @@ -1306,14 +1306,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lstrmm_kernel_L8_M8_BEGIN: +L(strmm_kernel_L8_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lstrmm_kernel_L8_M4_BEGIN + ble L(strmm_kernel_L8_M4_BEGIN) -.Lstrmm_kernel_L8_M8_20: +L(strmm_kernel_L8_M8_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1334,7 +1334,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // L = K / 8 cmp counterL , #2 // is there at least 16 to do? - blt .Lstrmm_kernel_L8_M8_32 + blt L(strmm_kernel_L8_M8_32) KERNEL8x8_I // do one in the K KERNEL8x8_M2 // do another in the K @@ -1346,10 +1346,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x8_M2 subs counterL, counterL, #2 - ble .Lstrmm_kernel_L8_M8_22a + ble L(strmm_kernel_L8_M8_22a) .align 5 -.Lstrmm_kernel_L8_M8_22: +L(strmm_kernel_L8_M8_22): KERNEL8x8_M1 KERNEL8x8_M2 @@ -1361,9 +1361,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x8_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M8_22 + bgt L(strmm_kernel_L8_M8_22) -.Lstrmm_kernel_L8_M8_22a: +L(strmm_kernel_L8_M8_22a): KERNEL8x8_M1 KERNEL8x8_M2 @@ -1374,12 +1374,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x8_M1 KERNEL8x8_E - b .Lstrmm_kernel_L8_M8_44 + b L(strmm_kernel_L8_M8_44) -.Lstrmm_kernel_L8_M8_32: +L(strmm_kernel_L8_M8_32): tst counterL, #1 - ble .Lstrmm_kernel_L8_M8_40 + ble L(strmm_kernel_L8_M8_40) KERNEL8x8_I KERNEL8x8_M2 @@ -1390,25 +1390,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x8_M1 KERNEL8x8_E - b .Lstrmm_kernel_L8_M8_44 + b L(strmm_kernel_L8_M8_44) -.Lstrmm_kernel_L8_M8_40: +L(strmm_kernel_L8_M8_40): INIT8x8 -.Lstrmm_kernel_L8_M8_44: +L(strmm_kernel_L8_M8_44): ands counterL , tempK, #7 - ble .Lstrmm_kernel_L8_M8_100 + ble L(strmm_kernel_L8_M8_100) -.Lstrmm_kernel_L8_M8_46: +L(strmm_kernel_L8_M8_46): KERNEL8x8_SUB subs counterL, counterL, 1 - bgt .Lstrmm_kernel_L8_M8_46 + bgt L(strmm_kernel_L8_M8_46) -.Lstrmm_kernel_L8_M8_100: +L(strmm_kernel_L8_M8_100): SAVE8x8 @@ -1427,22 +1427,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #8 #endif -.Lstrmm_kernel_L8_M8_END: +L(strmm_kernel_L8_M8_END): subs counterI, counterI, #1 - bne .Lstrmm_kernel_L8_M8_20 + bne L(strmm_kernel_L8_M8_20) /******************************************************************************/ -.Lstrmm_kernel_L8_M4_BEGIN: +L(strmm_kernel_L8_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lstrmm_kernel_L8_END + ble L(strmm_kernel_L8_END) tst counterI, #4 - ble .Lstrmm_kernel_L8_M2_BEGIN + ble L(strmm_kernel_L8_M2_BEGIN) -.Lstrmm_kernel_L8_M4_20: +L(strmm_kernel_L8_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1464,54 +1464,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lstrmm_kernel_L8_M4_32 + blt L(strmm_kernel_L8_M4_32) KERNEL4x8_I // do one in the K KERNEL4x8_M2 // do another in the K subs counterL, counterL, #2 - ble .Lstrmm_kernel_L8_M4_22a + ble L(strmm_kernel_L8_M4_22a) .align 5 -.Lstrmm_kernel_L8_M4_22: +L(strmm_kernel_L8_M4_22): KERNEL4x8_M1 KERNEL4x8_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M4_22 + bgt L(strmm_kernel_L8_M4_22) -.Lstrmm_kernel_L8_M4_22a: +L(strmm_kernel_L8_M4_22a): KERNEL4x8_M1 KERNEL4x8_E - b .Lstrmm_kernel_L8_M4_44 + b L(strmm_kernel_L8_M4_44) -.Lstrmm_kernel_L8_M4_32: +L(strmm_kernel_L8_M4_32): tst counterL, #1 - ble .Lstrmm_kernel_L8_M4_40 + ble L(strmm_kernel_L8_M4_40) KERNEL4x8_I KERNEL4x8_E - b .Lstrmm_kernel_L8_M4_44 + b L(strmm_kernel_L8_M4_44) -.Lstrmm_kernel_L8_M4_40: +L(strmm_kernel_L8_M4_40): INIT4x8 -.Lstrmm_kernel_L8_M4_44: +L(strmm_kernel_L8_M4_44): ands counterL , tempK, #1 - ble .Lstrmm_kernel_L8_M4_100 + ble L(strmm_kernel_L8_M4_100) -.Lstrmm_kernel_L8_M4_46: +L(strmm_kernel_L8_M4_46): KERNEL4x8_SUB -.Lstrmm_kernel_L8_M4_100: +L(strmm_kernel_L8_M4_100): SAVE4x8 @@ -1531,20 +1531,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L8_M4_END: +L(strmm_kernel_L8_M4_END): /******************************************************************************/ -.Lstrmm_kernel_L8_M2_BEGIN: +L(strmm_kernel_L8_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L8_END + ble L(strmm_kernel_L8_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L8_M1_BEGIN + ble L(strmm_kernel_L8_M1_BEGIN) -.Lstrmm_kernel_L8_M2_20: +L(strmm_kernel_L8_M2_20): INIT2x8 @@ -1568,9 +1568,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L8_M2_40 + ble L(strmm_kernel_L8_M2_40) -.Lstrmm_kernel_L8_M2_22: +L(strmm_kernel_L8_M2_22): KERNEL2x8_SUB KERNEL2x8_SUB @@ -1583,22 +1583,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M2_22 + bgt L(strmm_kernel_L8_M2_22) -.Lstrmm_kernel_L8_M2_40: +L(strmm_kernel_L8_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L8_M2_100 + ble L(strmm_kernel_L8_M2_100) -.Lstrmm_kernel_L8_M2_42: +L(strmm_kernel_L8_M2_42): KERNEL2x8_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M2_42 + bgt L(strmm_kernel_L8_M2_42) -.Lstrmm_kernel_L8_M2_100: +L(strmm_kernel_L8_M2_100): SAVE2x8 @@ -1618,16 +1618,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L8_M2_END: +L(strmm_kernel_L8_M2_END): /******************************************************************************/ -.Lstrmm_kernel_L8_M1_BEGIN: +L(strmm_kernel_L8_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L8_END + ble L(strmm_kernel_L8_END) -.Lstrmm_kernel_L8_M1_20: +L(strmm_kernel_L8_M1_20): INIT1x8 @@ -1651,9 +1651,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L8_M1_40 + ble L(strmm_kernel_L8_M1_40) -.Lstrmm_kernel_L8_M1_22: +L(strmm_kernel_L8_M1_22): KERNEL1x8_SUB KERNEL1x8_SUB KERNEL1x8_SUB @@ -1665,22 +1665,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M1_22 + bgt L(strmm_kernel_L8_M1_22) -.Lstrmm_kernel_L8_M1_40: +L(strmm_kernel_L8_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L8_M1_100 + ble L(strmm_kernel_L8_M1_100) -.Lstrmm_kernel_L8_M1_42: +L(strmm_kernel_L8_M1_42): KERNEL1x8_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_M1_42 + bgt L(strmm_kernel_L8_M1_42) -.Lstrmm_kernel_L8_M1_100: +L(strmm_kernel_L8_M1_100): SAVE1x8 @@ -1700,7 +1700,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #1 #endif -.Lstrmm_kernel_L8_END: +L(strmm_kernel_L8_END): lsl temp, origK, #5 // B = B + K * 4 * 8 add origPB, origPB, temp @@ -1709,19 +1709,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Lstrmm_kernel_L8_BEGIN + bgt L(strmm_kernel_L8_BEGIN) /******************************************************************************/ /******************************************************************************/ -.Lstrmm_kernel_L4_BEGIN: +L(strmm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #7 - ble .Lstrmm_kernel_L999 + ble L(strmm_kernel_L999) tst counterJ , #4 - ble .Lstrmm_kernel_L2_BEGIN + ble L(strmm_kernel_L2_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1735,14 +1735,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lstrmm_kernel_L4_M8_BEGIN: +L(strmm_kernel_L4_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI, #0 - ble .Lstrmm_kernel_L4_M4_BEGIN + ble L(strmm_kernel_L4_M4_BEGIN) -.Lstrmm_kernel_L4_M8_20: +L(strmm_kernel_L4_M8_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1764,54 +1764,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lstrmm_kernel_L4_M8_32 + blt L(strmm_kernel_L4_M8_32) KERNEL8x4_I // do one in the K KERNEL8x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lstrmm_kernel_L4_M8_22a + ble L(strmm_kernel_L4_M8_22a) .align 5 -.Lstrmm_kernel_L4_M8_22: +L(strmm_kernel_L4_M8_22): KERNEL8x4_M1 KERNEL8x4_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M8_22 + bgt L(strmm_kernel_L4_M8_22) -.Lstrmm_kernel_L4_M8_22a: +L(strmm_kernel_L4_M8_22a): KERNEL8x4_M1 KERNEL8x4_E - b .Lstrmm_kernel_L4_M8_44 + b L(strmm_kernel_L4_M8_44) -.Lstrmm_kernel_L4_M8_32: +L(strmm_kernel_L4_M8_32): tst counterL, #1 - ble .Lstrmm_kernel_L4_M8_40 + ble L(strmm_kernel_L4_M8_40) KERNEL8x4_I KERNEL8x4_E - b .Lstrmm_kernel_L4_M8_44 + b L(strmm_kernel_L4_M8_44) -.Lstrmm_kernel_L4_M8_40: +L(strmm_kernel_L4_M8_40): INIT8x4 -.Lstrmm_kernel_L4_M8_44: +L(strmm_kernel_L4_M8_44): ands counterL , tempK, #1 - ble .Lstrmm_kernel_L4_M8_100 + ble L(strmm_kernel_L4_M8_100) -.Lstrmm_kernel_L4_M8_46: +L(strmm_kernel_L4_M8_46): KERNEL8x4_SUB -.Lstrmm_kernel_L4_M8_100: +L(strmm_kernel_L4_M8_100): SAVE8x4 @@ -1830,22 +1830,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #8 #endif -.Lstrmm_kernel_L4_M8_END: +L(strmm_kernel_L4_M8_END): subs counterI, counterI, #1 - bne .Lstrmm_kernel_L4_M8_20 + bne L(strmm_kernel_L4_M8_20) /******************************************************************************/ -.Lstrmm_kernel_L4_M4_BEGIN: +L(strmm_kernel_L4_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) tst counterI, #4 - ble .Lstrmm_kernel_L4_M2_BEGIN + ble L(strmm_kernel_L4_M2_BEGIN) -.Lstrmm_kernel_L4_M4_20: +L(strmm_kernel_L4_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1865,54 +1865,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #1 // L = K / 2 cmp counterL , #2 // is there at least 4 to do? - blt .Lstrmm_kernel_L4_M4_32 + blt L(strmm_kernel_L4_M4_32) KERNEL4x4_I // do one in the K KERNEL4x4_M2 // do another in the K subs counterL, counterL, #2 - ble .Lstrmm_kernel_L4_M4_22a + ble L(strmm_kernel_L4_M4_22a) .align 5 -.Lstrmm_kernel_L4_M4_22: +L(strmm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M4_22 + bgt L(strmm_kernel_L4_M4_22) -.Lstrmm_kernel_L4_M4_22a: +L(strmm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_E - b .Lstrmm_kernel_L4_M4_44 + b L(strmm_kernel_L4_M4_44) -.Lstrmm_kernel_L4_M4_32: +L(strmm_kernel_L4_M4_32): tst counterL, #1 - ble .Lstrmm_kernel_L4_M4_40 + ble L(strmm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_E - b .Lstrmm_kernel_L4_M4_44 + b L(strmm_kernel_L4_M4_44) -.Lstrmm_kernel_L4_M4_40: +L(strmm_kernel_L4_M4_40): INIT4x4 -.Lstrmm_kernel_L4_M4_44: +L(strmm_kernel_L4_M4_44): ands counterL , tempK, #1 - ble .Lstrmm_kernel_L4_M4_100 + ble L(strmm_kernel_L4_M4_100) -.Lstrmm_kernel_L4_M4_46: +L(strmm_kernel_L4_M4_46): KERNEL4x4_SUB -.Lstrmm_kernel_L4_M4_100: +L(strmm_kernel_L4_M4_100): SAVE4x4 @@ -1930,20 +1930,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L4_M4_END: +L(strmm_kernel_L4_M4_END): /******************************************************************************/ -.Lstrmm_kernel_L4_M2_BEGIN: +L(strmm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L4_M1_BEGIN + ble L(strmm_kernel_L4_M1_BEGIN) -.Lstrmm_kernel_L4_M2_20: +L(strmm_kernel_L4_M2_20): INIT2x4 @@ -1966,9 +1966,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L4_M2_40 + ble L(strmm_kernel_L4_M2_40) -.Lstrmm_kernel_L4_M2_22: +L(strmm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1981,22 +1981,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M2_22 + bgt L(strmm_kernel_L4_M2_22) -.Lstrmm_kernel_L4_M2_40: +L(strmm_kernel_L4_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L4_M2_100 + ble L(strmm_kernel_L4_M2_100) -.Lstrmm_kernel_L4_M2_42: +L(strmm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M2_42 + bgt L(strmm_kernel_L4_M2_42) -.Lstrmm_kernel_L4_M2_100: +L(strmm_kernel_L4_M2_100): SAVE2x4 @@ -2015,16 +2015,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L4_M2_END: +L(strmm_kernel_L4_M2_END): /******************************************************************************/ -.Lstrmm_kernel_L4_M1_BEGIN: +L(strmm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L4_END + ble L(strmm_kernel_L4_END) -.Lstrmm_kernel_L4_M1_20: +L(strmm_kernel_L4_M1_20): INIT1x4 @@ -2047,9 +2047,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L4_M1_40 + ble L(strmm_kernel_L4_M1_40) -.Lstrmm_kernel_L4_M1_22: +L(strmm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -2061,22 +2061,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M1_22 + bgt L(strmm_kernel_L4_M1_22) -.Lstrmm_kernel_L4_M1_40: +L(strmm_kernel_L4_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L4_M1_100 + ble L(strmm_kernel_L4_M1_100) -.Lstrmm_kernel_L4_M1_42: +L(strmm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_M1_42 + bgt L(strmm_kernel_L4_M1_42) -.Lstrmm_kernel_L4_M1_100: +L(strmm_kernel_L4_M1_100): SAVE1x4 @@ -2095,7 +2095,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #1 #endif -.Lstrmm_kernel_L4_END: +L(strmm_kernel_L4_END): add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4 #if !defined(LEFT) add tempOffset, tempOffset, #4 @@ -2104,14 +2104,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ /******************************************************************************/ -.Lstrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(strmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lstrmm_kernel_L999 + ble L(strmm_kernel_L999) tst counterJ , #2 - ble .Lstrmm_kernel_L1_BEGIN + ble L(strmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -2124,14 +2124,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lstrmm_kernel_L2_M8_BEGIN: +L(strmm_kernel_L2_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 // counterI = counterI / 8 cmp counterI,#0 - ble .Lstrmm_kernel_L2_M4_BEGIN + ble L(strmm_kernel_L2_M4_BEGIN) -.Lstrmm_kernel_L2_M8_20: +L(strmm_kernel_L2_M8_20): INIT8x2 @@ -2154,10 +2154,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M8_40 + ble L(strmm_kernel_L2_M8_40) .align 5 -.Lstrmm_kernel_L2_M8_22: +L(strmm_kernel_L2_M8_22): KERNEL8x2_SUB KERNEL8x2_SUB KERNEL8x2_SUB @@ -2169,22 +2169,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M8_22 + bgt L(strmm_kernel_L2_M8_22) -.Lstrmm_kernel_L2_M8_40: +L(strmm_kernel_L2_M8_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M8_100 + ble L(strmm_kernel_L2_M8_100) -.Lstrmm_kernel_L2_M8_42: +L(strmm_kernel_L2_M8_42): KERNEL8x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M8_42 + bgt L(strmm_kernel_L2_M8_42) -.Lstrmm_kernel_L2_M8_100: +L(strmm_kernel_L2_M8_100): SAVE8x2 @@ -2203,23 +2203,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #8 #endif -.Lstrmm_kernel_L2_M8_END: +L(strmm_kernel_L2_M8_END): subs counterI, counterI, #1 - bgt .Lstrmm_kernel_L2_M8_20 + bgt L(strmm_kernel_L2_M8_20) /******************************************************************************/ -.Lstrmm_kernel_L2_M4_BEGIN: +L(strmm_kernel_L2_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) tst counterI, #4 - ble .Lstrmm_kernel_L2_M2_BEGIN + ble L(strmm_kernel_L2_M2_BEGIN) -.Lstrmm_kernel_L2_M4_20: +L(strmm_kernel_L2_M4_20): INIT4x2 @@ -2242,10 +2242,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M4_40 + ble L(strmm_kernel_L2_M4_40) .align 5 -.Lstrmm_kernel_L2_M4_22: +L(strmm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -2257,22 +2257,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M4_22 + bgt L(strmm_kernel_L2_M4_22) -.Lstrmm_kernel_L2_M4_40: +L(strmm_kernel_L2_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M4_100 + ble L(strmm_kernel_L2_M4_100) -.Lstrmm_kernel_L2_M4_42: +L(strmm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M4_42 + bgt L(strmm_kernel_L2_M4_42) -.Lstrmm_kernel_L2_M4_100: +L(strmm_kernel_L2_M4_100): SAVE4x2 @@ -2291,20 +2291,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L2_M4_END: +L(strmm_kernel_L2_M4_END): /******************************************************************************/ -.Lstrmm_kernel_L2_M2_BEGIN: +L(strmm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L2_M1_BEGIN + ble L(strmm_kernel_L2_M1_BEGIN) -.Lstrmm_kernel_L2_M2_20: +L(strmm_kernel_L2_M2_20): INIT2x2 @@ -2327,9 +2327,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lstrmm_kernel_L2_M2_40 + ble L(strmm_kernel_L2_M2_40) -.Lstrmm_kernel_L2_M2_22: +L(strmm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -2342,22 +2342,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M2_22 + bgt L(strmm_kernel_L2_M2_22) -.Lstrmm_kernel_L2_M2_40: +L(strmm_kernel_L2_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M2_100 + ble L(strmm_kernel_L2_M2_100) -.Lstrmm_kernel_L2_M2_42: +L(strmm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M2_42 + bgt L(strmm_kernel_L2_M2_42) -.Lstrmm_kernel_L2_M2_100: +L(strmm_kernel_L2_M2_100): SAVE2x2 #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) @@ -2376,16 +2376,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L2_M2_END: +L(strmm_kernel_L2_M2_END): /******************************************************************************/ -.Lstrmm_kernel_L2_M1_BEGIN: +L(strmm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L2_END + ble L(strmm_kernel_L2_END) -.Lstrmm_kernel_L2_M1_20: +L(strmm_kernel_L2_M1_20): INIT1x2 @@ -2408,9 +2408,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lstrmm_kernel_L2_M1_40 + ble L(strmm_kernel_L2_M1_40) -.Lstrmm_kernel_L2_M1_22: +L(strmm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -2422,22 +2422,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M1_22 + bgt L(strmm_kernel_L2_M1_22) -.Lstrmm_kernel_L2_M1_40: +L(strmm_kernel_L2_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L2_M1_100 + ble L(strmm_kernel_L2_M1_100) -.Lstrmm_kernel_L2_M1_42: +L(strmm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_M1_42 + bgt L(strmm_kernel_L2_M1_42) -.Lstrmm_kernel_L2_M1_100: +L(strmm_kernel_L2_M1_100): SAVE1x2 @@ -2456,7 +2456,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #1 #endif -.Lstrmm_kernel_L2_END: +L(strmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -2465,11 +2465,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ /******************************************************************************/ -.Lstrmm_kernel_L1_BEGIN: +L(strmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lstrmm_kernel_L999 // done + ble L(strmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -2482,14 +2482,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lstrmm_kernel_L1_M8_BEGIN: +L(strmm_kernel_L1_M8_BEGIN): mov counterI, origM asr counterI, counterI, #3 cmp counterI, #0 - ble .Lstrmm_kernel_L1_M4_BEGIN + ble L(strmm_kernel_L1_M4_BEGIN) -.Lstrmm_kernel_L1_M8_20: +L(strmm_kernel_L1_M8_20): INIT8x1 @@ -2512,10 +2512,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M8_40 + ble L(strmm_kernel_L1_M8_40) .align 5 -.Lstrmm_kernel_L1_M8_22: +L(strmm_kernel_L1_M8_22): KERNEL8x1_SUB KERNEL8x1_SUB KERNEL8x1_SUB @@ -2527,22 +2527,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M8_22 + bgt L(strmm_kernel_L1_M8_22) -.Lstrmm_kernel_L1_M8_40: +L(strmm_kernel_L1_M8_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M8_100 + ble L(strmm_kernel_L1_M8_100) -.Lstrmm_kernel_L1_M8_42: +L(strmm_kernel_L1_M8_42): KERNEL8x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M8_42 + bgt L(strmm_kernel_L1_M8_42) -.Lstrmm_kernel_L1_M8_100: +L(strmm_kernel_L1_M8_100): SAVE8x1 @@ -2561,23 +2561,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #8 #endif -.Lstrmm_kernel_L1_M8_END: +L(strmm_kernel_L1_M8_END): subs counterI, counterI, #1 - bgt .Lstrmm_kernel_L1_M8_20 + bgt L(strmm_kernel_L1_M8_20) /******************************************************************************/ -.Lstrmm_kernel_L1_M4_BEGIN: +L(strmm_kernel_L1_M4_BEGIN): mov counterI, origM tst counterI , #7 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) tst counterI, #4 - ble .Lstrmm_kernel_L1_M2_BEGIN + ble L(strmm_kernel_L1_M2_BEGIN) -.Lstrmm_kernel_L1_M4_20: +L(strmm_kernel_L1_M4_20): INIT4x1 @@ -2600,10 +2600,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M4_40 + ble L(strmm_kernel_L1_M4_40) .align 5 -.Lstrmm_kernel_L1_M4_22: +L(strmm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -2615,22 +2615,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M4_22 + bgt L(strmm_kernel_L1_M4_22) -.Lstrmm_kernel_L1_M4_40: +L(strmm_kernel_L1_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M4_100 + ble L(strmm_kernel_L1_M4_100) -.Lstrmm_kernel_L1_M4_42: +L(strmm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M4_42 + bgt L(strmm_kernel_L1_M4_42) -.Lstrmm_kernel_L1_M4_100: +L(strmm_kernel_L1_M4_100): SAVE4x1 @@ -2649,20 +2649,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #4 #endif -.Lstrmm_kernel_L1_M4_END: +L(strmm_kernel_L1_M4_END): /******************************************************************************/ -.Lstrmm_kernel_L1_M2_BEGIN: +L(strmm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lstrmm_kernel_L1_M1_BEGIN + ble L(strmm_kernel_L1_M1_BEGIN) -.Lstrmm_kernel_L1_M2_20: +L(strmm_kernel_L1_M2_20): INIT2x1 @@ -2685,9 +2685,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M2_40 + ble L(strmm_kernel_L1_M2_40) -.Lstrmm_kernel_L1_M2_22: +L(strmm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -2700,22 +2700,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M2_22 + bgt L(strmm_kernel_L1_M2_22) -.Lstrmm_kernel_L1_M2_40: +L(strmm_kernel_L1_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M2_100 + ble L(strmm_kernel_L1_M2_100) -.Lstrmm_kernel_L1_M2_42: +L(strmm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M2_42 + bgt L(strmm_kernel_L1_M2_42) -.Lstrmm_kernel_L1_M2_100: +L(strmm_kernel_L1_M2_100): SAVE2x1 @@ -2734,16 +2734,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(LEFT) add tempOffset, tempOffset, #2 #endif -.Lstrmm_kernel_L1_M2_END: +L(strmm_kernel_L1_M2_END): /******************************************************************************/ -.Lstrmm_kernel_L1_M1_BEGIN: +L(strmm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lstrmm_kernel_L1_END + ble L(strmm_kernel_L1_END) -.Lstrmm_kernel_L1_M1_20: +L(strmm_kernel_L1_M1_20): INIT1x1 @@ -2766,9 +2766,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lstrmm_kernel_L1_M1_40 + ble L(strmm_kernel_L1_M1_40) -.Lstrmm_kernel_L1_M1_22: +L(strmm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -2780,30 +2780,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M1_22 + bgt L(strmm_kernel_L1_M1_22) -.Lstrmm_kernel_L1_M1_40: +L(strmm_kernel_L1_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lstrmm_kernel_L1_M1_100 + ble L(strmm_kernel_L1_M1_100) -.Lstrmm_kernel_L1_M1_42: +L(strmm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_M1_42 + bgt L(strmm_kernel_L1_M1_42) -.Lstrmm_kernel_L1_M1_100: +L(strmm_kernel_L1_M1_100): SAVE1x1 -.Lstrmm_kernel_L1_END: +L(strmm_kernel_L1_END): /******************************************************************************/ -.Lstrmm_kernel_L999: +L(strmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/strmm_kernel_sve_v1x8.S b/kernel/arm64/strmm_kernel_sve_v1x8.S index 3c45e3e298..106b6d2051 100644 --- a/kernel/arm64/strmm_kernel_sve_v1x8.S +++ b/kernel/arm64/strmm_kernel_sve_v1x8.S @@ -472,13 +472,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #3 // J = J / 8 cmp counterJ, #0 - ble .Lstrmm_kernel_L4_BEGIN + ble L(strmm_kernel_L4_BEGIN) /******************************************************************************/ /* Repeat this as long as there are 8 left in N */ .align 5 -.Lstrmm_kernel_L8_BEGIN: +L(strmm_kernel_L8_BEGIN): mov pCRow0, pC add pC, pC, LDC, lsl #3 // add 8 x LDC @@ -489,7 +489,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lstrmm_kernel_L8_Mv1_BEGIN: +L(strmm_kernel_L8_Mv1_BEGIN): /* Loop over M is done in an SVE fashion. This has the benefit of the last M%SVE_LEN iterations being done in a single sweep */ mov counterI, #0 @@ -497,7 +497,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension .align 5 -.Lstrmm_kernel_L8_Mv1_20: +L(strmm_kernel_L8_Mv1_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -521,7 +521,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // L = K / 8 cmp counterL , #2 // is there at least 4 to do? - blt .Lstrmm_kernel_L8_Mv1_32 + blt L(strmm_kernel_L8_Mv1_32) KERNELv1x8_I KERNELv1x8_M2 @@ -533,10 +533,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lstrmm_kernel_L8_Mv1_22a + ble L(strmm_kernel_L8_Mv1_22a) .align 5 -.Lstrmm_kernel_L8_Mv1_22: +L(strmm_kernel_L8_Mv1_22): KERNELv1x8_M1 KERNELv1x8_M2 @@ -548,10 +548,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M2 subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L8_Mv1_22 + bgt L(strmm_kernel_L8_Mv1_22) .align 5 -.Lstrmm_kernel_L8_Mv1_22a: +L(strmm_kernel_L8_Mv1_22a): KERNELv1x8_M1 KERNELv1x8_M2 @@ -562,13 +562,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_M1 KERNELv1x8_E - b .Lstrmm_kernel_L8_Mv1_44 + b L(strmm_kernel_L8_Mv1_44) .align 5 -.Lstrmm_kernel_L8_Mv1_32: +L(strmm_kernel_L8_Mv1_32): tst counterL, #1 - ble .Lstrmm_kernel_L8_Mv1_40 + ble L(strmm_kernel_L8_Mv1_40) KERNELv1x8_I KERNELv1x8_M2 @@ -580,26 +580,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x8_E - b .Lstrmm_kernel_L8_Mv1_44 + b L(strmm_kernel_L8_Mv1_44) -.Lstrmm_kernel_L8_Mv1_40: +L(strmm_kernel_L8_Mv1_40): INITv1x8 -.Lstrmm_kernel_L8_Mv1_44: +L(strmm_kernel_L8_Mv1_44): ands counterL , tempK, #7 - ble .Lstrmm_kernel_L8_Mv1_100 + ble L(strmm_kernel_L8_Mv1_100) .align 5 -.Lstrmm_kernel_L8_Mv1_46: +L(strmm_kernel_L8_Mv1_46): KERNELv1x8_SUB subs counterL, counterL, #1 - bne .Lstrmm_kernel_L8_Mv1_46 + bne L(strmm_kernel_L8_Mv1_46) -.Lstrmm_kernel_L8_Mv1_100: +L(strmm_kernel_L8_Mv1_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] @@ -622,14 +622,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, lanes #endif -.Lstrmm_kernel_L8_Mv1_END: +L(strmm_kernel_L8_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lstrmm_kernel_L8_Mv1_20 + b.any L(strmm_kernel_L8_Mv1_20) -.Lstrmm_kernel_L8_END: +L(strmm_kernel_L8_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 8 * 4 @@ -639,17 +639,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Lstrmm_kernel_L8_BEGIN + bgt L(strmm_kernel_L8_BEGIN) /******************************************************************************/ /* Repeat the same thing if 4 left in N */ .align 5 -.Lstrmm_kernel_L4_BEGIN: +L(strmm_kernel_L4_BEGIN): mov counterJ , origN tst counterJ , #4 - ble .Lstrmm_kernel_L2_BEGIN + ble L(strmm_kernel_L2_BEGIN) #if defined(LEFT) mov tempOffset, offset @@ -661,14 +661,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lstrmm_kernel_L4_Mv1_BEGIN: +L(strmm_kernel_L4_Mv1_BEGIN): mov counterI, #0 whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s .align 5 -.Lstrmm_kernel_L4_Mv1_20: +L(strmm_kernel_L4_Mv1_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -692,10 +692,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Lstrmm_kernel_L4_Mv1_44 + ble L(strmm_kernel_L4_Mv1_44) .align 5 -.Lstrmm_kernel_L4_Mv1_22: +L(strmm_kernel_L4_Mv1_22): KERNELv1x4_SUB KERNELv1x4_SUB @@ -707,22 +707,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L4_Mv1_22 + bgt L(strmm_kernel_L4_Mv1_22) -.Lstrmm_kernel_L4_Mv1_44: +L(strmm_kernel_L4_Mv1_44): ands counterL , tempK, #7 - ble .Lstrmm_kernel_L4_Mv1_100 + ble L(strmm_kernel_L4_Mv1_100) .align 5 -.Lstrmm_kernel_L4_Mv1_46: +L(strmm_kernel_L4_Mv1_46): KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Lstrmm_kernel_L4_Mv1_46 + bne L(strmm_kernel_L4_Mv1_46) -.Lstrmm_kernel_L4_Mv1_100: +L(strmm_kernel_L4_Mv1_100): SAVEv1x4 @@ -742,15 +742,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, lanes #endif -.Lstrmm_kernel_L4_Mv1_END: +L(strmm_kernel_L4_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lstrmm_kernel_L4_Mv1_20 + b.any L(strmm_kernel_L4_Mv1_20) -.Lstrmm_kernel_L4_END: +L(strmm_kernel_L4_END): lsl temp, origK, #4 add origPB, origPB, temp // B = B + K * 4 * 4 #if !defined(LEFT) @@ -761,11 +761,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /* Repeat the same thing if 2 left in N */ .align 5 -.Lstrmm_kernel_L2_BEGIN: +L(strmm_kernel_L2_BEGIN): mov counterJ , origN tst counterJ , #2 - ble .Lstrmm_kernel_L1_BEGIN + ble L(strmm_kernel_L1_BEGIN) mov pCRow0, pC @@ -777,14 +777,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lstrmm_kernel_L2_Mv1_BEGIN: +L(strmm_kernel_L2_Mv1_BEGIN): mov counterI, #0 whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s .align 5 -.Lstrmm_kernel_L2_Mv1_20: +L(strmm_kernel_L2_Mv1_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -808,10 +808,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // L = K / 8 cmp counterL , #0 // is there at least 4 to do? - ble .Lstrmm_kernel_L2_Mv1_44 + ble L(strmm_kernel_L2_Mv1_44) .align 5 -.Lstrmm_kernel_L2_Mv1_22: +L(strmm_kernel_L2_Mv1_22): KERNELv1x2_SUB KERNELv1x2_SUB @@ -823,22 +823,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L2_Mv1_22 + bgt L(strmm_kernel_L2_Mv1_22) -.Lstrmm_kernel_L2_Mv1_44: +L(strmm_kernel_L2_Mv1_44): ands counterL , tempK, #7 - ble .Lstrmm_kernel_L2_Mv1_100 + ble L(strmm_kernel_L2_Mv1_100) .align 5 -.Lstrmm_kernel_L2_Mv1_46: +L(strmm_kernel_L2_Mv1_46): KERNELv1x2_SUB subs counterL, counterL, #1 - bne .Lstrmm_kernel_L2_Mv1_46 + bne L(strmm_kernel_L2_Mv1_46) -.Lstrmm_kernel_L2_Mv1_100: +L(strmm_kernel_L2_Mv1_100): SAVEv1x2 @@ -859,15 +859,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.Lstrmm_kernel_L2_Mv1_END: +L(strmm_kernel_L2_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lstrmm_kernel_L2_Mv1_20 + b.any L(strmm_kernel_L2_Mv1_20) -.Lstrmm_kernel_L2_END: +L(strmm_kernel_L2_END): add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 #if !defined(LEFT) add tempOffset, tempOffset, #2 @@ -877,11 +877,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /* Repeat the same thing if 1 left in N */ .align 5 -.Lstrmm_kernel_L1_BEGIN: +L(strmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lstrmm_kernel_L999 // done + ble L(strmm_kernel_L999) // done mov pCRow0, pC @@ -893,14 +893,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lstrmm_kernel_L1_Mv1_BEGIN: +L(strmm_kernel_L1_Mv1_BEGIN): mov counterI, #0 whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s .align 5 -.Lstrmm_kernel_L1_Mv1_20: +L(strmm_kernel_L1_Mv1_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -924,10 +924,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // L = K / 8 cmp counterL , #0 // is there at least 8 to do? - ble .Lstrmm_kernel_L1_Mv1_44 + ble L(strmm_kernel_L1_Mv1_44) .align 5 -.Lstrmm_kernel_L1_Mv1_22: +L(strmm_kernel_L1_Mv1_22): KERNELv1x1_SUB KERNELv1x1_SUB @@ -939,22 +939,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_Mv1_22 + bgt L(strmm_kernel_L1_Mv1_22) -.Lstrmm_kernel_L1_Mv1_44: +L(strmm_kernel_L1_Mv1_44): ands counterL , tempK, #7 - ble .Lstrmm_kernel_L1_Mv1_100 + ble L(strmm_kernel_L1_Mv1_100) .align 5 -.Lstrmm_kernel_L1_Mv1_46: +L(strmm_kernel_L1_Mv1_46): KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lstrmm_kernel_L1_Mv1_46 + bgt L(strmm_kernel_L1_Mv1_46) -.Lstrmm_kernel_L1_Mv1_100: +L(strmm_kernel_L1_Mv1_100): SAVEv1x1 @@ -976,19 +976,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lstrmm_kernel_L1_Mv1_END: +L(strmm_kernel_L1_Mv1_END): incw counterI whilelt p1.s, counterI, origM //SVE instruction cntp lanes, p0, p1.s - b.any .Lstrmm_kernel_L1_Mv1_20 + b.any L(strmm_kernel_L1_Mv1_20) -.Lstrmm_kernel_L1_END: +L(strmm_kernel_L1_END): /******************************************************************************/ -.Lstrmm_kernel_L999: +L(strmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/sum.S b/kernel/arm64/sum.S index 16d0dc4e44..6308b86a15 100644 --- a/kernel/arm64/sum.S +++ b/kernel/arm64/sum.S @@ -114,52 +114,52 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif cmp N, xzr - ble .Lsum_kernel_L999 + ble L(sum_kernel_L999) cmp INC_X, xzr - ble .Lsum_kernel_L999 + ble L(sum_kernel_L999) cmp INC_X, #1 - bne .Lsum_kernel_S_BEGIN + bne L(sum_kernel_S_BEGIN) -.Lsum_kernel_F_BEGIN: +L(sum_kernel_F_BEGIN): asr I, N, #3 cmp I, xzr - beq .Lsum_kernel_F1 + beq L(sum_kernel_F1) -.Lsum_kernel_F8: +L(sum_kernel_F8): KERNEL_F8 subs I, I, #1 - bne .Lsum_kernel_F8 + bne L(sum_kernel_F8) KERNEL_F8_FINALIZE -.Lsum_kernel_F1: +L(sum_kernel_F1): ands I, N, #7 - ble .Lsum_kernel_L999 + ble L(sum_kernel_L999) -.Lsum_kernel_F10: +L(sum_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lsum_kernel_F10 + bne L(sum_kernel_F10) -.Lsum_kernel_L999: +L(sum_kernel_L999): ret -.Lsum_kernel_S_BEGIN: +L(sum_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lsum_kernel_S1 + ble L(sum_kernel_S1) -.Lsum_kernel_S4: +L(sum_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -167,19 +167,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lsum_kernel_S4 + bne L(sum_kernel_S4) -.Lsum_kernel_S1: +L(sum_kernel_S1): ands I, N, #3 - ble .Lsum_kernel_L999 + ble L(sum_kernel_L999) -.Lsum_kernel_S10: +L(sum_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lsum_kernel_S10 + bne L(sum_kernel_S10) ret diff --git a/kernel/arm64/swap.S b/kernel/arm64/swap.S index 184e02e9cf..a4076fd851 100644 --- a/kernel/arm64/swap.S +++ b/kernel/arm64/swap.S @@ -193,50 +193,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Lswap_kernel_L999 + ble L(swap_kernel_L999) cmp INC_X, #1 - bne .Lswap_kernel_S_BEGIN + bne L(swap_kernel_S_BEGIN) cmp INC_Y, #1 - bne .Lswap_kernel_S_BEGIN + bne L(swap_kernel_S_BEGIN) -.Lswap_kernel_F_BEGIN: +L(swap_kernel_F_BEGIN): asr I, N, #3 cmp I, xzr - beq .Lswap_kernel_F1 + beq L(swap_kernel_F1) -.Lswap_kernel_F8: +L(swap_kernel_F8): KERNEL_F8 subs I, I, #1 - bne .Lswap_kernel_F8 + bne L(swap_kernel_F8) -.Lswap_kernel_F1: +L(swap_kernel_F1): ands I, N, #7 - ble .Lswap_kernel_L999 + ble L(swap_kernel_L999) -.Lswap_kernel_F10: +L(swap_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lswap_kernel_F10 + bne L(swap_kernel_F10) - b .Lswap_kernel_L999 + b L(swap_kernel_L999) -.Lswap_kernel_S_BEGIN: +L(swap_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lswap_kernel_S1 + ble L(swap_kernel_S1) -.Lswap_kernel_S4: +L(swap_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -244,21 +244,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lswap_kernel_S4 + bne L(swap_kernel_S4) -.Lswap_kernel_S1: +L(swap_kernel_S1): ands I, N, #3 - ble .Lswap_kernel_L999 + ble L(swap_kernel_L999) -.Lswap_kernel_S10: +L(swap_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lswap_kernel_S10 + bne L(swap_kernel_S10) -.Lswap_kernel_L999: +L(swap_kernel_L999): mov w0, wzr ret diff --git a/kernel/arm64/swap_thunderx2t99.S b/kernel/arm64/swap_thunderx2t99.S index 8b97622f2c..9783196c02 100644 --- a/kernel/arm64/swap_thunderx2t99.S +++ b/kernel/arm64/swap_thunderx2t99.S @@ -104,56 +104,56 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Lswap_kernel_L999 + ble L(swap_kernel_L999) cmp INC_X, #1 - bne .Lswap_kernel_S_BEGIN + bne L(swap_kernel_S_BEGIN) cmp INC_Y, #1 - bne .Lswap_kernel_S_BEGIN + bne L(swap_kernel_S_BEGIN) -.Lswap_kernel_F_BEGIN: +L(swap_kernel_F_BEGIN): INIT asr I, N, #N_DIV_SHIFT cmp I, xzr - beq .Lswap_kernel_F1 + beq L(swap_kernel_F1) .align 5 -.Lswap_kernel_F: +L(swap_kernel_F): KERNEL_F subs I, I, #1 - bne .Lswap_kernel_F + bne L(swap_kernel_F) -.Lswap_kernel_F1: +L(swap_kernel_F1): #if defined(DOUBLE) && defined(COMPLEX) - b .Lswap_kernel_L999 + b L(swap_kernel_L999) #else ands I, N, #N_REM_MASK - ble .Lswap_kernel_L999 + ble L(swap_kernel_L999) #endif -.Lswap_kernel_F10: +L(swap_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lswap_kernel_F10 + bne L(swap_kernel_F10) - b .Lswap_kernel_L999 + b L(swap_kernel_L999) -.Lswap_kernel_S_BEGIN: +L(swap_kernel_S_BEGIN): INIT asr I, N, #2 cmp I, xzr - ble .Lswap_kernel_S1 + ble L(swap_kernel_S1) -.Lswap_kernel_S4: +L(swap_kernel_S4): KERNEL_F1 KERNEL_F1 @@ -161,21 +161,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_F1 subs I, I, #1 - bne .Lswap_kernel_S4 + bne L(swap_kernel_S4) -.Lswap_kernel_S1: +L(swap_kernel_S1): ands I, N, #3 - ble .Lswap_kernel_L999 + ble L(swap_kernel_L999) -.Lswap_kernel_S10: +L(swap_kernel_S10): KERNEL_F1 subs I, I, #1 - bne .Lswap_kernel_S10 + bne L(swap_kernel_S10) -.Lswap_kernel_L999: +L(swap_kernel_L999): mov w0, wzr ret diff --git a/kernel/arm64/zamax.S b/kernel/arm64/zamax.S index 3b49c4fffa..5c7457130b 100644 --- a/kernel/arm64/zamax.S +++ b/kernel/arm64/zamax.S @@ -184,62 +184,62 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Lzamax_kernel_zero + ble L(zamax_kernel_zero) cmp INC_X, xzr - ble .Lzamax_kernel_zero + ble L(zamax_kernel_zero) cmp INC_X, #1 - bne .Lzamax_kernel_S_BEGIN + bne L(zamax_kernel_S_BEGIN) -.Lzamax_kernel_F_BEGIN: +L(zamax_kernel_F_BEGIN): asr I, N, #2 cmp I, xzr - beq .Lzamax_kernel_F1_INIT + beq L(zamax_kernel_F1_INIT) INIT_F4 subs I, I, #1 - beq .Lzamax_kernel_F1 + beq L(zamax_kernel_F1) -.Lzamax_kernel_F4: +L(zamax_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lzamax_kernel_F4 + bne L(zamax_kernel_F4) -.Lzamax_kernel_F1: +L(zamax_kernel_F1): ands I, N, #3 - ble .Lzamax_kernel_L999 + ble L(zamax_kernel_L999) -.Lzamax_kernel_F10: +L(zamax_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lzamax_kernel_F10 + bne L(zamax_kernel_F10) ret -.Lzamax_kernel_F1_INIT: +L(zamax_kernel_F1_INIT): INIT_F1 subs N, N, #1 - b .Lzamax_kernel_F1 + b L(zamax_kernel_F1) -.Lzamax_kernel_S_BEGIN: +L(zamax_kernel_S_BEGIN): INIT_S subs N, N, #1 - ble .Lzamax_kernel_L999 + ble L(zamax_kernel_L999) asr I, N, #2 cmp I, xzr - ble .Lzamax_kernel_S1 + ble L(zamax_kernel_S1) -.Lzamax_kernel_S4: +L(zamax_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -247,25 +247,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lzamax_kernel_S4 + bne L(zamax_kernel_S4) -.Lzamax_kernel_S1: +L(zamax_kernel_S1): ands I, N, #3 - ble .Lzamax_kernel_L999 + ble L(zamax_kernel_L999) -.Lzamax_kernel_S10: +L(zamax_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lzamax_kernel_S10 + bne L(zamax_kernel_S10) -.Lzamax_kernel_L999: +L(zamax_kernel_L999): ret -.Lzamax_kernel_zero: +L(zamax_kernel_zero): fmov MAXF, REG0 ret diff --git a/kernel/arm64/zasum.S b/kernel/arm64/zasum.S index 0d5ec952bb..038dedcc5e 100644 --- a/kernel/arm64/zasum.S +++ b/kernel/arm64/zasum.S @@ -92,52 +92,52 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. fmov SUMF, REG0 cmp N, xzr - ble .Lzasum_kernel_L999 + ble L(zasum_kernel_L999) cmp INC_X, xzr - ble .Lzasum_kernel_L999 + ble L(zasum_kernel_L999) cmp INC_X, #1 - bne .Lzasum_kernel_S_BEGIN + bne L(zasum_kernel_S_BEGIN) -.Lzasum_kernel_F_BEGIN: +L(zasum_kernel_F_BEGIN): asr I, N, #2 cmp I, xzr - beq .Lzasum_kernel_F1 + beq L(zasum_kernel_F1) -.Lzasum_kernel_F4: +L(zasum_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lzasum_kernel_F4 + bne L(zasum_kernel_F4) KERNEL_F4_FINALIZE -.Lzasum_kernel_F1: +L(zasum_kernel_F1): ands I, N, #3 - ble .Lzasum_kernel_L999 + ble L(zasum_kernel_L999) -.Lzasum_kernel_F10: +L(zasum_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lzasum_kernel_F10 + bne L(zasum_kernel_F10) -.Lzasum_kernel_L999: +L(zasum_kernel_L999): ret -.Lzasum_kernel_S_BEGIN: +L(zasum_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lzasum_kernel_S1 + ble L(zasum_kernel_S1) -.Lzasum_kernel_S4: +L(zasum_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -145,19 +145,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lzasum_kernel_S4 + bne L(zasum_kernel_S4) -.Lzasum_kernel_S1: +L(zasum_kernel_S1): ands I, N, #3 - ble .Lzasum_kernel_L999 + ble L(zasum_kernel_L999) -.Lzasum_kernel_S10: +L(zasum_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lzasum_kernel_S10 + bne L(zasum_kernel_S10) ret diff --git a/kernel/arm64/zaxpy.S b/kernel/arm64/zaxpy.S index 46d7b04788..6a26284902 100644 --- a/kernel/arm64/zaxpy.S +++ b/kernel/arm64/zaxpy.S @@ -241,62 +241,62 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Lzaxpy_kernel_L999 + ble L(zaxpy_kernel_L999) mov Y_COPY, Y fcmp DA_R, #0.0 - bne .L1 + bne L(1) fcmp DA_I, #0.0 - beq .Lzaxpy_kernel_L999 + beq L(zaxpy_kernel_L999) -.L1: +L(1): INIT cmp INC_X, #1 - bne .Lzaxpy_kernel_S_BEGIN + bne L(zaxpy_kernel_S_BEGIN) cmp INC_Y, #1 - bne .Lzaxpy_kernel_S_BEGIN + bne L(zaxpy_kernel_S_BEGIN) -.Lzaxpy_kernel_F_BEGIN: +L(zaxpy_kernel_F_BEGIN): asr I, N, #2 cmp I, xzr - beq .Lzaxpy_kernel_F1 + beq L(zaxpy_kernel_F1) KERNEL_INIT_F4 -.Lzaxpy_kernel_F4: +L(zaxpy_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lzaxpy_kernel_F4 + bne L(zaxpy_kernel_F4) -.Lzaxpy_kernel_F1: +L(zaxpy_kernel_F1): ands I, N, #3 - ble .Lzaxpy_kernel_L999 + ble L(zaxpy_kernel_L999) -.Lzaxpy_kernel_F10: +L(zaxpy_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lzaxpy_kernel_F10 + bne L(zaxpy_kernel_F10) mov w0, wzr ret -.Lzaxpy_kernel_S_BEGIN: +L(zaxpy_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lzaxpy_kernel_S1 + ble L(zaxpy_kernel_S1) -.Lzaxpy_kernel_S4: +L(zaxpy_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -304,21 +304,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lzaxpy_kernel_S4 + bne L(zaxpy_kernel_S4) -.Lzaxpy_kernel_S1: +L(zaxpy_kernel_S1): ands I, N, #3 - ble .Lzaxpy_kernel_L999 + ble L(zaxpy_kernel_L999) -.Lzaxpy_kernel_S10: +L(zaxpy_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lzaxpy_kernel_S10 + bne L(zaxpy_kernel_S10) -.Lzaxpy_kernel_L999: +L(zaxpy_kernel_L999): mov w0, wzr ret diff --git a/kernel/arm64/zdot.S b/kernel/arm64/zdot.S index 044ace3b82..0e6bc1f608 100644 --- a/kernel/arm64/zdot.S +++ b/kernel/arm64/zdot.S @@ -229,51 +229,51 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif cmp N, xzr - ble .Lzdot_kernel_L999 + ble L(zdot_kernel_L999) cmp INC_X, #1 - bne .Lzdot_kernel_S_BEGIN + bne L(zdot_kernel_S_BEGIN) cmp INC_Y, #1 - bne .Lzdot_kernel_S_BEGIN + bne L(zdot_kernel_S_BEGIN) -.Lzdot_kernel_F_BEGIN: +L(zdot_kernel_F_BEGIN): asr I, N, #2 cmp I, xzr - beq .Lzdot_kernel_F1 + beq L(zdot_kernel_F1) -.Lzdot_kernel_F4: +L(zdot_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lzdot_kernel_F4 + bne L(zdot_kernel_F4) KERNEL_F4_FINALIZE -.Lzdot_kernel_F1: +L(zdot_kernel_F1): ands I, N, #3 - ble .Lzdot_kernel_L999 + ble L(zdot_kernel_L999) -.Lzdot_kernel_F10: +L(zdot_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lzdot_kernel_F10 + bne L(zdot_kernel_F10) ret -.Lzdot_kernel_S_BEGIN: +L(zdot_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lzdot_kernel_S1 + ble L(zdot_kernel_S1) -.Lzdot_kernel_S4: +L(zdot_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -281,21 +281,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lzdot_kernel_S4 + bne L(zdot_kernel_S4) -.Lzdot_kernel_S1: +L(zdot_kernel_S1): ands I, N, #3 - ble .Lzdot_kernel_L999 + ble L(zdot_kernel_L999) -.Lzdot_kernel_S10: +L(zdot_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lzdot_kernel_S10 + bne L(zdot_kernel_S10) -.Lzdot_kernel_L999: +L(zdot_kernel_L999): ret diff --git a/kernel/arm64/zgemm_kernel_4x4.S b/kernel/arm64/zgemm_kernel_4x4.S index a65c4f581d..d4f2145609 100644 --- a/kernel/arm64/zgemm_kernel_4x4.S +++ b/kernel/arm64/zgemm_kernel_4x4.S @@ -1099,9 +1099,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lzgemm_kernel_L2_BEGIN + ble L(zgemm_kernel_L2_BEGIN) -.Lzgemm_kernel_L4_BEGIN: +L(zgemm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -1111,20 +1111,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lzgemm_kernel_L4_M4_BEGIN: +L(zgemm_kernel_L4_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lzgemm_kernel_L4_M2_BEGIN + ble L(zgemm_kernel_L4_M2_BEGIN) .align 5 -.Lzgemm_kernel_L4_M4_20: +L(zgemm_kernel_L4_M4_20): mov pB, origPB asr counterL , origK, #3 cmp counterL , #2 - blt .Lzgemm_kernel_L4_M4_32 + blt L(zgemm_kernel_L4_M4_32) KERNEL4x4_I KERNEL4x4_M2 @@ -1136,10 +1136,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lzgemm_kernel_L4_M4_22a + ble L(zgemm_kernel_L4_M4_22a) .align 5 -.Lzgemm_kernel_L4_M4_22: +L(zgemm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 @@ -1151,10 +1151,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L4_M4_22 + bgt L(zgemm_kernel_L4_M4_22) .align 5 -.Lzgemm_kernel_L4_M4_22a: +L(zgemm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_M2 @@ -1165,13 +1165,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - b .Lzgemm_kernel_L4_M4_44 + b L(zgemm_kernel_L4_M4_44) .align 5 -.Lzgemm_kernel_L4_M4_32: +L(zgemm_kernel_L4_M4_32): tst counterL, #1 - ble .Lzgemm_kernel_L4_M4_40 + ble L(zgemm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_M2 @@ -1182,55 +1182,55 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - b .Lzgemm_kernel_L4_M4_44 + b L(zgemm_kernel_L4_M4_44) -.Lzgemm_kernel_L4_M4_40: +L(zgemm_kernel_L4_M4_40): INIT4x4 -.Lzgemm_kernel_L4_M4_44: +L(zgemm_kernel_L4_M4_44): ands counterL , origK, #7 - ble .Lzgemm_kernel_L4_M4_100 + ble L(zgemm_kernel_L4_M4_100) .align 5 -.Lzgemm_kernel_L4_M4_46: +L(zgemm_kernel_L4_M4_46): KERNEL4x4_SUB subs counterL, counterL, #1 - bne .Lzgemm_kernel_L4_M4_46 + bne L(zgemm_kernel_L4_M4_46) -.Lzgemm_kernel_L4_M4_100: +L(zgemm_kernel_L4_M4_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVE4x4 -.Lzgemm_kernel_L4_M4_END: +L(zgemm_kernel_L4_M4_END): subs counterI, counterI, #1 - bne .Lzgemm_kernel_L4_M4_20 + bne L(zgemm_kernel_L4_M4_20) -.Lzgemm_kernel_L4_M2_BEGIN: +L(zgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lzgemm_kernel_L4_END + ble L(zgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lzgemm_kernel_L4_M1_BEGIN + ble L(zgemm_kernel_L4_M1_BEGIN) -.Lzgemm_kernel_L4_M2_20: +L(zgemm_kernel_L4_M2_20): INIT2x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lzgemm_kernel_L4_M2_40 + ble L(zgemm_kernel_L4_M2_40) -.Lzgemm_kernel_L4_M2_22: +L(zgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1243,43 +1243,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L4_M2_22 + bgt L(zgemm_kernel_L4_M2_22) -.Lzgemm_kernel_L4_M2_40: +L(zgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L4_M2_100 + ble L(zgemm_kernel_L4_M2_100) -.Lzgemm_kernel_L4_M2_42: +L(zgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L4_M2_42 + bgt L(zgemm_kernel_L4_M2_42) -.Lzgemm_kernel_L4_M2_100: +L(zgemm_kernel_L4_M2_100): SAVE2x4 -.Lzgemm_kernel_L4_M2_END: +L(zgemm_kernel_L4_M2_END): -.Lzgemm_kernel_L4_M1_BEGIN: +L(zgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lzgemm_kernel_L4_END + ble L(zgemm_kernel_L4_END) -.Lzgemm_kernel_L4_M1_20: +L(zgemm_kernel_L4_M1_20): INIT1x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lzgemm_kernel_L4_M1_40 + ble L(zgemm_kernel_L4_M1_40) -.Lzgemm_kernel_L4_M1_22: +L(zgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1291,45 +1291,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L4_M1_22 + bgt L(zgemm_kernel_L4_M1_22) -.Lzgemm_kernel_L4_M1_40: +L(zgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L4_M1_100 + ble L(zgemm_kernel_L4_M1_100) -.Lzgemm_kernel_L4_M1_42: +L(zgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L4_M1_42 + bgt L(zgemm_kernel_L4_M1_42) -.Lzgemm_kernel_L4_M1_100: +L(zgemm_kernel_L4_M1_100): SAVE1x4 -.Lzgemm_kernel_L4_END: +L(zgemm_kernel_L4_END): lsl temp, origK, #6 add origPB, origPB, temp // B = B + K * 4 * 8 * 2 subs counterJ, counterJ , #1 // j-- - bgt .Lzgemm_kernel_L4_BEGIN + bgt L(zgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Lzgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(zgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lzgemm_kernel_L999 + ble L(zgemm_kernel_L999) tst counterJ , #2 - ble .Lzgemm_kernel_L1_BEGIN + ble L(zgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1339,24 +1339,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lzgemm_kernel_L2_M4_BEGIN: +L(zgemm_kernel_L2_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI,#0 - ble .Lzgemm_kernel_L2_M2_BEGIN + ble L(zgemm_kernel_L2_M2_BEGIN) -.Lzgemm_kernel_L2_M4_20: +L(zgemm_kernel_L2_M4_20): INIT4x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lzgemm_kernel_L2_M4_40 + ble L(zgemm_kernel_L2_M4_40) .align 5 -.Lzgemm_kernel_L2_M4_22: +L(zgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1368,50 +1368,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M4_22 + bgt L(zgemm_kernel_L2_M4_22) -.Lzgemm_kernel_L2_M4_40: +L(zgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L2_M4_100 + ble L(zgemm_kernel_L2_M4_100) -.Lzgemm_kernel_L2_M4_42: +L(zgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M4_42 + bgt L(zgemm_kernel_L2_M4_42) -.Lzgemm_kernel_L2_M4_100: +L(zgemm_kernel_L2_M4_100): SAVE4x2 -.Lzgemm_kernel_L2_M4_END: +L(zgemm_kernel_L2_M4_END): subs counterI, counterI, #1 - bgt .Lzgemm_kernel_L2_M4_20 + bgt L(zgemm_kernel_L2_M4_20) -.Lzgemm_kernel_L2_M2_BEGIN: +L(zgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lzgemm_kernel_L2_END + ble L(zgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lzgemm_kernel_L2_M1_BEGIN + ble L(zgemm_kernel_L2_M1_BEGIN) -.Lzgemm_kernel_L2_M2_20: +L(zgemm_kernel_L2_M2_20): INIT2x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lzgemm_kernel_L2_M2_40 + ble L(zgemm_kernel_L2_M2_40) -.Lzgemm_kernel_L2_M2_22: +L(zgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1424,43 +1424,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M2_22 + bgt L(zgemm_kernel_L2_M2_22) -.Lzgemm_kernel_L2_M2_40: +L(zgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L2_M2_100 + ble L(zgemm_kernel_L2_M2_100) -.Lzgemm_kernel_L2_M2_42: +L(zgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M2_42 + bgt L(zgemm_kernel_L2_M2_42) -.Lzgemm_kernel_L2_M2_100: +L(zgemm_kernel_L2_M2_100): SAVE2x2 -.Lzgemm_kernel_L2_M2_END: +L(zgemm_kernel_L2_M2_END): -.Lzgemm_kernel_L2_M1_BEGIN: +L(zgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lzgemm_kernel_L2_END + ble L(zgemm_kernel_L2_END) -.Lzgemm_kernel_L2_M1_20: +L(zgemm_kernel_L2_M1_20): INIT1x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lzgemm_kernel_L2_M1_40 + ble L(zgemm_kernel_L2_M1_40) -.Lzgemm_kernel_L2_M1_22: +L(zgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1472,37 +1472,37 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M1_22 + bgt L(zgemm_kernel_L2_M1_22) -.Lzgemm_kernel_L2_M1_40: +L(zgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L2_M1_100 + ble L(zgemm_kernel_L2_M1_100) -.Lzgemm_kernel_L2_M1_42: +L(zgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M1_42 + bgt L(zgemm_kernel_L2_M1_42) -.Lzgemm_kernel_L2_M1_100: +L(zgemm_kernel_L2_M1_100): SAVE1x2 -.Lzgemm_kernel_L2_END: +L(zgemm_kernel_L2_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 2 * 8 * 2 /******************************************************************************/ -.Lzgemm_kernel_L1_BEGIN: +L(zgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lzgemm_kernel_L999 // done + ble L(zgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1512,24 +1512,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lzgemm_kernel_L1_M4_BEGIN: +L(zgemm_kernel_L1_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lzgemm_kernel_L1_M2_BEGIN + ble L(zgemm_kernel_L1_M2_BEGIN) -.Lzgemm_kernel_L1_M4_20: +L(zgemm_kernel_L1_M4_20): INIT4x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lzgemm_kernel_L1_M4_40 + ble L(zgemm_kernel_L1_M4_40) .align 5 -.Lzgemm_kernel_L1_M4_22: +L(zgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1541,50 +1541,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M4_22 + bgt L(zgemm_kernel_L1_M4_22) -.Lzgemm_kernel_L1_M4_40: +L(zgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L1_M4_100 + ble L(zgemm_kernel_L1_M4_100) -.Lzgemm_kernel_L1_M4_42: +L(zgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M4_42 + bgt L(zgemm_kernel_L1_M4_42) -.Lzgemm_kernel_L1_M4_100: +L(zgemm_kernel_L1_M4_100): SAVE4x1 -.Lzgemm_kernel_L1_M4_END: +L(zgemm_kernel_L1_M4_END): subs counterI, counterI, #1 - bgt .Lzgemm_kernel_L1_M4_20 + bgt L(zgemm_kernel_L1_M4_20) -.Lzgemm_kernel_L1_M2_BEGIN: +L(zgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lzgemm_kernel_L1_END + ble L(zgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lzgemm_kernel_L1_M1_BEGIN + ble L(zgemm_kernel_L1_M1_BEGIN) -.Lzgemm_kernel_L1_M2_20: +L(zgemm_kernel_L1_M2_20): INIT2x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lzgemm_kernel_L1_M2_40 + ble L(zgemm_kernel_L1_M2_40) -.Lzgemm_kernel_L1_M2_22: +L(zgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1597,43 +1597,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M2_22 + bgt L(zgemm_kernel_L1_M2_22) -.Lzgemm_kernel_L1_M2_40: +L(zgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L1_M2_100 + ble L(zgemm_kernel_L1_M2_100) -.Lzgemm_kernel_L1_M2_42: +L(zgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M2_42 + bgt L(zgemm_kernel_L1_M2_42) -.Lzgemm_kernel_L1_M2_100: +L(zgemm_kernel_L1_M2_100): SAVE2x1 -.Lzgemm_kernel_L1_M2_END: +L(zgemm_kernel_L1_M2_END): -.Lzgemm_kernel_L1_M1_BEGIN: +L(zgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lzgemm_kernel_L1_END + ble L(zgemm_kernel_L1_END) -.Lzgemm_kernel_L1_M1_20: +L(zgemm_kernel_L1_M1_20): INIT1x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lzgemm_kernel_L1_M1_40 + ble L(zgemm_kernel_L1_M1_40) -.Lzgemm_kernel_L1_M1_22: +L(zgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1645,30 +1645,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M1_22 + bgt L(zgemm_kernel_L1_M1_22) -.Lzgemm_kernel_L1_M1_40: +L(zgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L1_M1_100 + ble L(zgemm_kernel_L1_M1_100) -.Lzgemm_kernel_L1_M1_42: +L(zgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M1_42 + bgt L(zgemm_kernel_L1_M1_42) -.Lzgemm_kernel_L1_M1_100: +L(zgemm_kernel_L1_M1_100): SAVE1x1 -.Lzgemm_kernel_L1_END: +L(zgemm_kernel_L1_END): -.Lzgemm_kernel_L999: +L(zgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/zgemm_kernel_4x4_thunderx2t99.S b/kernel/arm64/zgemm_kernel_4x4_thunderx2t99.S index 8e6ff655de..6db92d1c6c 100644 --- a/kernel/arm64/zgemm_kernel_4x4_thunderx2t99.S +++ b/kernel/arm64/zgemm_kernel_4x4_thunderx2t99.S @@ -1109,9 +1109,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lzgemm_kernel_L2_BEGIN + ble L(zgemm_kernel_L2_BEGIN) -.Lzgemm_kernel_L4_BEGIN: +L(zgemm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -1121,20 +1121,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lzgemm_kernel_L4_M4_BEGIN: +L(zgemm_kernel_L4_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lzgemm_kernel_L4_M2_BEGIN + ble L(zgemm_kernel_L4_M2_BEGIN) .align 5 -.Lzgemm_kernel_L4_M4_20: +L(zgemm_kernel_L4_M4_20): mov pB, origPB asr counterL , origK, #3 cmp counterL , #2 - blt .Lzgemm_kernel_L4_M4_32 + blt L(zgemm_kernel_L4_M4_32) KERNEL4x4_I KERNEL4x4_M2 @@ -1146,10 +1146,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lzgemm_kernel_L4_M4_22a + ble L(zgemm_kernel_L4_M4_22a) .align 5 -.Lzgemm_kernel_L4_M4_22: +L(zgemm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 @@ -1161,10 +1161,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L4_M4_22 + bgt L(zgemm_kernel_L4_M4_22) .align 5 -.Lzgemm_kernel_L4_M4_22a: +L(zgemm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_M2 @@ -1175,13 +1175,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - b .Lzgemm_kernel_L4_M4_44 + b L(zgemm_kernel_L4_M4_44) .align 5 -.Lzgemm_kernel_L4_M4_32: +L(zgemm_kernel_L4_M4_32): tst counterL, #1 - ble .Lzgemm_kernel_L4_M4_40 + ble L(zgemm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_M2 @@ -1192,55 +1192,55 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - b .Lzgemm_kernel_L4_M4_44 + b L(zgemm_kernel_L4_M4_44) -.Lzgemm_kernel_L4_M4_40: +L(zgemm_kernel_L4_M4_40): INIT4x4 -.Lzgemm_kernel_L4_M4_44: +L(zgemm_kernel_L4_M4_44): ands counterL , origK, #7 - ble .Lzgemm_kernel_L4_M4_100 + ble L(zgemm_kernel_L4_M4_100) .align 5 -.Lzgemm_kernel_L4_M4_46: +L(zgemm_kernel_L4_M4_46): KERNEL4x4_SUB subs counterL, counterL, #1 - bne .Lzgemm_kernel_L4_M4_46 + bne L(zgemm_kernel_L4_M4_46) -.Lzgemm_kernel_L4_M4_100: +L(zgemm_kernel_L4_M4_100): prfm PLDL1KEEP, [pA] prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] SAVE4x4 -.Lzgemm_kernel_L4_M4_END: +L(zgemm_kernel_L4_M4_END): subs counterI, counterI, #1 - bne .Lzgemm_kernel_L4_M4_20 + bne L(zgemm_kernel_L4_M4_20) -.Lzgemm_kernel_L4_M2_BEGIN: +L(zgemm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lzgemm_kernel_L4_END + ble L(zgemm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lzgemm_kernel_L4_M1_BEGIN + ble L(zgemm_kernel_L4_M1_BEGIN) -.Lzgemm_kernel_L4_M2_20: +L(zgemm_kernel_L4_M2_20): INIT2x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lzgemm_kernel_L4_M2_40 + ble L(zgemm_kernel_L4_M2_40) -.Lzgemm_kernel_L4_M2_22: +L(zgemm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1253,43 +1253,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L4_M2_22 + bgt L(zgemm_kernel_L4_M2_22) -.Lzgemm_kernel_L4_M2_40: +L(zgemm_kernel_L4_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L4_M2_100 + ble L(zgemm_kernel_L4_M2_100) -.Lzgemm_kernel_L4_M2_42: +L(zgemm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L4_M2_42 + bgt L(zgemm_kernel_L4_M2_42) -.Lzgemm_kernel_L4_M2_100: +L(zgemm_kernel_L4_M2_100): SAVE2x4 -.Lzgemm_kernel_L4_M2_END: +L(zgemm_kernel_L4_M2_END): -.Lzgemm_kernel_L4_M1_BEGIN: +L(zgemm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lzgemm_kernel_L4_END + ble L(zgemm_kernel_L4_END) -.Lzgemm_kernel_L4_M1_20: +L(zgemm_kernel_L4_M1_20): INIT1x4 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lzgemm_kernel_L4_M1_40 + ble L(zgemm_kernel_L4_M1_40) -.Lzgemm_kernel_L4_M1_22: +L(zgemm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1301,45 +1301,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L4_M1_22 + bgt L(zgemm_kernel_L4_M1_22) -.Lzgemm_kernel_L4_M1_40: +L(zgemm_kernel_L4_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L4_M1_100 + ble L(zgemm_kernel_L4_M1_100) -.Lzgemm_kernel_L4_M1_42: +L(zgemm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L4_M1_42 + bgt L(zgemm_kernel_L4_M1_42) -.Lzgemm_kernel_L4_M1_100: +L(zgemm_kernel_L4_M1_100): SAVE1x4 -.Lzgemm_kernel_L4_END: +L(zgemm_kernel_L4_END): lsl temp, origK, #6 add origPB, origPB, temp // B = B + K * 4 * 8 * 2 subs counterJ, counterJ , #1 // j-- - bgt .Lzgemm_kernel_L4_BEGIN + bgt L(zgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Lzgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(zgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lzgemm_kernel_L999 + ble L(zgemm_kernel_L999) tst counterJ , #2 - ble .Lzgemm_kernel_L1_BEGIN + ble L(zgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1349,24 +1349,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lzgemm_kernel_L2_M4_BEGIN: +L(zgemm_kernel_L2_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI,#0 - ble .Lzgemm_kernel_L2_M2_BEGIN + ble L(zgemm_kernel_L2_M2_BEGIN) -.Lzgemm_kernel_L2_M4_20: +L(zgemm_kernel_L2_M4_20): INIT4x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lzgemm_kernel_L2_M4_40 + ble L(zgemm_kernel_L2_M4_40) .align 5 -.Lzgemm_kernel_L2_M4_22: +L(zgemm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1378,50 +1378,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M4_22 + bgt L(zgemm_kernel_L2_M4_22) -.Lzgemm_kernel_L2_M4_40: +L(zgemm_kernel_L2_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L2_M4_100 + ble L(zgemm_kernel_L2_M4_100) -.Lzgemm_kernel_L2_M4_42: +L(zgemm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M4_42 + bgt L(zgemm_kernel_L2_M4_42) -.Lzgemm_kernel_L2_M4_100: +L(zgemm_kernel_L2_M4_100): SAVE4x2 -.Lzgemm_kernel_L2_M4_END: +L(zgemm_kernel_L2_M4_END): subs counterI, counterI, #1 - bgt .Lzgemm_kernel_L2_M4_20 + bgt L(zgemm_kernel_L2_M4_20) -.Lzgemm_kernel_L2_M2_BEGIN: +L(zgemm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lzgemm_kernel_L2_END + ble L(zgemm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lzgemm_kernel_L2_M1_BEGIN + ble L(zgemm_kernel_L2_M1_BEGIN) -.Lzgemm_kernel_L2_M2_20: +L(zgemm_kernel_L2_M2_20): INIT2x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lzgemm_kernel_L2_M2_40 + ble L(zgemm_kernel_L2_M2_40) -.Lzgemm_kernel_L2_M2_22: +L(zgemm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1434,43 +1434,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M2_22 + bgt L(zgemm_kernel_L2_M2_22) -.Lzgemm_kernel_L2_M2_40: +L(zgemm_kernel_L2_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L2_M2_100 + ble L(zgemm_kernel_L2_M2_100) -.Lzgemm_kernel_L2_M2_42: +L(zgemm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M2_42 + bgt L(zgemm_kernel_L2_M2_42) -.Lzgemm_kernel_L2_M2_100: +L(zgemm_kernel_L2_M2_100): SAVE2x2 -.Lzgemm_kernel_L2_M2_END: +L(zgemm_kernel_L2_M2_END): -.Lzgemm_kernel_L2_M1_BEGIN: +L(zgemm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lzgemm_kernel_L2_END + ble L(zgemm_kernel_L2_END) -.Lzgemm_kernel_L2_M1_20: +L(zgemm_kernel_L2_M1_20): INIT1x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lzgemm_kernel_L2_M1_40 + ble L(zgemm_kernel_L2_M1_40) -.Lzgemm_kernel_L2_M1_22: +L(zgemm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1482,37 +1482,37 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M1_22 + bgt L(zgemm_kernel_L2_M1_22) -.Lzgemm_kernel_L2_M1_40: +L(zgemm_kernel_L2_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L2_M1_100 + ble L(zgemm_kernel_L2_M1_100) -.Lzgemm_kernel_L2_M1_42: +L(zgemm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_M1_42 + bgt L(zgemm_kernel_L2_M1_42) -.Lzgemm_kernel_L2_M1_100: +L(zgemm_kernel_L2_M1_100): SAVE1x2 -.Lzgemm_kernel_L2_END: +L(zgemm_kernel_L2_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 2 * 8 * 2 /******************************************************************************/ -.Lzgemm_kernel_L1_BEGIN: +L(zgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lzgemm_kernel_L999 // done + ble L(zgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1522,24 +1522,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lzgemm_kernel_L1_M4_BEGIN: +L(zgemm_kernel_L1_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lzgemm_kernel_L1_M2_BEGIN + ble L(zgemm_kernel_L1_M2_BEGIN) -.Lzgemm_kernel_L1_M4_20: +L(zgemm_kernel_L1_M4_20): INIT4x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lzgemm_kernel_L1_M4_40 + ble L(zgemm_kernel_L1_M4_40) .align 5 -.Lzgemm_kernel_L1_M4_22: +L(zgemm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1551,50 +1551,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M4_22 + bgt L(zgemm_kernel_L1_M4_22) -.Lzgemm_kernel_L1_M4_40: +L(zgemm_kernel_L1_M4_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L1_M4_100 + ble L(zgemm_kernel_L1_M4_100) -.Lzgemm_kernel_L1_M4_42: +L(zgemm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M4_42 + bgt L(zgemm_kernel_L1_M4_42) -.Lzgemm_kernel_L1_M4_100: +L(zgemm_kernel_L1_M4_100): SAVE4x1 -.Lzgemm_kernel_L1_M4_END: +L(zgemm_kernel_L1_M4_END): subs counterI, counterI, #1 - bgt .Lzgemm_kernel_L1_M4_20 + bgt L(zgemm_kernel_L1_M4_20) -.Lzgemm_kernel_L1_M2_BEGIN: +L(zgemm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lzgemm_kernel_L1_END + ble L(zgemm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lzgemm_kernel_L1_M1_BEGIN + ble L(zgemm_kernel_L1_M1_BEGIN) -.Lzgemm_kernel_L1_M2_20: +L(zgemm_kernel_L1_M2_20): INIT2x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lzgemm_kernel_L1_M2_40 + ble L(zgemm_kernel_L1_M2_40) -.Lzgemm_kernel_L1_M2_22: +L(zgemm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1607,43 +1607,43 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M2_22 + bgt L(zgemm_kernel_L1_M2_22) -.Lzgemm_kernel_L1_M2_40: +L(zgemm_kernel_L1_M2_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L1_M2_100 + ble L(zgemm_kernel_L1_M2_100) -.Lzgemm_kernel_L1_M2_42: +L(zgemm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M2_42 + bgt L(zgemm_kernel_L1_M2_42) -.Lzgemm_kernel_L1_M2_100: +L(zgemm_kernel_L1_M2_100): SAVE2x1 -.Lzgemm_kernel_L1_M2_END: +L(zgemm_kernel_L1_M2_END): -.Lzgemm_kernel_L1_M1_BEGIN: +L(zgemm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lzgemm_kernel_L1_END + ble L(zgemm_kernel_L1_END) -.Lzgemm_kernel_L1_M1_20: +L(zgemm_kernel_L1_M1_20): INIT1x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lzgemm_kernel_L1_M1_40 + ble L(zgemm_kernel_L1_M1_40) -.Lzgemm_kernel_L1_M1_22: +L(zgemm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1655,30 +1655,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M1_22 + bgt L(zgemm_kernel_L1_M1_22) -.Lzgemm_kernel_L1_M1_40: +L(zgemm_kernel_L1_M1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L1_M1_100 + ble L(zgemm_kernel_L1_M1_100) -.Lzgemm_kernel_L1_M1_42: +L(zgemm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_M1_42 + bgt L(zgemm_kernel_L1_M1_42) -.Lzgemm_kernel_L1_M1_100: +L(zgemm_kernel_L1_M1_100): SAVE1x1 -.Lzgemm_kernel_L1_END: +L(zgemm_kernel_L1_END): -.Lzgemm_kernel_L999: +L(zgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/zgemm_kernel_sve_v1x4.S b/kernel/arm64/zgemm_kernel_sve_v1x4.S index a043948d69..9331f119b8 100644 --- a/kernel/arm64/zgemm_kernel_sve_v1x4.S +++ b/kernel/arm64/zgemm_kernel_sve_v1x4.S @@ -533,10 +533,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lzgemm_kernel_L2_BEGIN + ble L(zgemm_kernel_L2_BEGIN) /******************************************************************************/ -.Lzgemm_kernel_L4_BEGIN: +L(zgemm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -546,7 +546,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = start of A array -.Lzgemm_kernel_L4_Mv1_BEGIN: +L(zgemm_kernel_L4_Mv1_BEGIN): /* Loop over M is done in an SVE fashion. This has the benefit of the last M%SVE_LEN iterations being done in a single sweep */ mov counterI, #0 @@ -554,14 +554,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension .align 5 -.Lzgemm_kernel_L4_Mv1_20: +L(zgemm_kernel_L4_Mv1_20): mov pB, origPB INITv1x4 // fill with zeros asr counterL , origK, #3 cmp counterL , #2 - blt .Lzgemm_kernel_L4_Mv1_32 + blt L(zgemm_kernel_L4_Mv1_32) KERNELv1x4_I KERNELv1x4_M2 @@ -573,10 +573,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lzgemm_kernel_L4_Mv1_22a + ble L(zgemm_kernel_L4_Mv1_22a) .align 5 -.Lzgemm_kernel_L4_Mv1_22: +L(zgemm_kernel_L4_Mv1_22): KERNELv1x4_M1 KERNELv1x4_M2 @@ -588,10 +588,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M2 subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L4_Mv1_22 + bgt L(zgemm_kernel_L4_Mv1_22) .align 5 -.Lzgemm_kernel_L4_Mv1_22a: +L(zgemm_kernel_L4_Mv1_22a): KERNELv1x4_M1 KERNELv1x4_M2 @@ -602,13 +602,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M1 KERNELv1x4_E - b .Lzgemm_kernel_L4_Mv1_44 + b L(zgemm_kernel_L4_Mv1_44) .align 5 -.Lzgemm_kernel_L4_Mv1_32: +L(zgemm_kernel_L4_Mv1_32): tst counterL, #1 - ble .Lzgemm_kernel_L4_Mv1_40 + ble L(zgemm_kernel_L4_Mv1_40) KERNELv1x4_I KERNELv1x4_M2 @@ -619,56 +619,56 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M1 KERNELv1x4_E - b .Lzgemm_kernel_L4_Mv1_44 + b L(zgemm_kernel_L4_Mv1_44) -.Lzgemm_kernel_L4_Mv1_40: +L(zgemm_kernel_L4_Mv1_40): INITv1x4 -.Lzgemm_kernel_L4_Mv1_44: +L(zgemm_kernel_L4_Mv1_44): ands counterL , origK, #7 - ble .Lzgemm_kernel_L4_Mv1_100 + ble L(zgemm_kernel_L4_Mv1_100) .align 5 -.Lzgemm_kernel_L4_Mv1_46: +L(zgemm_kernel_L4_Mv1_46): KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Lzgemm_kernel_L4_Mv1_46 + bne L(zgemm_kernel_L4_Mv1_46) -.Lzgemm_kernel_L4_Mv1_100: +L(zgemm_kernel_L4_Mv1_100): SAVEv1x4 -.Lzgemm_kernel_L4_Mv1_END: +L(zgemm_kernel_L4_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension - b.any .Lzgemm_kernel_L4_Mv1_20 + b.any L(zgemm_kernel_L4_Mv1_20) -.Lzgemm_kernel_L4_END: +L(zgemm_kernel_L4_END): lsl temp, origK, #6 add origPB, origPB, temp // B = B + K * 4 * 8 * 2 subs counterJ, counterJ , #1 // j-- - bgt .Lzgemm_kernel_L4_BEGIN + bgt L(zgemm_kernel_L4_BEGIN) /******************************************************************************/ -.Lzgemm_kernel_L2_BEGIN: // less than 2 left in N direction +L(zgemm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lzgemm_kernel_L999 + ble L(zgemm_kernel_L999) tst counterJ , #2 - ble .Lzgemm_kernel_L1_BEGIN + ble L(zgemm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC add pCRow1, pCRow0, LDC @@ -679,24 +679,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lzgemm_kernel_L2_Mv1_BEGIN: +L(zgemm_kernel_L2_Mv1_BEGIN): mov counterI, #0 whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d -.Lzgemm_kernel_L2_Mv1_20: +L(zgemm_kernel_L2_Mv1_20): INITv1x2 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lzgemm_kernel_L2_Mv1_40 + ble L(zgemm_kernel_L2_Mv1_40) .align 5 -.Lzgemm_kernel_L2_Mv1_22: +L(zgemm_kernel_L2_Mv1_22): KERNELv1x2_SUB KERNELv1x2_SUB KERNELv1x2_SUB @@ -708,45 +708,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_Mv1_22 + bgt L(zgemm_kernel_L2_Mv1_22) -.Lzgemm_kernel_L2_Mv1_40: +L(zgemm_kernel_L2_Mv1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L2_Mv1_100 + ble L(zgemm_kernel_L2_Mv1_100) -.Lzgemm_kernel_L2_Mv1_42: +L(zgemm_kernel_L2_Mv1_42): KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L2_Mv1_42 + bgt L(zgemm_kernel_L2_Mv1_42) -.Lzgemm_kernel_L2_Mv1_100: +L(zgemm_kernel_L2_Mv1_100): SAVEv1x2 -.Lzgemm_kernel_L2_Mv1_END: +L(zgemm_kernel_L2_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Lzgemm_kernel_L2_Mv1_20 + b.any L(zgemm_kernel_L2_Mv1_20) -.Lzgemm_kernel_L2_END: +L(zgemm_kernel_L2_END): lsl temp, origK, #5 add origPB, origPB, temp // B = B + K * 2 * 8 * 2 /******************************************************************************/ -.Lzgemm_kernel_L1_BEGIN: +L(zgemm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lzgemm_kernel_L999 // done + ble L(zgemm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -754,24 +754,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lzgemm_kernel_L1_Mv1_BEGIN: +L(zgemm_kernel_L1_Mv1_BEGIN): mov counterI, #0 whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d -.Lzgemm_kernel_L1_Mv1_20: +L(zgemm_kernel_L1_Mv1_20): INITv1x1 mov pB, origPB asr counterL , origK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lzgemm_kernel_L1_Mv1_40 + ble L(zgemm_kernel_L1_Mv1_40) .align 5 -.Lzgemm_kernel_L1_Mv1_22: +L(zgemm_kernel_L1_Mv1_22): KERNELv1x1_SUB KERNELv1x1_SUB KERNELv1x1_SUB @@ -783,37 +783,37 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_Mv1_22 + bgt L(zgemm_kernel_L1_Mv1_22) -.Lzgemm_kernel_L1_Mv1_40: +L(zgemm_kernel_L1_Mv1_40): ands counterL , origK, #7 // counterL = counterL % 8 - ble .Lzgemm_kernel_L1_Mv1_100 + ble L(zgemm_kernel_L1_Mv1_100) -.Lzgemm_kernel_L1_Mv1_42: +L(zgemm_kernel_L1_Mv1_42): KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lzgemm_kernel_L1_Mv1_42 + bgt L(zgemm_kernel_L1_Mv1_42) -.Lzgemm_kernel_L1_Mv1_100: +L(zgemm_kernel_L1_Mv1_100): SAVEv1x1 -.Lzgemm_kernel_L1_Mv1_END: +L(zgemm_kernel_L1_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Lzgemm_kernel_L1_Mv1_20 + b.any L(zgemm_kernel_L1_Mv1_20) -.Lzgemm_kernel_L1_END: +L(zgemm_kernel_L1_END): /******************************************************************************/ -.Lzgemm_kernel_L999: +L(zgemm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/zgemv_n.S b/kernel/arm64/zgemv_n.S index 28afcada5a..143e28cffc 100644 --- a/kernel/arm64/zgemv_n.S +++ b/kernel/arm64/zgemv_n.S @@ -364,9 +364,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SAVE_REGS cmp N, xzr - ble .Lzgemv_n_kernel_L999 + ble L(zgemv_n_kernel_L999) cmp M, xzr - ble .Lzgemv_n_kernel_L999 + ble L(zgemv_n_kernel_L999) lsl LDA, LDA, #SHZ lsl INC_X, INC_X, #SHZ @@ -375,9 +375,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT cmp INC_Y, #1 - bne .Lzgemv_n_kernel_S_BEGIN + bne L(zgemv_n_kernel_S_BEGIN) -.Lzgemv_n_kernel_F_LOOP: +L(zgemv_n_kernel_F_LOOP): mov A_PTR, A mov Y_IPTR, Y mov Y_OPTR, Y @@ -387,40 +387,40 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #2 cmp I, xzr - beq .Lzgemv_n_kernel_F1 + beq L(zgemv_n_kernel_F1) -.Lzgemv_n_kernel_F4: +L(zgemv_n_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lzgemv_n_kernel_F4 + bne L(zgemv_n_kernel_F4) -.Lzgemv_n_kernel_F1: +L(zgemv_n_kernel_F1): ands I, M, #3 - ble .Lzgemv_n_kernel_F_END + ble L(zgemv_n_kernel_F_END) -.Lzgemv_n_kernel_F10: +L(zgemv_n_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lzgemv_n_kernel_F10 + bne L(zgemv_n_kernel_F10) -.Lzgemv_n_kernel_F_END: +L(zgemv_n_kernel_F_END): add A, A, LDA subs J, J, #1 - bne .Lzgemv_n_kernel_F_LOOP + bne L(zgemv_n_kernel_F_LOOP) - b .Lzgemv_n_kernel_L999 + b L(zgemv_n_kernel_L999) -.Lzgemv_n_kernel_S_BEGIN: +L(zgemv_n_kernel_S_BEGIN): INIT_S -.Lzgemv_n_kernel_S_LOOP: +L(zgemv_n_kernel_S_LOOP): mov A_PTR, A mov Y_IPTR, Y mov Y_OPTR, Y @@ -430,9 +430,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #2 cmp I, xzr - ble .Lzgemv_n_kernel_S1 + ble L(zgemv_n_kernel_S1) -.Lzgemv_n_kernel_S4: +L(zgemv_n_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -440,27 +440,27 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lzgemv_n_kernel_S4 + bne L(zgemv_n_kernel_S4) -.Lzgemv_n_kernel_S1: +L(zgemv_n_kernel_S1): ands I, M, #3 - ble .Lzgemv_n_kernel_S_END + ble L(zgemv_n_kernel_S_END) -.Lzgemv_n_kernel_S10: +L(zgemv_n_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lzgemv_n_kernel_S10 + bne L(zgemv_n_kernel_S10) -.Lzgemv_n_kernel_S_END: +L(zgemv_n_kernel_S_END): add A, A, LDA subs J, J, #1 - bne .Lzgemv_n_kernel_S_LOOP + bne L(zgemv_n_kernel_S_LOOP) -.Lzgemv_n_kernel_L999: +L(zgemv_n_kernel_L999): RESTORE_REGS mov w0, wzr diff --git a/kernel/arm64/zgemv_t.S b/kernel/arm64/zgemv_t.S index 0151029c77..38320c33da 100644 --- a/kernel/arm64/zgemv_t.S +++ b/kernel/arm64/zgemv_t.S @@ -292,9 +292,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SAVE_REGS cmp N, xzr - ble .Lzgemv_t_kernel_L999 + ble L(zgemv_t_kernel_L999) cmp M, xzr - ble .Lzgemv_t_kernel_L999 + ble L(zgemv_t_kernel_L999) lsl LDA, LDA, #SHZ lsl INC_Y, INC_Y, #SHZ @@ -303,9 +303,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. INIT cmp INC_X, #1 - bne .Lzgemv_t_kernel_S_BEGIN + bne L(zgemv_t_kernel_S_BEGIN) -.Lzgemv_t_kernel_F_LOOP: +L(zgemv_t_kernel_F_LOOP): mov A_PTR, A mov X_PTR, X @@ -314,30 +314,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #2 cmp I, xzr - beq .Lzgemv_t_kernel_F1 + beq L(zgemv_t_kernel_F1) -.Lzgemv_t_kernel_F4: +L(zgemv_t_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lzgemv_t_kernel_F4 + bne L(zgemv_t_kernel_F4) KERNEL_F4_FINALIZE -.Lzgemv_t_kernel_F1: +L(zgemv_t_kernel_F1): ands I, M, #3 - ble .Lzgemv_t_kernel_F_END + ble L(zgemv_t_kernel_F_END) -.Lzgemv_t_kernel_F10: +L(zgemv_t_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lzgemv_t_kernel_F10 + bne L(zgemv_t_kernel_F10) -.Lzgemv_t_kernel_F_END: +L(zgemv_t_kernel_F_END): #if !defined(DOUBLE) ld1 {v4.2s}, [Y] @@ -355,15 +355,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add A, A, LDA subs J, J, #1 - bne .Lzgemv_t_kernel_F_LOOP + bne L(zgemv_t_kernel_F_LOOP) - b .Lzgemv_t_kernel_L999 + b L(zgemv_t_kernel_L999) -.Lzgemv_t_kernel_S_BEGIN: +L(zgemv_t_kernel_S_BEGIN): INIT_S -.Lzgemv_t_kernel_S_LOOP: +L(zgemv_t_kernel_S_LOOP): mov A_PTR, A mov X_PTR, X @@ -371,9 +371,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #2 cmp I, xzr - ble .Lzgemv_t_kernel_S1 + ble L(zgemv_t_kernel_S1) -.Lzgemv_t_kernel_S4: +L(zgemv_t_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -381,21 +381,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lzgemv_t_kernel_S4 + bne L(zgemv_t_kernel_S4) -.Lzgemv_t_kernel_S1: +L(zgemv_t_kernel_S1): ands I, M, #3 - ble .Lzgemv_t_kernel_S_END + ble L(zgemv_t_kernel_S_END) -.Lzgemv_t_kernel_S10: +L(zgemv_t_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lzgemv_t_kernel_S10 + bne L(zgemv_t_kernel_S10) -.Lzgemv_t_kernel_S_END: +L(zgemv_t_kernel_S_END): #if !defined(DOUBLE) ld1 {v4.2s}, [Y] @@ -413,9 +413,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add A, A, LDA subs J, J, #1 - bne .Lzgemv_t_kernel_S_LOOP + bne L(zgemv_t_kernel_S_LOOP) -.Lzgemv_t_kernel_L999: +L(zgemv_t_kernel_L999): RESTORE_REGS mov w0, wzr ret diff --git a/kernel/arm64/znrm2.S b/kernel/arm64/znrm2.S index a530b80f0d..99289a6e81 100644 --- a/kernel/arm64/znrm2.S +++ b/kernel/arm64/znrm2.S @@ -226,43 +226,43 @@ KERNEL_S1_END: INIT cmp N, #0 - ble .Lznrm2_kernel_L999 + ble L(znrm2_kernel_L999) cmp INC_X, #0 - beq .Lznrm2_kernel_L999 + beq L(znrm2_kernel_L999) cmp INC_X, #1 - bne .Lznrm2_kernel_S_BEGIN + bne L(znrm2_kernel_S_BEGIN) -.Lznrm2_kernel_F_BEGIN: +L(znrm2_kernel_F_BEGIN): asr I, N, #3 // I = N / 8 cmp I, xzr - ble .Lznrm2_kernel_F1 + ble L(znrm2_kernel_F1) -.Lznrm2_kernel_F8: +L(znrm2_kernel_F8): KERNEL_F8 subs I, I, #1 - bne .Lznrm2_kernel_F8 + bne L(znrm2_kernel_F8) -.Lznrm2_kernel_F1: +L(znrm2_kernel_F1): ands I, N, #7 - ble .Lznrm2_kernel_L999 + ble L(znrm2_kernel_L999) -.Lznrm2_kernel_F10: +L(znrm2_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lznrm2_kernel_F10 + bne L(znrm2_kernel_F10) - b .Lznrm2_kernel_L999 + b L(znrm2_kernel_L999) -.Lznrm2_kernel_S_BEGIN: +L(znrm2_kernel_S_BEGIN): INIT_S @@ -270,15 +270,15 @@ KERNEL_S1_END: .align 5 -.Lznrm2_kernel_S10: +L(znrm2_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lznrm2_kernel_S10 + bne L(znrm2_kernel_S10) -.Lznrm2_kernel_L999: +L(znrm2_kernel_L999): fsqrt SSQ, SSQ fmul SSQ, SCALE, SSQ diff --git a/kernel/arm64/zrot.S b/kernel/arm64/zrot.S index b5e510ebea..c6bdefae3f 100644 --- a/kernel/arm64/zrot.S +++ b/kernel/arm64/zrot.S @@ -181,54 +181,54 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE cmp N, xzr - ble .Lzrot_kernel_L999 + ble L(zrot_kernel_L999) INIT cmp INC_X, #1 - bne .Lzrot_kernel_S_BEGIN + bne L(zrot_kernel_S_BEGIN) cmp INC_Y, #1 - bne .Lzrot_kernel_S_BEGIN + bne L(zrot_kernel_S_BEGIN) -.Lzrot_kernel_F_BEGIN: +L(zrot_kernel_F_BEGIN): asr I, N, #2 cmp I, xzr - beq .Lzrot_kernel_F1 + beq L(zrot_kernel_F1) KERNEL_INIT_F4 -.Lzrot_kernel_F4: +L(zrot_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lzrot_kernel_F4 + bne L(zrot_kernel_F4) -.Lzrot_kernel_F1: +L(zrot_kernel_F1): ands I, N, #3 - ble .Lzrot_kernel_L999 + ble L(zrot_kernel_L999) -.Lzrot_kernel_F10: +L(zrot_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lzrot_kernel_F10 + bne L(zrot_kernel_F10) mov w0, wzr ret -.Lzrot_kernel_S_BEGIN: +L(zrot_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lzrot_kernel_S1 + ble L(zrot_kernel_S1) -.Lzrot_kernel_S4: +L(zrot_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -236,21 +236,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lzrot_kernel_S4 + bne L(zrot_kernel_S4) -.Lzrot_kernel_S1: +L(zrot_kernel_S1): ands I, N, #3 - ble .Lzrot_kernel_L999 + ble L(zrot_kernel_L999) -.Lzrot_kernel_S10: +L(zrot_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lzrot_kernel_S10 + bne L(zrot_kernel_S10) -.Lzrot_kernel_L999: +L(zrot_kernel_L999): mov w0, wzr ret diff --git a/kernel/arm64/zscal.S b/kernel/arm64/zscal.S index 97d8a8b7ad..15d9e3bb2e 100644 --- a/kernel/arm64/zscal.S +++ b/kernel/arm64/zscal.S @@ -215,74 +215,74 @@ zscal_begin: mov X_COPY, X cmp N, xzr - ble .Lzscal_kernel_L999 + ble L(zscal_kernel_L999) ldr FLAG, [sp] cmp FLAG, #1 -beq .Lzscal_kernel_RI_non_zero +beq L(zscal_kernel_RI_non_zero) fcmp DA_R, #0.0 - bne .Lzscal_kernel_R_non_zero + bne L(zscal_kernel_R_non_zero) fcmp DA_I, #0.0 - beq .Lzscal_kernel_RI_zero + beq L(zscal_kernel_RI_zero) -// b .Lzscal_kernel_R_zero +// b L(zscal_kernel_R_zero) -.Lzscal_kernel_R_non_zero: +L(zscal_kernel_R_non_zero): fcmp DA_I, #0.0 -//QUAK beq .Lzscal_kernel_I_zero +//QUAK beq L(zscal_kernel_I_zero) /******************************************************************************* * A_R != 0 && A_I != 0 *******************************************************************************/ -.Lzscal_kernel_RI_non_zero: +L(zscal_kernel_RI_non_zero): INIT cmp INC_X, #1 - bne .Lzscal_kernel_S_BEGIN + bne L(zscal_kernel_S_BEGIN) -.Lzscal_kernel_F_BEGIN: +L(zscal_kernel_F_BEGIN): asr I, N, #2 cmp I, xzr - beq .Lzscal_kernel_F1 + beq L(zscal_kernel_F1) KERNEL_INIT_F4 -.Lzscal_kernel_F4: +L(zscal_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lzscal_kernel_F4 + bne L(zscal_kernel_F4) -.Lzscal_kernel_F1: +L(zscal_kernel_F1): ands I, N, #3 - ble .Lzscal_kernel_L999 + ble L(zscal_kernel_L999) -.Lzscal_kernel_F10: +L(zscal_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lzscal_kernel_F10 + bne L(zscal_kernel_F10) mov w0, wzr ret -.Lzscal_kernel_S_BEGIN: +L(zscal_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lzscal_kernel_S1 + ble L(zscal_kernel_S1) -.Lzscal_kernel_S4: +L(zscal_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -290,21 +290,21 @@ beq .Lzscal_kernel_RI_non_zero KERNEL_S1 subs I, I, #1 - bne .Lzscal_kernel_S4 + bne L(zscal_kernel_S4) -.Lzscal_kernel_S1: +L(zscal_kernel_S1): ands I, N, #3 - ble .Lzscal_kernel_L999 + ble L(zscal_kernel_L999) -.Lzscal_kernel_S10: +L(zscal_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lzscal_kernel_S10 + bne L(zscal_kernel_S10) -.Lzscal_kernel_L999: +L(zscal_kernel_L999): mov w0, wzr ret @@ -313,7 +313,7 @@ beq .Lzscal_kernel_RI_non_zero * A_R == 0 && A_I != 0 *******************************************************************************/ -.Lzscal_kernel_R_zero: +L(zscal_kernel_R_zero): INIT_S #if !defined(DOUBLE) @@ -326,7 +326,7 @@ beq .Lzscal_kernel_RI_non_zero ins v1.d[1], v2.d[0] // v1 = -DA_I, DA_I #endif -.Lzscal_kernel_R_zero_1: +L(zscal_kernel_R_zero_1): #if !defined(DOUBLE) ld1 {v2.2s}, [X] // X1, X0 fmul v2.2s, v2.2s, v1.2s // -DA_I*X1, DA_I*X0 @@ -340,7 +340,7 @@ beq .Lzscal_kernel_RI_non_zero #endif add X, X, INC_X subs N, N, #1 - bne .Lzscal_kernel_R_zero_1 + bne L(zscal_kernel_R_zero_1) mov w0, wzr ret @@ -349,7 +349,7 @@ beq .Lzscal_kernel_RI_non_zero * A_R != 0 && A_I == 0 *******************************************************************************/ -.Lzscal_kernel_I_zero: +L(zscal_kernel_I_zero): INIT_S #if !defined(DOUBLE) ins v0.s[1], v0.s[0] // v0 = DA_R, DA_R @@ -357,7 +357,7 @@ beq .Lzscal_kernel_RI_non_zero ins v0.d[1], v0.d[0] // v0 = DA_R, DA_R #endif -.Lzscal_kernel_I_zero_1: +L(zscal_kernel_I_zero_1): #if !defined(DOUBLE) ld1 {v2.2s}, [X] // X1, X0 fmul v2.2s, v2.2s, v0.2s // DA_R*X1, DA_R*X0 @@ -369,7 +369,7 @@ beq .Lzscal_kernel_RI_non_zero #endif add X, X, INC_X subs N, N, #1 - bne .Lzscal_kernel_I_zero_1 + bne L(zscal_kernel_I_zero_1) mov w0, wzr ret @@ -378,16 +378,16 @@ beq .Lzscal_kernel_RI_non_zero * A_R == 0 && A_I == 0 *******************************************************************************/ -.Lzscal_kernel_RI_zero: +L(zscal_kernel_RI_zero): INIT_S -.Lzscal_kernel_RI_zero_1: +L(zscal_kernel_RI_zero_1): stp DA_R, DA_I, [X] add X, X, INC_X subs N, N, #1 - bne .Lzscal_kernel_RI_zero_1 + bne L(zscal_kernel_RI_zero_1) mov w0, wzr ret diff --git a/kernel/arm64/zsum.S b/kernel/arm64/zsum.S index 67ea3cb4d0..27d8716991 100644 --- a/kernel/arm64/zsum.S +++ b/kernel/arm64/zsum.S @@ -86,52 +86,52 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. fmov SUMF, REG0 cmp N, xzr - ble .Lzsum_kernel_L999 + ble L(zsum_kernel_L999) cmp INC_X, xzr - ble .Lzsum_kernel_L999 + ble L(zsum_kernel_L999) cmp INC_X, #1 - bne .Lzsum_kernel_S_BEGIN + bne L(zsum_kernel_S_BEGIN) -.Lzsum_kernel_F_BEGIN: +L(zsum_kernel_F_BEGIN): asr I, N, #2 cmp I, xzr - beq .Lzsum_kernel_F1 + beq L(zsum_kernel_F1) -.Lzsum_kernel_F4: +L(zsum_kernel_F4): KERNEL_F4 subs I, I, #1 - bne .Lzsum_kernel_F4 + bne L(zsum_kernel_F4) KERNEL_F4_FINALIZE -.Lzsum_kernel_F1: +L(zsum_kernel_F1): ands I, N, #3 - ble .Lzsum_kernel_L999 + ble L(zsum_kernel_L999) -.Lzsum_kernel_F10: +L(zsum_kernel_F10): KERNEL_F1 subs I, I, #1 - bne .Lzsum_kernel_F10 + bne L(zsum_kernel_F10) -.Lzsum_kernel_L999: +L(zsum_kernel_L999): ret -.Lzsum_kernel_S_BEGIN: +L(zsum_kernel_S_BEGIN): INIT_S asr I, N, #2 cmp I, xzr - ble .Lzsum_kernel_S1 + ble L(zsum_kernel_S1) -.Lzsum_kernel_S4: +L(zsum_kernel_S4): KERNEL_S1 KERNEL_S1 @@ -139,19 +139,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_S1 subs I, I, #1 - bne .Lzsum_kernel_S4 + bne L(zsum_kernel_S4) -.Lzsum_kernel_S1: +L(zsum_kernel_S1): ands I, N, #3 - ble .Lzsum_kernel_L999 + ble L(zsum_kernel_L999) -.Lzsum_kernel_S10: +L(zsum_kernel_S10): KERNEL_S1 subs I, I, #1 - bne .Lzsum_kernel_S10 + bne L(zsum_kernel_S10) ret diff --git a/kernel/arm64/ztrmm_kernel_4x4.S b/kernel/arm64/ztrmm_kernel_4x4.S index cd053b896c..b1874903dd 100644 --- a/kernel/arm64/ztrmm_kernel_4x4.S +++ b/kernel/arm64/ztrmm_kernel_4x4.S @@ -1078,9 +1078,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lztrmm_kernel_L2_BEGIN + ble L(ztrmm_kernel_L2_BEGIN) -.Lztrmm_kernel_L4_BEGIN: +L(ztrmm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -1094,15 +1094,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = start of A array -.Lztrmm_kernel_L4_M4_BEGIN: +L(ztrmm_kernel_L4_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lztrmm_kernel_L4_M2_BEGIN + ble L(ztrmm_kernel_L4_M2_BEGIN) .align 5 -.Lztrmm_kernel_L4_M4_20: +L(ztrmm_kernel_L4_M4_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -1123,7 +1123,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 cmp counterL , #2 - blt .Lztrmm_kernel_L4_M4_32 + blt L(ztrmm_kernel_L4_M4_32) KERNEL4x4_I KERNEL4x4_M2 @@ -1135,10 +1135,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 subs counterL, counterL, #2 - ble .Lztrmm_kernel_L4_M4_22a + ble L(ztrmm_kernel_L4_M4_22a) .align 5 -.Lztrmm_kernel_L4_M4_22: +L(ztrmm_kernel_L4_M4_22): KERNEL4x4_M1 KERNEL4x4_M2 @@ -1150,10 +1150,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M2 subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L4_M4_22 + bgt L(ztrmm_kernel_L4_M4_22) .align 5 -.Lztrmm_kernel_L4_M4_22a: +L(ztrmm_kernel_L4_M4_22a): KERNEL4x4_M1 KERNEL4x4_M2 @@ -1164,13 +1164,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - b .Lztrmm_kernel_L4_M4_44 + b L(ztrmm_kernel_L4_M4_44) .align 5 -.Lztrmm_kernel_L4_M4_32: +L(ztrmm_kernel_L4_M4_32): tst counterL, #1 - ble .Lztrmm_kernel_L4_M4_40 + ble L(ztrmm_kernel_L4_M4_40) KERNEL4x4_I KERNEL4x4_M2 @@ -1181,26 +1181,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x4_M1 KERNEL4x4_E - b .Lztrmm_kernel_L4_M4_44 + b L(ztrmm_kernel_L4_M4_44) -.Lztrmm_kernel_L4_M4_40: +L(ztrmm_kernel_L4_M4_40): INIT4x4 -.Lztrmm_kernel_L4_M4_44: +L(ztrmm_kernel_L4_M4_44): ands counterL , tempK, #7 - ble .Lztrmm_kernel_L4_M4_100 + ble L(ztrmm_kernel_L4_M4_100) .align 5 -.Lztrmm_kernel_L4_M4_46: +L(ztrmm_kernel_L4_M4_46): KERNEL4x4_SUB subs counterL, counterL, #1 - bne .Lztrmm_kernel_L4_M4_46 + bne L(ztrmm_kernel_L4_M4_46) -.Lztrmm_kernel_L4_M4_100: +L(ztrmm_kernel_L4_M4_100): SAVE4x4 @@ -1223,20 +1223,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. prfm PLDL1KEEP, [pA, #64] prfm PLDL1KEEP, [origPB] -.Lztrmm_kernel_L4_M4_END: +L(ztrmm_kernel_L4_M4_END): subs counterI, counterI, #1 - bne .Lztrmm_kernel_L4_M4_20 + bne L(ztrmm_kernel_L4_M4_20) -.Lztrmm_kernel_L4_M2_BEGIN: +L(ztrmm_kernel_L4_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lztrmm_kernel_L4_END + ble L(ztrmm_kernel_L4_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lztrmm_kernel_L4_M1_BEGIN + ble L(ztrmm_kernel_L4_M1_BEGIN) -.Lztrmm_kernel_L4_M2_20: +L(ztrmm_kernel_L4_M2_20): INIT2x4 @@ -1260,9 +1260,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lztrmm_kernel_L4_M2_40 + ble L(ztrmm_kernel_L4_M2_40) -.Lztrmm_kernel_L4_M2_22: +L(ztrmm_kernel_L4_M2_22): KERNEL2x4_SUB KERNEL2x4_SUB @@ -1275,22 +1275,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L4_M2_22 + bgt L(ztrmm_kernel_L4_M2_22) -.Lztrmm_kernel_L4_M2_40: +L(ztrmm_kernel_L4_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lztrmm_kernel_L4_M2_100 + ble L(ztrmm_kernel_L4_M2_100) -.Lztrmm_kernel_L4_M2_42: +L(ztrmm_kernel_L4_M2_42): KERNEL2x4_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L4_M2_42 + bgt L(ztrmm_kernel_L4_M2_42) -.Lztrmm_kernel_L4_M2_100: +L(ztrmm_kernel_L4_M2_100): SAVE2x4 @@ -1310,15 +1310,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lztrmm_kernel_L4_M2_END: +L(ztrmm_kernel_L4_M2_END): -.Lztrmm_kernel_L4_M1_BEGIN: +L(ztrmm_kernel_L4_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lztrmm_kernel_L4_END + ble L(ztrmm_kernel_L4_END) -.Lztrmm_kernel_L4_M1_20: +L(ztrmm_kernel_L4_M1_20): INIT1x4 @@ -1342,9 +1342,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lztrmm_kernel_L4_M1_40 + ble L(ztrmm_kernel_L4_M1_40) -.Lztrmm_kernel_L4_M1_22: +L(ztrmm_kernel_L4_M1_22): KERNEL1x4_SUB KERNEL1x4_SUB KERNEL1x4_SUB @@ -1356,22 +1356,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L4_M1_22 + bgt L(ztrmm_kernel_L4_M1_22) -.Lztrmm_kernel_L4_M1_40: +L(ztrmm_kernel_L4_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lztrmm_kernel_L4_M1_100 + ble L(ztrmm_kernel_L4_M1_100) -.Lztrmm_kernel_L4_M1_42: +L(ztrmm_kernel_L4_M1_42): KERNEL1x4_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L4_M1_42 + bgt L(ztrmm_kernel_L4_M1_42) -.Lztrmm_kernel_L4_M1_100: +L(ztrmm_kernel_L4_M1_100): SAVE1x4 @@ -1392,7 +1392,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.Lztrmm_kernel_L4_END: +L(ztrmm_kernel_L4_END): lsl temp, origK, #6 add origPB, origPB, temp // B = B + K * 4 * 8 * 2 @@ -1402,19 +1402,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Lztrmm_kernel_L4_BEGIN + bgt L(ztrmm_kernel_L4_BEGIN) /******************************************************************************/ -.Lztrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(ztrmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lztrmm_kernel_L999 // error, N was less than 4? + ble L(ztrmm_kernel_L999) // error, N was less than 4? tst counterJ , #2 - ble .Lztrmm_kernel_L1_BEGIN + ble L(ztrmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC @@ -1426,14 +1426,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lztrmm_kernel_L2_M4_BEGIN: +L(ztrmm_kernel_L2_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI,#0 - ble .Lztrmm_kernel_L2_M2_BEGIN + ble L(ztrmm_kernel_L2_M2_BEGIN) -.Lztrmm_kernel_L2_M4_20: +L(ztrmm_kernel_L2_M4_20): INIT4x2 @@ -1457,10 +1457,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lztrmm_kernel_L2_M4_40 + ble L(ztrmm_kernel_L2_M4_40) .align 5 -.Lztrmm_kernel_L2_M4_22: +L(ztrmm_kernel_L2_M4_22): KERNEL4x2_SUB KERNEL4x2_SUB KERNEL4x2_SUB @@ -1472,22 +1472,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L2_M4_22 + bgt L(ztrmm_kernel_L2_M4_22) -.Lztrmm_kernel_L2_M4_40: +L(ztrmm_kernel_L2_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lztrmm_kernel_L2_M4_100 + ble L(ztrmm_kernel_L2_M4_100) -.Lztrmm_kernel_L2_M4_42: +L(ztrmm_kernel_L2_M4_42): KERNEL4x2_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L2_M4_42 + bgt L(ztrmm_kernel_L2_M4_42) -.Lztrmm_kernel_L2_M4_100: +L(ztrmm_kernel_L2_M4_100): SAVE4x2 @@ -1507,22 +1507,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lztrmm_kernel_L2_M4_END: +L(ztrmm_kernel_L2_M4_END): subs counterI, counterI, #1 - bgt .Lztrmm_kernel_L2_M4_20 + bgt L(ztrmm_kernel_L2_M4_20) -.Lztrmm_kernel_L2_M2_BEGIN: +L(ztrmm_kernel_L2_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lztrmm_kernel_L2_END + ble L(ztrmm_kernel_L2_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lztrmm_kernel_L2_M1_BEGIN + ble L(ztrmm_kernel_L2_M1_BEGIN) -.Lztrmm_kernel_L2_M2_20: +L(ztrmm_kernel_L2_M2_20): INIT2x2 @@ -1546,9 +1546,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lztrmm_kernel_L2_M2_40 + ble L(ztrmm_kernel_L2_M2_40) -.Lztrmm_kernel_L2_M2_22: +L(ztrmm_kernel_L2_M2_22): KERNEL2x2_SUB KERNEL2x2_SUB @@ -1561,22 +1561,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L2_M2_22 + bgt L(ztrmm_kernel_L2_M2_22) -.Lztrmm_kernel_L2_M2_40: +L(ztrmm_kernel_L2_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lztrmm_kernel_L2_M2_100 + ble L(ztrmm_kernel_L2_M2_100) -.Lztrmm_kernel_L2_M2_42: +L(ztrmm_kernel_L2_M2_42): KERNEL2x2_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L2_M2_42 + bgt L(ztrmm_kernel_L2_M2_42) -.Lztrmm_kernel_L2_M2_100: +L(ztrmm_kernel_L2_M2_100): SAVE2x2 @@ -1596,15 +1596,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lztrmm_kernel_L2_M2_END: +L(ztrmm_kernel_L2_M2_END): -.Lztrmm_kernel_L2_M1_BEGIN: +L(ztrmm_kernel_L2_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lztrmm_kernel_L2_END + ble L(ztrmm_kernel_L2_END) -.Lztrmm_kernel_L2_M1_20: +L(ztrmm_kernel_L2_M1_20): INIT1x2 @@ -1628,9 +1628,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL, #0 - ble .Lztrmm_kernel_L2_M1_40 + ble L(ztrmm_kernel_L2_M1_40) -.Lztrmm_kernel_L2_M1_22: +L(ztrmm_kernel_L2_M1_22): KERNEL1x2_SUB KERNEL1x2_SUB KERNEL1x2_SUB @@ -1642,22 +1642,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L2_M1_22 + bgt L(ztrmm_kernel_L2_M1_22) -.Lztrmm_kernel_L2_M1_40: +L(ztrmm_kernel_L2_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lztrmm_kernel_L2_M1_100 + ble L(ztrmm_kernel_L2_M1_100) -.Lztrmm_kernel_L2_M1_42: +L(ztrmm_kernel_L2_M1_42): KERNEL1x2_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L2_M1_42 + bgt L(ztrmm_kernel_L2_M1_42) -.Lztrmm_kernel_L2_M1_100: +L(ztrmm_kernel_L2_M1_100): SAVE1x2 @@ -1678,7 +1678,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -.Lztrmm_kernel_L2_END: +L(ztrmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -1688,11 +1688,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lztrmm_kernel_L1_BEGIN: +L(ztrmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lztrmm_kernel_L999 // done + ble L(ztrmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -1706,14 +1706,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lztrmm_kernel_L1_M4_BEGIN: +L(ztrmm_kernel_L1_M4_BEGIN): mov counterI, origM asr counterI, counterI, #2 // counterI = counterI / 4 cmp counterI, #0 - ble .Lztrmm_kernel_L1_M2_BEGIN + ble L(ztrmm_kernel_L1_M2_BEGIN) -.Lztrmm_kernel_L1_M4_20: +L(ztrmm_kernel_L1_M4_20): INIT4x1 @@ -1737,10 +1737,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lztrmm_kernel_L1_M4_40 + ble L(ztrmm_kernel_L1_M4_40) .align 5 -.Lztrmm_kernel_L1_M4_22: +L(ztrmm_kernel_L1_M4_22): KERNEL4x1_SUB KERNEL4x1_SUB KERNEL4x1_SUB @@ -1752,22 +1752,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L1_M4_22 + bgt L(ztrmm_kernel_L1_M4_22) -.Lztrmm_kernel_L1_M4_40: +L(ztrmm_kernel_L1_M4_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lztrmm_kernel_L1_M4_100 + ble L(ztrmm_kernel_L1_M4_100) -.Lztrmm_kernel_L1_M4_42: +L(ztrmm_kernel_L1_M4_42): KERNEL4x1_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L1_M4_42 + bgt L(ztrmm_kernel_L1_M4_42) -.Lztrmm_kernel_L1_M4_100: +L(ztrmm_kernel_L1_M4_100): SAVE4x1 @@ -1787,22 +1787,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #4 #endif -.Lztrmm_kernel_L1_M4_END: +L(ztrmm_kernel_L1_M4_END): subs counterI, counterI, #1 - bgt .Lztrmm_kernel_L1_M4_20 + bgt L(ztrmm_kernel_L1_M4_20) -.Lztrmm_kernel_L1_M2_BEGIN: +L(ztrmm_kernel_L1_M2_BEGIN): mov counterI, origM tst counterI , #3 - ble .Lztrmm_kernel_L1_END + ble L(ztrmm_kernel_L1_END) tst counterI, #2 // counterI = counterI / 2 - ble .Lztrmm_kernel_L1_M1_BEGIN + ble L(ztrmm_kernel_L1_M1_BEGIN) -.Lztrmm_kernel_L1_M2_20: +L(ztrmm_kernel_L1_M2_20): INIT2x1 @@ -1826,9 +1826,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lztrmm_kernel_L1_M2_40 + ble L(ztrmm_kernel_L1_M2_40) -.Lztrmm_kernel_L1_M2_22: +L(ztrmm_kernel_L1_M2_22): KERNEL2x1_SUB KERNEL2x1_SUB @@ -1841,22 +1841,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L1_M2_22 + bgt L(ztrmm_kernel_L1_M2_22) -.Lztrmm_kernel_L1_M2_40: +L(ztrmm_kernel_L1_M2_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lztrmm_kernel_L1_M2_100 + ble L(ztrmm_kernel_L1_M2_100) -.Lztrmm_kernel_L1_M2_42: +L(ztrmm_kernel_L1_M2_42): KERNEL2x1_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L1_M2_42 + bgt L(ztrmm_kernel_L1_M2_42) -.Lztrmm_kernel_L1_M2_100: +L(ztrmm_kernel_L1_M2_100): SAVE2x1 @@ -1876,15 +1876,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, #2 #endif -.Lztrmm_kernel_L1_M2_END: +L(ztrmm_kernel_L1_M2_END): -.Lztrmm_kernel_L1_M1_BEGIN: +L(ztrmm_kernel_L1_M1_BEGIN): tst counterI, #1 // counterI = counterI % 2 - ble .Lztrmm_kernel_L1_END + ble L(ztrmm_kernel_L1_END) -.Lztrmm_kernel_L1_M1_20: +L(ztrmm_kernel_L1_M1_20): INIT1x1 @@ -1908,9 +1908,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lztrmm_kernel_L1_M1_40 + ble L(ztrmm_kernel_L1_M1_40) -.Lztrmm_kernel_L1_M1_22: +L(ztrmm_kernel_L1_M1_22): KERNEL1x1_SUB KERNEL1x1_SUB KERNEL1x1_SUB @@ -1922,30 +1922,30 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L1_M1_22 + bgt L(ztrmm_kernel_L1_M1_22) -.Lztrmm_kernel_L1_M1_40: +L(ztrmm_kernel_L1_M1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lztrmm_kernel_L1_M1_100 + ble L(ztrmm_kernel_L1_M1_100) -.Lztrmm_kernel_L1_M1_42: +L(ztrmm_kernel_L1_M1_42): KERNEL1x1_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L1_M1_42 + bgt L(ztrmm_kernel_L1_M1_42) -.Lztrmm_kernel_L1_M1_100: +L(ztrmm_kernel_L1_M1_100): SAVE1x1 -.Lztrmm_kernel_L1_END: +L(ztrmm_kernel_L1_END): -.Lztrmm_kernel_L999: +L(ztrmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] diff --git a/kernel/arm64/ztrmm_kernel_sve_v1x4.S b/kernel/arm64/ztrmm_kernel_sve_v1x4.S index b71a3d39ed..22be5a82c5 100644 --- a/kernel/arm64/ztrmm_kernel_sve_v1x4.S +++ b/kernel/arm64/ztrmm_kernel_sve_v1x4.S @@ -584,10 +584,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov counterJ, origN asr counterJ, counterJ, #2 // J = J / 4 cmp counterJ, #0 - ble .Lztrmm_kernel_L2_BEGIN + ble L(ztrmm_kernel_L2_BEGIN) /******************************************************************************/ -.Lztrmm_kernel_L4_BEGIN: +L(ztrmm_kernel_L4_BEGIN): mov pCRow0, pC add pCRow1, pCRow0, LDC add pCRow2, pCRow1, LDC @@ -600,7 +600,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif mov pA, origPA // pA = start of A array -.Lztrmm_kernel_L4_Mv1_BEGIN: +L(ztrmm_kernel_L4_Mv1_BEGIN): /* Loop over M is done in an SVE fashion. This has the benefit of the last M%SVE_LEN iterations being done in a single sweep */ mov counterI, #0 @@ -608,7 +608,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension .align 5 -.Lztrmm_kernel_L4_Mv1_20: +L(ztrmm_kernel_L4_Mv1_20): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) mov pB, origPB @@ -631,7 +631,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 cmp counterL , #2 - blt .Lztrmm_kernel_L4_Mv1_32 + blt L(ztrmm_kernel_L4_Mv1_32) KERNELv1x4_I KERNELv1x4_M2 @@ -643,10 +643,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M2 subs counterL, counterL, #2 // subtract 2 - ble .Lztrmm_kernel_L4_Mv1_22a + ble L(ztrmm_kernel_L4_Mv1_22a) .align 5 -.Lztrmm_kernel_L4_Mv1_22: +L(ztrmm_kernel_L4_Mv1_22): KERNELv1x4_M1 KERNELv1x4_M2 @@ -658,10 +658,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M2 subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L4_Mv1_22 + bgt L(ztrmm_kernel_L4_Mv1_22) .align 5 -.Lztrmm_kernel_L4_Mv1_22a: +L(ztrmm_kernel_L4_Mv1_22a): KERNELv1x4_M1 KERNELv1x4_M2 @@ -672,13 +672,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M1 KERNELv1x4_E - b .Lztrmm_kernel_L4_Mv1_44 + b L(ztrmm_kernel_L4_Mv1_44) .align 5 -.Lztrmm_kernel_L4_Mv1_32: +L(ztrmm_kernel_L4_Mv1_32): tst counterL, #1 - ble .Lztrmm_kernel_L4_Mv1_40 + ble L(ztrmm_kernel_L4_Mv1_40) KERNELv1x4_I KERNELv1x4_M2 @@ -689,26 +689,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x4_M1 KERNELv1x4_E - b .Lztrmm_kernel_L4_Mv1_44 + b L(ztrmm_kernel_L4_Mv1_44) -.Lztrmm_kernel_L4_Mv1_40: +L(ztrmm_kernel_L4_Mv1_40): INITv1x4 -.Lztrmm_kernel_L4_Mv1_44: +L(ztrmm_kernel_L4_Mv1_44): ands counterL , tempK, #7 - ble .Lztrmm_kernel_L4_Mv1_100 + ble L(ztrmm_kernel_L4_Mv1_100) .align 5 -.Lztrmm_kernel_L4_Mv1_46: +L(ztrmm_kernel_L4_Mv1_46): KERNELv1x4_SUB subs counterL, counterL, #1 - bne .Lztrmm_kernel_L4_Mv1_46 + bne L(ztrmm_kernel_L4_Mv1_46) -.Lztrmm_kernel_L4_Mv1_100: +L(ztrmm_kernel_L4_Mv1_100): #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) sub tempK, origK, tempOffset @@ -732,16 +732,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SAVEv1x4 -.Lztrmm_kernel_L4_Mv1_END: +L(ztrmm_kernel_L4_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension - b.any .Lztrmm_kernel_L4_Mv1_20 + b.any L(ztrmm_kernel_L4_Mv1_20) -.Lztrmm_kernel_L4_END: +L(ztrmm_kernel_L4_END): lsl temp, origK, #6 add origPB, origPB, temp // B = B + K * 4 * 8 * 2 @@ -751,19 +751,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif subs counterJ, counterJ , #1 // j-- - bgt .Lztrmm_kernel_L4_BEGIN + bgt L(ztrmm_kernel_L4_BEGIN) /******************************************************************************/ -.Lztrmm_kernel_L2_BEGIN: // less than 2 left in N direction +L(ztrmm_kernel_L2_BEGIN): // less than 2 left in N direction mov counterJ , origN tst counterJ , #3 - ble .Lztrmm_kernel_L999 + ble L(ztrmm_kernel_L999) tst counterJ , #2 - ble .Lztrmm_kernel_L1_BEGIN + ble L(ztrmm_kernel_L1_BEGIN) mov pCRow0, pC // pCRow0 = pC add pCRow1, pCRow0, LDC @@ -778,14 +778,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Lztrmm_kernel_L2_Mv1_BEGIN: +L(ztrmm_kernel_L2_Mv1_BEGIN): mov counterI, #0 whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d -.Lztrmm_kernel_L2_Mv1_20: +L(ztrmm_kernel_L2_Mv1_20): INITv1x2 @@ -809,10 +809,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL,#0 - ble .Lztrmm_kernel_L2_Mv1_40 + ble L(ztrmm_kernel_L2_Mv1_40) .align 5 -.Lztrmm_kernel_L2_Mv1_22: +L(ztrmm_kernel_L2_Mv1_22): KERNELv1x2_SUB KERNELv1x2_SUB KERNELv1x2_SUB @@ -824,22 +824,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L2_Mv1_22 + bgt L(ztrmm_kernel_L2_Mv1_22) -.Lztrmm_kernel_L2_Mv1_40: +L(ztrmm_kernel_L2_Mv1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lztrmm_kernel_L2_Mv1_100 + ble L(ztrmm_kernel_L2_Mv1_100) -.Lztrmm_kernel_L2_Mv1_42: +L(ztrmm_kernel_L2_Mv1_42): KERNELv1x2_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L2_Mv1_42 + bgt L(ztrmm_kernel_L2_Mv1_42) -.Lztrmm_kernel_L2_Mv1_100: +L(ztrmm_kernel_L2_Mv1_100): SAVEv1x2 @@ -859,16 +859,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, lanes #endif -.Lztrmm_kernel_L2_Mv1_END: +L(ztrmm_kernel_L2_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Lztrmm_kernel_L2_Mv1_20 + b.any L(ztrmm_kernel_L2_Mv1_20) -.Lztrmm_kernel_L2_END: +L(ztrmm_kernel_L2_END): #if !defined(LEFT) add tempOffset, tempOffset, #2 #endif @@ -878,11 +878,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /******************************************************************************/ -.Lztrmm_kernel_L1_BEGIN: +L(ztrmm_kernel_L1_BEGIN): mov counterJ , origN tst counterJ , #1 - ble .Lztrmm_kernel_L999 // done + ble L(ztrmm_kernel_L999) // done mov pCRow0, pC // pCRow0 = C @@ -894,14 +894,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mov pA, origPA // pA = A -.Lztrmm_kernel_L1_Mv1_BEGIN: +L(ztrmm_kernel_L1_Mv1_BEGIN): mov counterI, #0 whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d -.Lztrmm_kernel_L1_Mv1_20: +L(ztrmm_kernel_L1_Mv1_20): INITv1x1 @@ -925,10 +925,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr counterL , tempK, #3 // counterL = counterL / 8 cmp counterL , #0 - ble .Lztrmm_kernel_L1_Mv1_40 + ble L(ztrmm_kernel_L1_Mv1_40) .align 5 -.Lztrmm_kernel_L1_Mv1_22: +L(ztrmm_kernel_L1_Mv1_22): KERNELv1x1_SUB KERNELv1x1_SUB KERNELv1x1_SUB @@ -940,22 +940,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L1_Mv1_22 + bgt L(ztrmm_kernel_L1_Mv1_22) -.Lztrmm_kernel_L1_Mv1_40: +L(ztrmm_kernel_L1_Mv1_40): ands counterL , tempK, #7 // counterL = counterL % 8 - ble .Lztrmm_kernel_L1_Mv1_100 + ble L(ztrmm_kernel_L1_Mv1_100) -.Lztrmm_kernel_L1_Mv1_42: +L(ztrmm_kernel_L1_Mv1_42): KERNELv1x1_SUB subs counterL, counterL, #1 - bgt .Lztrmm_kernel_L1_Mv1_42 + bgt L(ztrmm_kernel_L1_Mv1_42) -.Lztrmm_kernel_L1_Mv1_100: +L(ztrmm_kernel_L1_Mv1_100): SAVEv1x1 @@ -975,18 +975,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add tempOffset, tempOffset, lanes #endif -.Lztrmm_kernel_L1_Mv1_END: +L(ztrmm_kernel_L1_Mv1_END): incd counterI whilelt p1.d, counterI, origM //SVE instruction cntp lanes, p0, p1.d - b.any .Lztrmm_kernel_L1_Mv1_20 + b.any L(ztrmm_kernel_L1_Mv1_20) -.Lztrmm_kernel_L1_END: +L(ztrmm_kernel_L1_END): /******************************************************************************/ -.Lztrmm_kernel_L999: +L(ztrmm_kernel_L999): mov x0, #0 // set return value ldp d8, d9, [sp, #(0 * 16)] ldp d10, d11, [sp, #(1 * 16)] From 797634ec0bdfe24c83b3c0c75e0f3c91730ecf06 Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Thu, 1 Oct 2026 23:23:15 -0400 Subject: [PATCH 4/5] Make the remaining labels of the x86_64 and arm64 kernels local A few kernels name labels in ways that the previous two commits did not reach: * x86_64/dgemm_kernel_6x4_piledriver.S: ._L__ * arm64/nrm2.S, znrm2.S, zscal.S: no prefix at all * arm64/sgemm_direct_sme1_*.S: .M_Loop, .K_Loop, ... * arm64/dznrm2_thunderx2t99_fast.c: .Lname, in inline assembly * x86_64/dgemm_kernel_4x8_skylakex.c: .label, in inline assembly All but the fourth are symbols for every assembler. Spell them L(name). The SME kernels include common.h for that, and end in EPILOGUE like the others. The two C kernels, which no KERNEL file uses at present, get an L() of their own, which makes the string. clang always uses .subsections_via_symbols for C, so on Darwin the first did not assemble ("conditional branch requires assembler-local label") and the second had its loops split into blocks. The text of each object is unchanged, on ELF and on Darwin, except that of dgemm_kernel_4x8_skylakex.c on Darwin, where the jumps to its labels are now short; the label symbols are gone from it. Co-Authored-By: Claude Fable 5.1 --- kernel/arm64/dznrm2_thunderx2t99_fast.c | 48 +-- kernel/arm64/nrm2.S | 10 +- kernel/arm64/sgemm_direct_sme1_2VLx2VL.S | 30 +- kernel/arm64/sgemm_direct_sme1_preprocess.S | 20 +- kernel/arm64/znrm2.S | 38 +- kernel/arm64/zscal.S | 20 +- kernel/x86_64/dgemm_kernel_4x8_skylakex.c | 24 +- kernel/x86_64/dgemm_kernel_6x4_piledriver.S | 370 ++++++++++---------- 8 files changed, 290 insertions(+), 270 deletions(-) diff --git a/kernel/arm64/dznrm2_thunderx2t99_fast.c b/kernel/arm64/dznrm2_thunderx2t99_fast.c index 90558c2a02..37339cd847 100644 --- a/kernel/arm64/dznrm2_thunderx2t99_fast.c +++ b/kernel/arm64/dznrm2_thunderx2t99_fast.c @@ -44,6 +44,12 @@ extern int blas_level1_thread_with_return_value(int mode, BLASLONG m, BLASLONG n #define TMPF "d16" #define SSQ "d0" +#if defined(__APPLE__) +#define L(x) "L" #x +#else +#define L(x) ".L" #x +#endif + #if !defined(COMPLEX) #define N_DIV_SHIFT "5" #define N_REM_MASK "31" @@ -143,62 +149,62 @@ static double nrm2_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x) " fmov d6, xzr \n" " fmov d7, xzr \n" " cmp "N", xzr \n" - " ble .Lnrm2_kernel_L999 \n" + " ble "L(nrm2_kernel_L999)" \n" " cmp "INC_X", xzr \n" - " ble .Lnrm2_kernel_L999 \n" + " ble "L(nrm2_kernel_L999)" \n" " cmp "INC_X", #1 \n" - " bne .Lnrm2_kernel_S_BEGIN \n" + " bne "L(nrm2_kernel_S_BEGIN)" \n" - ".Lnrm2_kernel_F_BEGIN: \n" + L(nrm2_kernel_F_BEGIN)": \n" " lsl "INC_X", "INC_X", #"INC_SHIFT" \n" " asr "J", "N", #"N_DIV_SHIFT" \n" " cmp "J", xzr \n" - " beq .Lnrm2_kernel_F1 \n" + " beq "L(nrm2_kernel_F1)" \n" /* https://github.com/llvm/llvm-project/issues/149547 */ #if !(defined(__clang__) && defined(OS_WINDOWS)) " .align 5 \n" #endif - ".Lnrm2_kernel_F: \n" + L(nrm2_kernel_F)": \n" " "KERNEL_F" \n" " subs "J", "J", #1 \n" - " bne .Lnrm2_kernel_F \n" + " bne "L(nrm2_kernel_F)" \n" " "KERNEL_F_FINALIZE" \n" - ".Lnrm2_kernel_F1: \n" + L(nrm2_kernel_F1)": \n" " ands "J", "N", #"N_REM_MASK" \n" - " ble .Lnrm2_kernel_L999 \n" + " ble "L(nrm2_kernel_L999)" \n" - ".Lnrm2_kernel_F10: \n" + L(nrm2_kernel_F10)": \n" " "KERNEL_F1" \n" " subs "J", "J", #1 \n" - " bne .Lnrm2_kernel_F10 \n" - " b .Lnrm2_kernel_L999 \n" + " bne "L(nrm2_kernel_F10)" \n" + " b "L(nrm2_kernel_L999)" \n" - ".Lnrm2_kernel_S_BEGIN: \n" + L(nrm2_kernel_S_BEGIN)": \n" " lsl "INC_X", "INC_X", #"INC_SHIFT" \n" " asr "J", "N", #2 \n" " cmp "J", xzr \n" - " ble .Lnrm2_kernel_S1 \n" + " ble "L(nrm2_kernel_S1)" \n" - ".Lnrm2_kernel_S4: \n" + L(nrm2_kernel_S4)": \n" " "KERNEL_F1" \n" " "KERNEL_F1" \n" " "KERNEL_F1" \n" " "KERNEL_F1" \n" " subs "J", "J", #1 \n" - " bne .Lnrm2_kernel_S4 \n" + " bne "L(nrm2_kernel_S4)" \n" - ".Lnrm2_kernel_S1: \n" + L(nrm2_kernel_S1)": \n" " ands "J", "N", #3 \n" - " ble .Lnrm2_kernel_L999 \n" + " ble "L(nrm2_kernel_L999)" \n" - ".Lnrm2_kernel_S10: \n" + L(nrm2_kernel_S10)": \n" " "KERNEL_F1" \n" " subs "J", "J", #1 \n" - " bne .Lnrm2_kernel_S10 \n" + " bne "L(nrm2_kernel_S10)" \n" - ".Lnrm2_kernel_L999: \n" + L(nrm2_kernel_L999)": \n" " "KERNEL_FINALIZE" \n" " str "SSQ", [%[RET_]] \n" diff --git a/kernel/arm64/nrm2.S b/kernel/arm64/nrm2.S index a5e9404fff..c73307df74 100644 --- a/kernel/arm64/nrm2.S +++ b/kernel/arm64/nrm2.S @@ -79,20 +79,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ldr d4, [X] #endif fcmp d4, REGZERO - beq KERNEL_S1_NEXT + beq L(KERNEL_S1_NEXT) fabs d4, d4 fcmp SCALE, d4 - bge KERNEL_S1_SCALE_GE_X + bge L(KERNEL_S1_SCALE_GE_X) fdiv d2, SCALE, d4 fmul d2, d2, d2 fmul d3, SSQ, d2 fadd SSQ, REGONE, d3 fmov SCALE, d4 - b KERNEL_S1_NEXT -KERNEL_S1_SCALE_GE_X: + b L(KERNEL_S1_NEXT) +L(KERNEL_S1_SCALE_GE_X): fdiv d2, d4, SCALE fmla SSQ, d2, v2.d[0] -KERNEL_S1_NEXT: +L(KERNEL_S1_NEXT): add X, X, INC_X .endm diff --git a/kernel/arm64/sgemm_direct_sme1_2VLx2VL.S b/kernel/arm64/sgemm_direct_sme1_2VLx2VL.S index afb662c1fb..c6533e657c 100644 --- a/kernel/arm64/sgemm_direct_sme1_2VLx2VL.S +++ b/kernel/arm64/sgemm_direct_sme1_2VLx2VL.S @@ -3,6 +3,9 @@ SPDX-License-Identifier: BSD-3-Clause-Clear */ +#define ASSEMBLER +#include "common.h" + /*-------------------------------------------------------------------------- * SME1 based Matrix multiplication code for FP32 input matrices to FP32 * output matrix @@ -64,14 +67,14 @@ whilelt p2.s, M_cntr, M //Tile 0,1 predicate (M dimension) sub w21, w21, #2 //SVLs-2 -.M_Loop: +L(M_Loop): incw M_cntr whilelt p3.s, M_cntr, M //Tile 2,3 predicate (M dimension) mov Bptr, B_base //B_base mov Cptr, C_base //C_base whilelt p0.b, Bptr, N_exit //Tile 0/2 predicate (N dimension) -.N_Loop: +L(N_Loop): mov Aptr, A_base //Aptr = A_base mov Bptr0, Bptr //Bptr = B_base mov Cptr0, Cptr //Cptr0 = C_base @@ -92,7 +95,7 @@ // Aptr += SVLb addvl Aptr, Aptr, #1 -.K_Loop: +L(K_Loop): // ZA2 += 2nd Aptr vector OP 1st Bptr vector fmopa za2.s, p3/m, p0/m, z5.s, z2.s // Load 2nd vector from Bptr @@ -104,7 +107,7 @@ // ZA3 += 2nd Aptr vector OP 2nd Bptr vector fmopa za3.s, p3/m, p1/m, z5.s, z3.s cmp K, #2 - b.le process_K_less_than_equal_2 + b.le L(process_K_less_than_equal_2) // Load next 1st vector from Bptr ld1w {z6.s}, p0/z, [Bptr0, N, lsl #2] // ZA0 += 1st Aptr vector OP 1st Bptr vector @@ -132,7 +135,7 @@ // Aptr += 2*SVLb [Bytes] addvl Aptr, Aptr, #2 cmp Aptr, K_exit - b.mi .K_Loop + b.mi L(K_Loop) // ZA2 += 2nd Aptr vector OP 1st Bptr vector fmopa za2.s, p3/m, p0/m, z5.s, z2.s // Load next 2nd vector from Bptr @@ -142,13 +145,13 @@ // ZA3 += 2nd Aptr vector OP 2nd Bptr vector fmopa za3.s, p3/m, p1/m, z5.s, z3.s -process_K_less_than_equal_2: +L(process_K_less_than_equal_2): // Bptr += 2*ldb FP32 elements add Bptr0, Bptr0, N, lsl #2 cmp Aptr, Aptr_end - b.pl .Ktail_end + b.pl L(Ktail_end) -.Ktail_start: +L(Ktail_start): ld1w {z1.s}, p2/z, [Aptr] ld1w {z2.s}, p0/z, [Bptr0] ld1w {z3.s}, p1/z, [Bptr0, #1, MUL VL] @@ -158,7 +161,7 @@ process_K_less_than_equal_2: fmopa za1.s, p2/m, p1/m, z1.s, z3.s fmopa za3.s, p3/m, p1/m, z5.s, z3.s -.Ktail_end: +L(Ktail_end): mov w13, #0 psel p4, p0, p2.s[w13, 0] psel p5, p1, p2.s[w13, 0] @@ -173,7 +176,7 @@ process_K_less_than_equal_2: // Store to Cptr1 + N*SVLs st1w {za3h.s[w13, #0]}, p7, [Cptr1, C6, lsl #2] -.Loop_store_ZA: +L(Loop_store_ZA): psel p4, p0, p2.s[w13, 1] psel p5, p1, p2.s[w13, 1] psel p6, p0, p3.s[w13, 1] @@ -200,7 +203,7 @@ process_K_less_than_equal_2: st1w {za2h.s[w13, #0]}, p6, [Cptr0, C6, lsl #2] st1w {za3h.s[w13, #0]}, p7, [Cptr1, C6, lsl #2] cmp w13, w21 - b.mi .Loop_store_ZA + b.mi L(Loop_store_ZA) psel p4, p0, p2.s[w13, 1] psel p5, p1, p2.s[w13, 1] psel p6, p0, p3.s[w13, 1] @@ -212,12 +215,12 @@ process_K_less_than_equal_2: addvl Cptr, Cptr, #2 addvl Bptr, Bptr, #1 whilelt p0.b, Bptr, N_exit //1st Tile predicate (N dimension) - b.mi .N_Loop + b.mi L(N_Loop) add A_base, A_base, C5, lsl #3 //A_base += 2*K*SVLs FP32 elements add C_base, C_base, C6, lsl #3 //C_base += 2*N*SVLs FP32 elements incw M_cntr whilelt p2.s, M_cntr, M //1st Tile predicate (M dimension) - b.mi .M_Loop + b.mi L(M_Loop) smstop @@ -227,3 +230,4 @@ process_K_less_than_equal_2: ret + EPILOGUE diff --git a/kernel/arm64/sgemm_direct_sme1_preprocess.S b/kernel/arm64/sgemm_direct_sme1_preprocess.S index 6c51b0bf63..b7536b5a46 100644 --- a/kernel/arm64/sgemm_direct_sme1_preprocess.S +++ b/kernel/arm64/sgemm_direct_sme1_preprocess.S @@ -3,6 +3,9 @@ SPDX-License-Identifier: BSD-3-Clause-Clear */ +#define ASSEMBLER +#include "common.h" + /*---------------------------------------------------------------------------- * This function is used to re-arrange the elements of input matrix to * make it suitable for matrix outer product computation using SME for matrix @@ -60,18 +63,18 @@ //Predicate for stores ptrue p9.s -.M_Loop: +L(M_Loop): mov mat_ptr0, mat //Load base address of mat mov mat_mod_ptr, mat_mod //a_mod store base address add inner_loop_exit, mat, ncol, lsl #2 //Exit condition for inner loop whilelt p8.b, mat_ptr0, inner_loop_exit //Tile predicate (K dimension) -.Loop_process: +L(Loop_process): mov mat_ptr1, mat_ptr0 //Load_to_tile loop counter mov w12, #0 -.Load_to_tile: +L(Load_to_tile): psel p2, p8, p0.s[w12, 0] psel p3, p8, p0.s[w12, 1] psel p4, p8, p0.s[w12, 2] @@ -89,11 +92,11 @@ //Increment counter add w12, w12, #4 cmp w12, w9 - b.mi .Load_to_tile + b.mi L(Load_to_tile) // Store_from_tile loop counter mov w12, #0 -.Store_from_tile: +L(Store_from_tile): psel p2, p9, p8.s[w12, 0] psel p3, p9, p8.s[w12, 1] psel p4, p9, p8.s[w12, 2] @@ -110,18 +113,18 @@ addvl mat_mod_ptr, mat_mod_ptr, #4 //mat_mod_ptr += 4*SVLb add w12, w12, #4 //Increment counter cmp w12, w9 - b.mi .Store_from_tile + b.mi L(Store_from_tile) addvl mat_ptr0, mat_ptr0, #1 //mat_ptr0 += SVLb whilelt p8.b, mat_ptr0, inner_loop_exit - b.mi .Loop_process + b.mi L(Loop_process) add mat_mod, mat_mod, C3, lsl #2 //mat_mod+=SVLs*nbc FP32 elements add mat, mat, C3, lsl #2 //mat+=SVLs*nbc FP32 elements incw outer_loop_cntr whilelt p0.s, outer_loop_cntr, nrow - b.mi .M_Loop + b.mi L(M_Loop) smstop @@ -131,3 +134,4 @@ ret + EPILOGUE diff --git a/kernel/arm64/znrm2.S b/kernel/arm64/znrm2.S index 99289a6e81..03a72ae70d 100644 --- a/kernel/arm64/znrm2.S +++ b/kernel/arm64/znrm2.S @@ -123,69 +123,69 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if !defined(DOUBLE) ldr s4, [X] fcmp s4, REGZERO - beq KERNEL_S1_NEXT + beq L(KERNEL_S1_NEXT) fabs s4, s4 fcmp SCALE, s4 - bge KERNEL_S1_SCALE_GE_XR + bge L(KERNEL_S1_SCALE_GE_XR) fdiv s2, SCALE, s4 fmul s2, s2, s2 fmul s3, SSQ, s2 fadd SSQ, REGONE, s3 fmov SCALE, s4 - b KERNEL_S1_NEXT -KERNEL_S1_SCALE_GE_XR: + b L(KERNEL_S1_NEXT) +L(KERNEL_S1_SCALE_GE_XR): fdiv s2, s4, SCALE fmla SSQ, s2, v2.s[0] -KERNEL_S1_NEXT: +L(KERNEL_S1_NEXT): ldr s5, [X, #4] fcmp s5, REGZERO - beq KERNEL_S1_END + beq L(KERNEL_S1_END) fabs s5, s5 fcmp SCALE, s5 - bge KERNEL_S1_SCALE_GE_XI + bge L(KERNEL_S1_SCALE_GE_XI) fdiv s2, SCALE, s5 fmul s2, s2, s2 fmul s3, SSQ, s2 fadd SSQ, REGONE, s3 fmov SCALE, s5 - b KERNEL_S1_END -KERNEL_S1_SCALE_GE_XI: + b L(KERNEL_S1_END) +L(KERNEL_S1_SCALE_GE_XI): fdiv s2, s5, SCALE fmla SSQ, s2, v2.s[0] #else ldr d4, [X] fcmp d4, REGZERO - beq KERNEL_S1_NEXT + beq L(KERNEL_S1_NEXT) fabs d4, d4 fcmp SCALE, d4 - bge KERNEL_S1_SCALE_GE_XR + bge L(KERNEL_S1_SCALE_GE_XR) fdiv d2, SCALE, d4 fmul d2, d2, d2 fmul d3, SSQ, d2 fadd SSQ, REGONE, d3 fmov SCALE, d4 - b KERNEL_S1_NEXT -KERNEL_S1_SCALE_GE_XR: + b L(KERNEL_S1_NEXT) +L(KERNEL_S1_SCALE_GE_XR): fdiv d2, d4, SCALE fmla SSQ, d2, v2.d[0] -KERNEL_S1_NEXT: +L(KERNEL_S1_NEXT): ldr d5, [X, #8] fcmp d5, REGZERO - beq KERNEL_S1_END + beq L(KERNEL_S1_END) fabs d5, d5 fcmp SCALE, d5 - bge KERNEL_S1_SCALE_GE_XI + bge L(KERNEL_S1_SCALE_GE_XI) fdiv d2, SCALE, d5 fmul d2, d2, d2 fmul d3, SSQ, d2 fadd SSQ, REGONE, d3 fmov SCALE, d5 - b KERNEL_S1_END -KERNEL_S1_SCALE_GE_XI: + b L(KERNEL_S1_END) +L(KERNEL_S1_SCALE_GE_XI): fdiv d2, d5, SCALE fmla SSQ, d2, v2.d[0] #endif -KERNEL_S1_END: +L(KERNEL_S1_END): add X, X, INC_X .endm diff --git a/kernel/arm64/zscal.S b/kernel/arm64/zscal.S index 15d9e3bb2e..63d71deeaa 100644 --- a/kernel/arm64/zscal.S +++ b/kernel/arm64/zscal.S @@ -188,22 +188,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE - b zscal_begin -data_ar: + b L(zscal_begin) +L(data_ar): .word 0x3e44fae6 -data_ai: +L(data_ai): .word 0x3d320fa2 -data_xr: +L(data_xr): .word 0x3f4baff1 -data_xi: +L(data_xi): .word 0xbe8ef0bd -zscal_begin: +L(zscal_begin): - ldr s20, data_ar - ldr s21, data_ai - ldr s22, data_xr - ldr s23, data_xi + ldr s20, L(data_ar) + ldr s21, L(data_ai) + ldr s22, L(data_xr) + ldr s23, L(data_xi) fmul s24, s22, s21 fmla s24, s23, v20.s[0] diff --git a/kernel/x86_64/dgemm_kernel_4x8_skylakex.c b/kernel/x86_64/dgemm_kernel_4x8_skylakex.c index 6257e569ee..27848d0964 100644 --- a/kernel/x86_64/dgemm_kernel_4x8_skylakex.c +++ b/kernel/x86_64/dgemm_kernel_4x8_skylakex.c @@ -35,6 +35,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "common.h" #include +#if defined(__APPLE__) +#define L(x) "L" #x +#else +#define L(x) ".L" #x +#endif + /******************************************************************************************* * Macro definitions @@ -868,10 +874,10 @@ CNAME(BLASLONG m, BLASLONG n, BLASLONG k, double alpha, double * __restrict__ A, "vmovapd %%zmm1, %%zmm26\n" "vmovapd %%zmm1, %%zmm27\n" "vmovapd %%zmm1, %%zmm28\n" - "jmp .label24\n" + "jmp "L(label24)"\n" ".p2align 5\n" /* Inner math loop */ - ".label24:\n" + L(label24)":\n" "vmovupd -128(%[AO]),%%zmm0\n" "vmovupd -128(%[A1]),%%zmm10\n" "vmovupd -128(%[A2]),%%zmm20\n" @@ -924,7 +930,7 @@ CNAME(BLASLONG m, BLASLONG n, BLASLONG k, double alpha, double * __restrict__ A, "prefetch 512(%[A2])\n" "prefetch 512(%[BO])\n" "subl $1, %[kloop]\n" - "jg .label24\n" + "jg "L(label24)"\n" /* multiply the result by alpha */ "vbroadcastsd (%[alpha]), %%zmm9\n" /* And store additively in C */ @@ -1036,10 +1042,10 @@ CNAME(BLASLONG m, BLASLONG n, BLASLONG k, double alpha, double * __restrict__ A, "vmovapd %%zmm1, %%zmm16\n" "vmovapd %%zmm1, %%zmm17\n" "vmovapd %%zmm1, %%zmm18\n" - "jmp .label16\n" + "jmp "L(label16)"\n" ".p2align 5\n" /* Inner math loop */ - ".label16:\n" + L(label16)":\n" "vmovupd -128(%[AO]),%%zmm0\n" "vmovupd -128(%[A1]),%%zmm10\n" @@ -1081,7 +1087,7 @@ CNAME(BLASLONG m, BLASLONG n, BLASLONG k, double alpha, double * __restrict__ A, "prefetch 512(%[A1])\n" "prefetch 512(%[BO])\n" "subl $1, %[kloop]\n" - "jg .label16\n" + "jg "L(label16)"\n" /* multiply the result by alpha */ "vbroadcastsd (%[alpha]), %%zmm9\n" /* And store additively in C */ @@ -1164,10 +1170,10 @@ CNAME(BLASLONG m, BLASLONG n, BLASLONG k, double alpha, double * __restrict__ A, "vmovapd %%zmm1, %%zmm7\n" "vmovapd %%zmm1, %%zmm8\n" "vbroadcastsd (%[alpha]), %%zmm9\n" - "jmp .label1\n" + "jmp "L(label1)"\n" ".p2align 5\n" /* Inner math loop */ - ".label1:\n" + L(label1)":\n" "vmovupd -128(%[AO]),%%zmm0\n" "vfmadd231pd -96(%[BO])%{1to8%}, %%zmm0, %%zmm1\n" "vfmadd231pd -88(%[BO])%{1to8%}, %%zmm0, %%zmm2\n" @@ -1180,7 +1186,7 @@ CNAME(BLASLONG m, BLASLONG n, BLASLONG k, double alpha, double * __restrict__ A, "add $64, %[AO]\n" "add $64, %[BO]\n" "subl $1, %[kloop]\n" - "jg .label1\n" + "jg "L(label1)"\n" /* multiply the result by alpha and add to the memory */ "vfmadd213pd (%[C0]), %%zmm9, %%zmm1\n" "vfmadd213pd (%[C1]), %%zmm9, %%zmm2\n" diff --git a/kernel/x86_64/dgemm_kernel_6x4_piledriver.S b/kernel/x86_64/dgemm_kernel_6x4_piledriver.S index 66779648ba..90711d90a8 100644 --- a/kernel/x86_64/dgemm_kernel_6x4_piledriver.S +++ b/kernel/x86_64/dgemm_kernel_6x4_piledriver.S @@ -223,9 +223,9 @@ leaq (, LDC, SIZE), LDC MOVQ1280(_bk_j,j); SARQ1280($2,j); -JLE ._L_0_loopE; +JLE L(_L_0_loopE); ALIGN_4; -._L_0_bodyB:; +L(_L_0_bodyB):; MOVQ1280(_ptr_A,_ptr__A_0); MOVQ1280(_ptr_C,_ptr__C_0); LEAQ1280((_ptr_C,LDC,2),_ptr__C_1); @@ -234,8 +234,8 @@ SALQ1280($5,%rax); ADDQ1280(%rax,_pre_B); MOVQ1280(_bk_i,i); CMPQ1280($6,i); -JL ._L_1_loopE; -._L_1_bodyB:; +JL L(_L_1_loopE); +L(_L_1_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1282(XMM0,XMM0,XMM0); VXOR1282(XMM1,XMM1,XMM1); @@ -255,9 +255,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_2_loopE; +JLE L(_L_2_loopE); ALIGN_4; -._L_2_bodyB:; +L(_L_2_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM12); @@ -340,15 +340,15 @@ VMA1282(XMM13,XMM15,XMM10,XMM10); VMA1282(XMM14,XMM15,XMM11,XMM11); ADDQ1280($24*SIZE,_ptr__A_0); ADDQ1280($16*SIZE,_ptr__B_0); -._L_2_bodyE:; +L(_L_2_bodyE):; DECQ1280(k); -JG ._L_2_bodyB; +JG L(_L_2_bodyB); ALIGN_4; -._L_2_loopE:; +L(_L_2_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_3_loopE; +JLE L(_L_3_loopE); ALIGN_4; -._L_3_bodyB:; +L(_L_3_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM12); @@ -391,11 +391,11 @@ VMA1282(XMM13,XMM15,XMM10,XMM10); VMA1282(XMM14,XMM15,XMM11,XMM11); ADDQ1280($12*SIZE,_ptr__A_0); ADDQ1280($8*SIZE,_ptr__B_0); -._L_3_loopE:; +L(_L_3_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_4_loopE; +JLE L(_L_4_loopE); ALIGN_4; -._L_4_bodyB:; +L(_L_4_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM12); @@ -418,7 +418,7 @@ VMA1282(XMM13,XMM15,XMM10,XMM10); VMA1282(XMM14,XMM15,XMM11,XMM11); ADDQ1280($6*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_4_loopE:; +L(_L_4_loopE):; BROAD1282(alpha,XMM12); VLDU1282(0*SIZE(_ptr__C_0),XMM13); VMA21282(XMM12,XMM0,XMM13,XMM0); @@ -458,15 +458,15 @@ VMA21282(XMM12,XMM11,XMM15,XMM11); VSTU1282(XMM15,4*SIZE(_ptr__C_1,LDC,1)); ADDQ1280($6*SIZE,_ptr__C_0); ADDQ1280($6*SIZE,_ptr__C_1); -._L_1_bodyE:; +L(_L_1_bodyE):; SUBQ1280($6,i); -JG ._L_1_bodyB; +JG L(_L_1_bodyB); ALIGN_4; -._L_1_loopE:; +L(_L_1_loopE):; TESTQ1280($4,i); -JLE ._L_5_loopE; +JLE L(_L_5_loopE); ALIGN_4; -._L_5_bodyB:; +L(_L_5_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1282(XMM0,XMM0,XMM0); VXOR1282(XMM1,XMM1,XMM1); @@ -482,9 +482,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_6_loopE; +JLE L(_L_6_loopE); ALIGN_4; -._L_6_bodyB:; +L(_L_6_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM13); @@ -547,15 +547,15 @@ VMA1282(XMM13,XMM15,XMM6,XMM6); VMA1282(XMM14,XMM15,XMM7,XMM7); ADDQ1280($16*SIZE,_ptr__A_0); ADDQ1280($16*SIZE,_ptr__B_0); -._L_6_bodyE:; +L(_L_6_bodyE):; DECQ1280(k); -JG ._L_6_bodyB; +JG L(_L_6_bodyB); ALIGN_4; -._L_6_loopE:; +L(_L_6_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_7_loopE; +JLE L(_L_7_loopE); ALIGN_4; -._L_7_bodyB:; +L(_L_7_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM13); @@ -588,11 +588,11 @@ VMA1282(XMM13,XMM15,XMM6,XMM6); VMA1282(XMM14,XMM15,XMM7,XMM7); ADDQ1280($8*SIZE,_ptr__A_0); ADDQ1280($8*SIZE,_ptr__B_0); -._L_7_loopE:; +L(_L_7_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_8_loopE; +JLE L(_L_8_loopE); ALIGN_4; -._L_8_bodyB:; +L(_L_8_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM13); @@ -610,7 +610,7 @@ VMA1282(XMM13,XMM15,XMM6,XMM6); VMA1282(XMM14,XMM15,XMM7,XMM7); ADDQ1280($4*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_8_loopE:; +L(_L_8_loopE):; BROAD1282(alpha,XMM8); VLDU1282(0*SIZE(_ptr__C_0),XMM9); VMA21282(XMM8,XMM0,XMM9,XMM0); @@ -638,11 +638,11 @@ VMA21282(XMM8,XMM7,XMM9,XMM7); VSTU1282(XMM9,2*SIZE(_ptr__C_1,LDC,1)); ADDQ1280($4*SIZE,_ptr__C_0); ADDQ1280($4*SIZE,_ptr__C_1); -._L_5_loopE:; +L(_L_5_loopE):; TESTQ1280($2,i); -JLE ._L_9_loopE; +JLE L(_L_9_loopE); ALIGN_4; -._L_9_bodyB:; +L(_L_9_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1282(XMM0,XMM0,XMM0); VXOR1282(XMM1,XMM1,XMM1); @@ -654,9 +654,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_10_loopE; +JLE L(_L_10_loopE); ALIGN_4; -._L_10_bodyB:; +L(_L_10_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM14); @@ -699,15 +699,15 @@ BROAD1282(15*SIZE(_ptr__B_0),XMM15); VMA1282(XMM14,XMM15,XMM3,XMM3); ADDQ1280($8*SIZE,_ptr__A_0); ADDQ1280($16*SIZE,_ptr__B_0); -._L_10_bodyE:; +L(_L_10_bodyE):; DECQ1280(k); -JG ._L_10_bodyB; +JG L(_L_10_bodyB); ALIGN_4; -._L_10_loopE:; +L(_L_10_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_11_loopE; +JLE L(_L_11_loopE); ALIGN_4; -._L_11_bodyB:; +L(_L_11_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM14); @@ -730,11 +730,11 @@ BROAD1282(7*SIZE(_ptr__B_0),XMM15); VMA1282(XMM14,XMM15,XMM3,XMM3); ADDQ1280($4*SIZE,_ptr__A_0); ADDQ1280($8*SIZE,_ptr__B_0); -._L_11_loopE:; +L(_L_11_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_12_loopE; +JLE L(_L_12_loopE); ALIGN_4; -._L_12_bodyB:; +L(_L_12_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM14); @@ -747,7 +747,7 @@ BROAD1282(3*SIZE(_ptr__B_0),XMM15); VMA1282(XMM14,XMM15,XMM3,XMM3); ADDQ1280($2*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_12_loopE:; +L(_L_12_loopE):; BROAD1282(alpha,XMM4); VLDU1282(0*SIZE(_ptr__C_0),XMM5); VMA21282(XMM4,XMM0,XMM5,XMM0); @@ -763,11 +763,11 @@ VMA21282(XMM4,XMM3,XMM8,XMM3); VSTU1282(XMM8,0*SIZE(_ptr__C_1,LDC,1)); ADDQ1280($2*SIZE,_ptr__C_0); ADDQ1280($2*SIZE,_ptr__C_1); -._L_9_loopE:; +L(_L_9_loopE):; TESTQ1280($1,i); -JLE ._L_13_loopE; +JLE L(_L_13_loopE); ALIGN_4; -._L_13_bodyB:; +L(_L_13_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1281(XMM0,XMM0,XMM0); VXOR1281(XMM1,XMM1,XMM1); @@ -779,9 +779,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_14_loopE; +JLE L(_L_14_loopE); ALIGN_4; -._L_14_bodyB:; +L(_L_14_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1281(0*SIZE(_ptr__B_0),XMM15); VLD1281(0*SIZE(_ptr__A_0),XMM14); @@ -824,15 +824,15 @@ BROAD1281(15*SIZE(_ptr__B_0),XMM15); VMA1281(XMM14,XMM15,XMM3,XMM3); ADDQ1280($4*SIZE,_ptr__A_0); ADDQ1280($16*SIZE,_ptr__B_0); -._L_14_bodyE:; +L(_L_14_bodyE):; DECQ1280(k); -JG ._L_14_bodyB; +JG L(_L_14_bodyB); ALIGN_4; -._L_14_loopE:; +L(_L_14_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_15_loopE; +JLE L(_L_15_loopE); ALIGN_4; -._L_15_bodyB:; +L(_L_15_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1281(0*SIZE(_ptr__B_0),XMM15); VLD1281(0*SIZE(_ptr__A_0),XMM14); @@ -855,11 +855,11 @@ BROAD1281(7*SIZE(_ptr__B_0),XMM15); VMA1281(XMM14,XMM15,XMM3,XMM3); ADDQ1280($2*SIZE,_ptr__A_0); ADDQ1280($8*SIZE,_ptr__B_0); -._L_15_loopE:; +L(_L_15_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_16_loopE; +JLE L(_L_16_loopE); ALIGN_4; -._L_16_bodyB:; +L(_L_16_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1281(0*SIZE(_ptr__B_0),XMM15); VLD1281(0*SIZE(_ptr__A_0),XMM14); @@ -872,7 +872,7 @@ BROAD1281(3*SIZE(_ptr__B_0),XMM15); VMA1281(XMM14,XMM15,XMM3,XMM3); ADDQ1280($1*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_16_loopE:; +L(_L_16_loopE):; BROAD1281(alpha,XMM4); VLDU1281(0*SIZE(_ptr__C_0),XMM5); VMA21281(XMM4,XMM0,XMM5,XMM0); @@ -888,22 +888,22 @@ VMA21281(XMM4,XMM3,XMM8,XMM3); VSTU1281(XMM8,0*SIZE(_ptr__C_1,LDC,1)); ADDQ1280($1*SIZE,_ptr__C_0); ADDQ1280($1*SIZE,_ptr__C_1); -._L_13_loopE:; +L(_L_13_loopE):; MOVQ1280(LDC,%rax); SALQ1280($2,%rax); ADDQ1280(%rax,_ptr_C); MOVQ1280(_bk_l,%rax); SALQ1280($5,%rax); ADDQ1280(%rax,_ptr_B); -._L_0_bodyE:; +L(_L_0_bodyE):; DECQ1280(j); -JG ._L_0_bodyB; +JG L(_L_0_bodyB); ALIGN_4; -._L_0_loopE:; +L(_L_0_loopE):; TESTQ1280($2,_bk_j); -JLE ._L_17_loopE; +JLE L(_L_17_loopE); ALIGN_4; -._L_17_bodyB:; +L(_L_17_bodyB):; MOVQ1280(_ptr_A,_ptr__A_0); MOVQ1280(_ptr_C,_ptr__C_0); LEAQ1280((_ptr_C,LDC,1),_ptr__C_1); @@ -912,8 +912,8 @@ SALQ1280($4,%rax); ADDQ1280(%rax,_pre_B); MOVQ1280(_bk_i,i); CMPQ1280($6,i); -JL ._L_18_loopE; -._L_18_bodyB:; +JL L(_L_18_loopE); +L(_L_18_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1282(XMM0,XMM0,XMM0); VXOR1282(XMM1,XMM1,XMM1); @@ -927,9 +927,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_19_loopE; +JLE L(_L_19_loopE); ALIGN_4; -._L_19_bodyB:; +L(_L_19_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM12); @@ -980,15 +980,15 @@ VMA1282(XMM13,XMM15,XMM4,XMM4); VMA1282(XMM14,XMM15,XMM5,XMM5); ADDQ1280($24*SIZE,_ptr__A_0); ADDQ1280($8*SIZE,_ptr__B_0); -._L_19_bodyE:; +L(_L_19_bodyE):; DECQ1280(k); -JG ._L_19_bodyB; +JG L(_L_19_bodyB); ALIGN_4; -._L_19_loopE:; +L(_L_19_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_20_loopE; +JLE L(_L_20_loopE); ALIGN_4; -._L_20_bodyB:; +L(_L_20_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM12); @@ -1015,11 +1015,11 @@ VMA1282(XMM13,XMM15,XMM4,XMM4); VMA1282(XMM14,XMM15,XMM5,XMM5); ADDQ1280($12*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_20_loopE:; +L(_L_20_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_21_loopE; +JLE L(_L_21_loopE); ALIGN_4; -._L_21_bodyB:; +L(_L_21_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM12); @@ -1034,7 +1034,7 @@ VMA1282(XMM13,XMM15,XMM4,XMM4); VMA1282(XMM14,XMM15,XMM5,XMM5); ADDQ1280($6*SIZE,_ptr__A_0); ADDQ1280($2*SIZE,_ptr__B_0); -._L_21_loopE:; +L(_L_21_loopE):; BROAD1282(alpha,XMM6); VLDU1282(0*SIZE(_ptr__C_0),XMM7); VMA21282(XMM6,XMM0,XMM7,XMM0); @@ -1056,15 +1056,15 @@ VMA21282(XMM6,XMM5,XMM12,XMM5); VSTU1282(XMM12,4*SIZE(_ptr__C_0,LDC,1)); ADDQ1280($6*SIZE,_ptr__C_0); ADDQ1280($6*SIZE,_ptr__C_1); -._L_18_bodyE:; +L(_L_18_bodyE):; SUBQ1280($6,i); -JG ._L_18_bodyB; +JG L(_L_18_bodyB); ALIGN_4; -._L_18_loopE:; +L(_L_18_loopE):; TESTQ1280($4,i); -JLE ._L_22_loopE; +JLE L(_L_22_loopE); ALIGN_4; -._L_22_bodyB:; +L(_L_22_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1282(XMM0,XMM0,XMM0); VXOR1282(XMM1,XMM1,XMM1); @@ -1076,9 +1076,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_23_loopE; +JLE L(_L_23_loopE); ALIGN_4; -._L_23_bodyB:; +L(_L_23_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM13); @@ -1117,15 +1117,15 @@ VMA1282(XMM13,XMM15,XMM2,XMM2); VMA1282(XMM14,XMM15,XMM3,XMM3); ADDQ1280($16*SIZE,_ptr__A_0); ADDQ1280($8*SIZE,_ptr__B_0); -._L_23_bodyE:; +L(_L_23_bodyE):; DECQ1280(k); -JG ._L_23_bodyB; +JG L(_L_23_bodyB); ALIGN_4; -._L_23_loopE:; +L(_L_23_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_24_loopE; +JLE L(_L_24_loopE); ALIGN_4; -._L_24_bodyB:; +L(_L_24_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM13); @@ -1146,11 +1146,11 @@ VMA1282(XMM13,XMM15,XMM2,XMM2); VMA1282(XMM14,XMM15,XMM3,XMM3); ADDQ1280($8*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_24_loopE:; +L(_L_24_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_25_loopE; +JLE L(_L_25_loopE); ALIGN_4; -._L_25_bodyB:; +L(_L_25_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM13); @@ -1162,7 +1162,7 @@ VMA1282(XMM13,XMM15,XMM2,XMM2); VMA1282(XMM14,XMM15,XMM3,XMM3); ADDQ1280($4*SIZE,_ptr__A_0); ADDQ1280($2*SIZE,_ptr__B_0); -._L_25_loopE:; +L(_L_25_loopE):; BROAD1282(alpha,XMM4); VLDU1282(0*SIZE(_ptr__C_0),XMM5); VMA21282(XMM4,XMM0,XMM5,XMM0); @@ -1178,11 +1178,11 @@ VMA21282(XMM4,XMM3,XMM8,XMM3); VSTU1282(XMM8,2*SIZE(_ptr__C_0,LDC,1)); ADDQ1280($4*SIZE,_ptr__C_0); ADDQ1280($4*SIZE,_ptr__C_1); -._L_22_loopE:; +L(_L_22_loopE):; TESTQ1280($2,i); -JLE ._L_26_loopE; +JLE L(_L_26_loopE); ALIGN_4; -._L_26_bodyB:; +L(_L_26_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1282(XMM0,XMM0,XMM0); VXOR1282(XMM1,XMM1,XMM1); @@ -1192,9 +1192,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_27_loopE; +JLE L(_L_27_loopE); ALIGN_4; -._L_27_bodyB:; +L(_L_27_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM14); @@ -1221,15 +1221,15 @@ BROAD1282(7*SIZE(_ptr__B_0),XMM15); VMA1282(XMM14,XMM15,XMM1,XMM1); ADDQ1280($8*SIZE,_ptr__A_0); ADDQ1280($8*SIZE,_ptr__B_0); -._L_27_bodyE:; +L(_L_27_bodyE):; DECQ1280(k); -JG ._L_27_bodyB; +JG L(_L_27_bodyB); ALIGN_4; -._L_27_loopE:; +L(_L_27_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_28_loopE; +JLE L(_L_28_loopE); ALIGN_4; -._L_28_bodyB:; +L(_L_28_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM14); @@ -1244,11 +1244,11 @@ BROAD1282(3*SIZE(_ptr__B_0),XMM15); VMA1282(XMM14,XMM15,XMM1,XMM1); ADDQ1280($4*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_28_loopE:; +L(_L_28_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_29_loopE; +JLE L(_L_29_loopE); ALIGN_4; -._L_29_bodyB:; +L(_L_29_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM14); @@ -1257,7 +1257,7 @@ BROAD1282(1*SIZE(_ptr__B_0),XMM15); VMA1282(XMM14,XMM15,XMM1,XMM1); ADDQ1280($2*SIZE,_ptr__A_0); ADDQ1280($2*SIZE,_ptr__B_0); -._L_29_loopE:; +L(_L_29_loopE):; BROAD1282(alpha,XMM2); VLDU1282(0*SIZE(_ptr__C_0),XMM3); VMA21282(XMM2,XMM0,XMM3,XMM0); @@ -1267,11 +1267,11 @@ VMA21282(XMM2,XMM1,XMM4,XMM1); VSTU1282(XMM4,0*SIZE(_ptr__C_0,LDC,1)); ADDQ1280($2*SIZE,_ptr__C_0); ADDQ1280($2*SIZE,_ptr__C_1); -._L_26_loopE:; +L(_L_26_loopE):; TESTQ1280($1,i); -JLE ._L_30_loopE; +JLE L(_L_30_loopE); ALIGN_4; -._L_30_bodyB:; +L(_L_30_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1281(XMM0,XMM0,XMM0); VXOR1281(XMM1,XMM1,XMM1); @@ -1281,9 +1281,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_31_loopE; +JLE L(_L_31_loopE); ALIGN_4; -._L_31_bodyB:; +L(_L_31_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1281(0*SIZE(_ptr__B_0),XMM15); VLD1281(0*SIZE(_ptr__A_0),XMM14); @@ -1310,15 +1310,15 @@ BROAD1281(7*SIZE(_ptr__B_0),XMM15); VMA1281(XMM14,XMM15,XMM1,XMM1); ADDQ1280($4*SIZE,_ptr__A_0); ADDQ1280($8*SIZE,_ptr__B_0); -._L_31_bodyE:; +L(_L_31_bodyE):; DECQ1280(k); -JG ._L_31_bodyB; +JG L(_L_31_bodyB); ALIGN_4; -._L_31_loopE:; +L(_L_31_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_32_loopE; +JLE L(_L_32_loopE); ALIGN_4; -._L_32_bodyB:; +L(_L_32_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1281(0*SIZE(_ptr__B_0),XMM15); VLD1281(0*SIZE(_ptr__A_0),XMM14); @@ -1333,11 +1333,11 @@ BROAD1281(3*SIZE(_ptr__B_0),XMM15); VMA1281(XMM14,XMM15,XMM1,XMM1); ADDQ1280($2*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_32_loopE:; +L(_L_32_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_33_loopE; +JLE L(_L_33_loopE); ALIGN_4; -._L_33_bodyB:; +L(_L_33_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1281(0*SIZE(_ptr__B_0),XMM15); VLD1281(0*SIZE(_ptr__A_0),XMM14); @@ -1346,7 +1346,7 @@ BROAD1281(1*SIZE(_ptr__B_0),XMM15); VMA1281(XMM14,XMM15,XMM1,XMM1); ADDQ1280($1*SIZE,_ptr__A_0); ADDQ1280($2*SIZE,_ptr__B_0); -._L_33_loopE:; +L(_L_33_loopE):; BROAD1281(alpha,XMM2); VLDU1281(0*SIZE(_ptr__C_0),XMM3); VMA21281(XMM2,XMM0,XMM3,XMM0); @@ -1356,18 +1356,18 @@ VMA21281(XMM2,XMM1,XMM4,XMM1); VSTU1281(XMM4,0*SIZE(_ptr__C_0,LDC,1)); ADDQ1280($1*SIZE,_ptr__C_0); ADDQ1280($1*SIZE,_ptr__C_1); -._L_30_loopE:; +L(_L_30_loopE):; MOVQ1280(LDC,%rax); SALQ1280($1,%rax); ADDQ1280(%rax,_ptr_C); MOVQ1280(_bk_l,%rax); SALQ1280($4,%rax); ADDQ1280(%rax,_ptr_B); -._L_17_loopE:; +L(_L_17_loopE):; TESTQ1280($1,_bk_j); -JLE ._L_34_loopE; +JLE L(_L_34_loopE); ALIGN_4; -._L_34_bodyB:; +L(_L_34_bodyB):; MOVQ1280(_ptr_A,_ptr__A_0); MOVQ1280(_ptr_C,_ptr__C_0); MOVQ1280(_bk_l,%rax); @@ -1375,8 +1375,8 @@ SALQ1280($3,%rax); ADDQ1280(%rax,_pre_B); MOVQ1280(_bk_i,i); CMPQ1280($6,i); -JL ._L_35_loopE; -._L_35_bodyB:; +JL L(_L_35_loopE); +L(_L_35_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1282(XMM0,XMM0,XMM0); VXOR1282(XMM1,XMM1,XMM1); @@ -1387,9 +1387,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_36_loopE; +JLE L(_L_36_loopE); ALIGN_4; -._L_36_bodyB:; +L(_L_36_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM12); @@ -1424,15 +1424,15 @@ VLD1282(22*SIZE(_ptr__A_0),XMM14); VMA1282(XMM14,XMM15,XMM2,XMM2); ADDQ1280($24*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_36_bodyE:; +L(_L_36_bodyE):; DECQ1280(k); -JG ._L_36_bodyB; +JG L(_L_36_bodyB); ALIGN_4; -._L_36_loopE:; +L(_L_36_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_37_loopE; +JLE L(_L_37_loopE); ALIGN_4; -._L_37_bodyB:; +L(_L_37_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM12); @@ -1451,11 +1451,11 @@ VLD1282(10*SIZE(_ptr__A_0),XMM14); VMA1282(XMM14,XMM15,XMM2,XMM2); ADDQ1280($12*SIZE,_ptr__A_0); ADDQ1280($2*SIZE,_ptr__B_0); -._L_37_loopE:; +L(_L_37_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_38_loopE; +JLE L(_L_38_loopE); ALIGN_4; -._L_38_bodyB:; +L(_L_38_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM12); @@ -1466,7 +1466,7 @@ VLD1282(4*SIZE(_ptr__A_0),XMM14); VMA1282(XMM14,XMM15,XMM2,XMM2); ADDQ1280($6*SIZE,_ptr__A_0); ADDQ1280($1*SIZE,_ptr__B_0); -._L_38_loopE:; +L(_L_38_loopE):; BROAD1282(alpha,XMM3); VLDU1282(0*SIZE(_ptr__C_0),XMM4); VMA21282(XMM3,XMM0,XMM4,XMM0); @@ -1479,15 +1479,15 @@ VMA21282(XMM3,XMM2,XMM6,XMM2); VSTU1282(XMM6,4*SIZE(_ptr__C_0)); ADDQ1280($6*SIZE,_ptr__C_0); ADDQ1280($6*SIZE,_ptr__C_1); -._L_35_bodyE:; +L(_L_35_bodyE):; SUBQ1280($6,i); -JG ._L_35_bodyB; +JG L(_L_35_bodyB); ALIGN_4; -._L_35_loopE:; +L(_L_35_loopE):; TESTQ1280($4,i); -JLE ._L_39_loopE; +JLE L(_L_39_loopE); ALIGN_4; -._L_39_bodyB:; +L(_L_39_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1282(XMM0,XMM0,XMM0); VXOR1282(XMM1,XMM1,XMM1); @@ -1497,9 +1497,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_40_loopE; +JLE L(_L_40_loopE); ALIGN_4; -._L_40_bodyB:; +L(_L_40_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM13); @@ -1526,15 +1526,15 @@ VLD1282(14*SIZE(_ptr__A_0),XMM14); VMA1282(XMM14,XMM15,XMM1,XMM1); ADDQ1280($16*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_40_bodyE:; +L(_L_40_bodyE):; DECQ1280(k); -JG ._L_40_bodyB; +JG L(_L_40_bodyB); ALIGN_4; -._L_40_loopE:; +L(_L_40_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_41_loopE; +JLE L(_L_41_loopE); ALIGN_4; -._L_41_bodyB:; +L(_L_41_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM13); @@ -1549,11 +1549,11 @@ VLD1282(6*SIZE(_ptr__A_0),XMM14); VMA1282(XMM14,XMM15,XMM1,XMM1); ADDQ1280($8*SIZE,_ptr__A_0); ADDQ1280($2*SIZE,_ptr__B_0); -._L_41_loopE:; +L(_L_41_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_42_loopE; +JLE L(_L_42_loopE); ALIGN_4; -._L_42_bodyB:; +L(_L_42_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM13); @@ -1562,7 +1562,7 @@ VLD1282(2*SIZE(_ptr__A_0),XMM14); VMA1282(XMM14,XMM15,XMM1,XMM1); ADDQ1280($4*SIZE,_ptr__A_0); ADDQ1280($1*SIZE,_ptr__B_0); -._L_42_loopE:; +L(_L_42_loopE):; BROAD1282(alpha,XMM2); VLDU1282(0*SIZE(_ptr__C_0),XMM3); VMA21282(XMM2,XMM0,XMM3,XMM0); @@ -1572,11 +1572,11 @@ VMA21282(XMM2,XMM1,XMM4,XMM1); VSTU1282(XMM4,2*SIZE(_ptr__C_0)); ADDQ1280($4*SIZE,_ptr__C_0); ADDQ1280($4*SIZE,_ptr__C_1); -._L_39_loopE:; +L(_L_39_loopE):; TESTQ1280($2,i); -JLE ._L_43_loopE; +JLE L(_L_43_loopE); ALIGN_4; -._L_43_bodyB:; +L(_L_43_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1282(XMM0,XMM0,XMM0); PREFETCHN1280(3*SIZE(_ptr__C_0),N); @@ -1585,9 +1585,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_44_loopE; +JLE L(_L_44_loopE); ALIGN_4; -._L_44_bodyB:; +L(_L_44_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM14); @@ -1606,15 +1606,15 @@ VLD1282(6*SIZE(_ptr__A_0),XMM14); VMA1282(XMM14,XMM15,XMM0,XMM0); ADDQ1280($8*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_44_bodyE:; +L(_L_44_bodyE):; DECQ1280(k); -JG ._L_44_bodyB; +JG L(_L_44_bodyB); ALIGN_4; -._L_44_loopE:; +L(_L_44_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_45_loopE; +JLE L(_L_45_loopE); ALIGN_4; -._L_45_bodyB:; +L(_L_45_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM14); @@ -1625,29 +1625,29 @@ VLD1282(2*SIZE(_ptr__A_0),XMM14); VMA1282(XMM14,XMM15,XMM0,XMM0); ADDQ1280($4*SIZE,_ptr__A_0); ADDQ1280($2*SIZE,_ptr__B_0); -._L_45_loopE:; +L(_L_45_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_46_loopE; +JLE L(_L_46_loopE); ALIGN_4; -._L_46_bodyB:; +L(_L_46_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1282(0*SIZE(_ptr__B_0),XMM15); VLD1282(0*SIZE(_ptr__A_0),XMM14); VMA1282(XMM14,XMM15,XMM0,XMM0); ADDQ1280($2*SIZE,_ptr__A_0); ADDQ1280($1*SIZE,_ptr__B_0); -._L_46_loopE:; +L(_L_46_loopE):; BROAD1282(alpha,XMM1); VLDU1282(0*SIZE(_ptr__C_0),XMM2); VMA21282(XMM1,XMM0,XMM2,XMM0); VSTU1282(XMM2,0*SIZE(_ptr__C_0)); ADDQ1280($2*SIZE,_ptr__C_0); ADDQ1280($2*SIZE,_ptr__C_1); -._L_43_loopE:; +L(_L_43_loopE):; TESTQ1280($1,i); -JLE ._L_47_loopE; +JLE L(_L_47_loopE); ALIGN_4; -._L_47_bodyB:; +L(_L_47_bodyB):; MOVQ1280(_ptr_B,_ptr__B_0); VXOR1281(XMM0,XMM0,XMM0); PREFETCHN1280(3*SIZE(_ptr__C_0),N); @@ -1656,9 +1656,9 @@ PREFETCHN1280(3*SIZE(_ptr__C_1),N); PREFETCHN1280(11*SIZE(_ptr__C_1,LDC,1),N); MOVQ1280(_bk_l,k); SARQ1280($2,k); -JLE ._L_48_loopE; +JLE L(_L_48_loopE); ALIGN_4; -._L_48_bodyB:; +L(_L_48_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1281(0*SIZE(_ptr__B_0),XMM15); VLD1281(0*SIZE(_ptr__A_0),XMM14); @@ -1677,15 +1677,15 @@ VLD1281(3*SIZE(_ptr__A_0),XMM14); VMA1281(XMM14,XMM15,XMM0,XMM0); ADDQ1280($4*SIZE,_ptr__A_0); ADDQ1280($4*SIZE,_ptr__B_0); -._L_48_bodyE:; +L(_L_48_bodyE):; DECQ1280(k); -JG ._L_48_bodyB; +JG L(_L_48_bodyB); ALIGN_4; -._L_48_loopE:; +L(_L_48_loopE):; TESTQ1280($2,_bk_l); -JLE ._L_49_loopE; +JLE L(_L_49_loopE); ALIGN_4; -._L_49_bodyB:; +L(_L_49_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1281(0*SIZE(_ptr__B_0),XMM15); VLD1281(0*SIZE(_ptr__A_0),XMM14); @@ -1696,31 +1696,31 @@ VLD1281(1*SIZE(_ptr__A_0),XMM14); VMA1281(XMM14,XMM15,XMM0,XMM0); ADDQ1280($2*SIZE,_ptr__A_0); ADDQ1280($2*SIZE,_ptr__B_0); -._L_49_loopE:; +L(_L_49_loopE):; TESTQ1280($1,_bk_l); -JLE ._L_50_loopE; +JLE L(_L_50_loopE); ALIGN_4; -._L_50_bodyB:; +L(_L_50_bodyB):; PREFETCH01280(160*SIZE(_ptr__A_0),0); BROAD1281(0*SIZE(_ptr__B_0),XMM15); VLD1281(0*SIZE(_ptr__A_0),XMM14); VMA1281(XMM14,XMM15,XMM0,XMM0); ADDQ1280($1*SIZE,_ptr__A_0); ADDQ1280($1*SIZE,_ptr__B_0); -._L_50_loopE:; +L(_L_50_loopE):; BROAD1281(alpha,XMM1); VLDU1281(0*SIZE(_ptr__C_0),XMM2); VMA21281(XMM1,XMM0,XMM2,XMM0); VSTU1281(XMM2,0*SIZE(_ptr__C_0)); ADDQ1280($1*SIZE,_ptr__C_0); ADDQ1280($1*SIZE,_ptr__C_1); -._L_47_loopE:; +L(_L_47_loopE):; MOVQ1280(LDC,%rax); ADDQ1280(%rax,_ptr_C); MOVQ1280(_bk_l,%rax); SALQ1280($3,%rax); ADDQ1280(%rax,_ptr_B); -._L_34_loopE:; +L(_L_34_loopE):; vzeroupper movq 0(%rsp), %rbx; movq 8(%rsp), %rbp; From c7bc1bce0c2bbd719c457829da834e5cb37b456d Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Thu, 1 Oct 2026 23:14:21 -0400 Subject: [PATCH 5/5] arm64: use .subsections_via_symbols on Darwin Apple's linker never removes anything from an object without this directive, so a static link with -dead_strip kept every assembly kernel it had loaded, used or not. With it, a kernel that nothing refers to is left out, as the C kernels already are. This is safe now that every label inside a kernel is L(name), local to the assembler: each kernel object defines one symbol, its entry point, and so is one block for the linker. EPILOGUE, until now empty, is added to the five kernels that did not end in it. Co-Authored-By: Claude Fable 5.1 --- common_arm64.h | 4 ++++ kernel/arm64/axpy.S | 2 ++ kernel/arm64/daxpy_thunderx2t99.S | 2 ++ kernel/arm64/rot.S | 2 ++ kernel/arm64/zaxpy.S | 2 ++ kernel/arm64/zrot.S | 2 ++ 6 files changed, 14 insertions(+) diff --git a/common_arm64.h b/common_arm64.h index 511b884854..45e7bbcbf4 100644 --- a/common_arm64.h +++ b/common_arm64.h @@ -168,7 +168,11 @@ static inline int blas_quickdivide(blasint x, blasint y){ .endm +#if defined(__APPLE__) +#define EPILOGUE .subsections_via_symbols +#else #define EPILOGUE +#endif #define PROFCODE diff --git a/kernel/arm64/axpy.S b/kernel/arm64/axpy.S index 9aa1a4f6a3..22a9a8e4d9 100644 --- a/kernel/arm64/axpy.S +++ b/kernel/arm64/axpy.S @@ -207,3 +207,5 @@ L(axpy_kernel_L999): mov w0, wzr ret + + EPILOGUE diff --git a/kernel/arm64/daxpy_thunderx2t99.S b/kernel/arm64/daxpy_thunderx2t99.S index 5205032ccc..cf2e9fd13f 100644 --- a/kernel/arm64/daxpy_thunderx2t99.S +++ b/kernel/arm64/daxpy_thunderx2t99.S @@ -253,3 +253,5 @@ L(daxpy_kernel_L999): mov w0, wzr ret + + EPILOGUE diff --git a/kernel/arm64/rot.S b/kernel/arm64/rot.S index 33b404db4e..3f37e9b1e9 100644 --- a/kernel/arm64/rot.S +++ b/kernel/arm64/rot.S @@ -243,3 +243,5 @@ L(rot_kernel_L999): mov w0, wzr ret + + EPILOGUE diff --git a/kernel/arm64/zaxpy.S b/kernel/arm64/zaxpy.S index 6a26284902..ea29968fd3 100644 --- a/kernel/arm64/zaxpy.S +++ b/kernel/arm64/zaxpy.S @@ -322,3 +322,5 @@ L(zaxpy_kernel_L999): mov w0, wzr ret + + EPILOGUE diff --git a/kernel/arm64/zrot.S b/kernel/arm64/zrot.S index c6bdefae3f..8186e58092 100644 --- a/kernel/arm64/zrot.S +++ b/kernel/arm64/zrot.S @@ -254,3 +254,5 @@ L(zrot_kernel_L999): mov w0, wzr ret + + EPILOGUE