From 7e876d53bdf2c494f3b2e9ba5dbb912bde3f3762 Mon Sep 17 00:00:00 2001 From: Joris Gillis Date: Fri, 25 Sep 2026 12:39:02 +0200 Subject: [PATCH 1/2] Implement GEMMT with a blocked level-3 driver instead of a GEMV loop ?GEMMT / ?GEMMTR has been a loop over the columns of the triangle with one ?GEMV per column since it was added in 0.3.22. Besides being much slower than GEMM (issue #4921), every column above the GEMV multithreading threshold becomes its own thread dispatch, so the routine gains almost nothing from more cores. Add a real level-3 driver. No new microkernels are needed: the blocks that cross the diagonal are handled by the existing SYRK kernel, which already clips a block against the diagonal at register-block granularity. - driver/level3/level3_gemmt.c is derived from level3_syrk.c. The blocking, the diagonal handling and the loop structure are the same; the packed B buffer is filled from a second matrix, so the buffer sharing SYRK can do is not available, and op(A) / op(B) are selected by the NN/NT/.../CC defines that level3.c uses. - driver/level3/gemmt_k.c holds the triangular beta scaling and the choice of kernel, and is built once per uplo x op(A) x op(B) just as syrk_k.c is built per uplo x trans. - For complex, syrk_kernel.c is also built with CONJA / CONJB; the file already supported those defines but nothing instantiated them. interface/gemmt.c dispatches through a [uplo][op(B)][op(A)] table like interface/gemm.c, and above the existing SMP threshold splits the triangle into column ranges of equal area with syrk_thread(), so every thread writes a disjoint set of columns of C. Only the blocks that intersect the requested triangle are packed and computed, and only the triangle is scaled by beta. This also stops complex GEMMT from writing to the caller's B: conjugation used to be done by conjugating B in place and undoing it afterwards, which is unsafe for a shared or read-only B. dgemmt on an 8-core/16-thread i7-11800H, SKYLAKEX kernels, time for one call, with dsyrk on the same data for reference: n=1600 k=128, 1 thread: 18.7 ms -> 5.14 ms (dsyrk 5.12 ms) n=1600 k=128, 4 threads: 6.92 ms -> 1.54 ms (dsyrk 1.53 ms) n=1600 k=128, 16 threads: 11.7 ms -> 1.01 ms (dsyrk 1.01 ms) n=400 k=128, 16 threads: 1.31 ms -> 0.10 ms (dsyrk 0.11 ms) One deliberate deviation from level3_syrk.c: for a row band that lies entirely below the column block, level3_syrk.c bounds its column loop with `jjs < min_j`, which mixes an absolute index with a length and packs nothing when js > 0. The GEMMT driver uses `jjs < js + min_j`. Neither interface passes a row range, so the branch is only reachable by calling the driver directly; level3_syrk.c is left alone here. Add a utest case per precision at a size that is neither a multiple of the register blocking nor small enough to stay single-threaded, which is where the packing and the diagonal clipping have to agree. Co-Authored-By: Claude Opus 5 (1M context) Co-Authored-By: Claude Opus 5.5 (1M context) --- common_c.h | 40 ++ common_d.h | 9 + common_level3.h | 145 +++++ common_macro.h | 156 +++++ common_q.h | 9 + common_s.h | 9 + common_x.h | 40 ++ common_z.h | 40 ++ driver/level3/CMakeLists.txt | 25 + driver/level3/Makefile | 881 ++++++++++++++++++++++++++++ driver/level3/gemmt_k.c | 127 ++++ driver/level3/level3_gemmt.c | 508 ++++++++++++++++ interface/gemmt.c | 378 +++--------- utest/test_extensions/test_cgemmt.c | 56 ++ utest/test_extensions/test_dgemmt.c | 56 ++ utest/test_extensions/test_sgemmt.c | 56 ++ utest/test_extensions/test_zgemmt.c | 56 ++ 17 files changed, 2298 insertions(+), 293 deletions(-) create mode 100644 driver/level3/gemmt_k.c create mode 100644 driver/level3/level3_gemmt.c diff --git a/common_c.h b/common_c.h index 88f8f5dcdf..3aa9349d0a 100644 --- a/common_c.h +++ b/common_c.h @@ -514,6 +514,46 @@ #define CSYRK_KERNEL_U csyrk_kernel_U #define CSYRK_KERNEL_L csyrk_kernel_L +#define CGEMMT_UNN cgemmt_UNN +#define CGEMMT_UNT cgemmt_UNT +#define CGEMMT_UNR cgemmt_UNR +#define CGEMMT_UNC cgemmt_UNC +#define CGEMMT_UTN cgemmt_UTN +#define CGEMMT_UTT cgemmt_UTT +#define CGEMMT_UTR cgemmt_UTR +#define CGEMMT_UTC cgemmt_UTC +#define CGEMMT_URN cgemmt_URN +#define CGEMMT_URT cgemmt_URT +#define CGEMMT_URR cgemmt_URR +#define CGEMMT_URC cgemmt_URC +#define CGEMMT_UCN cgemmt_UCN +#define CGEMMT_UCT cgemmt_UCT +#define CGEMMT_UCR cgemmt_UCR +#define CGEMMT_UCC cgemmt_UCC +#define CGEMMT_LNN cgemmt_LNN +#define CGEMMT_LNT cgemmt_LNT +#define CGEMMT_LNR cgemmt_LNR +#define CGEMMT_LNC cgemmt_LNC +#define CGEMMT_LTN cgemmt_LTN +#define CGEMMT_LTT cgemmt_LTT +#define CGEMMT_LTR cgemmt_LTR +#define CGEMMT_LTC cgemmt_LTC +#define CGEMMT_LRN cgemmt_LRN +#define CGEMMT_LRT cgemmt_LRT +#define CGEMMT_LRR cgemmt_LRR +#define CGEMMT_LRC cgemmt_LRC +#define CGEMMT_LCN cgemmt_LCN +#define CGEMMT_LCT cgemmt_LCT +#define CGEMMT_LCR cgemmt_LCR +#define CGEMMT_LCC cgemmt_LCC + +#define CGEMMT_KERNEL_UCN cgemmt_kernel_UCN +#define CGEMMT_KERNEL_UNC cgemmt_kernel_UNC +#define CGEMMT_KERNEL_UCC cgemmt_kernel_UCC +#define CGEMMT_KERNEL_LCN cgemmt_kernel_LCN +#define CGEMMT_KERNEL_LNC cgemmt_kernel_LNC +#define CGEMMT_KERNEL_LCC cgemmt_kernel_LCC + #define CHERK_UN cherk_UN #define CHERK_LN cherk_LN #define CHERK_UC cherk_UC diff --git a/common_d.h b/common_d.h index a7a1e1cd14..200d840b0d 100644 --- a/common_d.h +++ b/common_d.h @@ -341,6 +341,15 @@ #define DSYRK_KERNEL_U dsyrk_kernel_U #define DSYRK_KERNEL_L dsyrk_kernel_L +#define DGEMMT_UNN dgemmt_UNN +#define DGEMMT_UNT dgemmt_UNT +#define DGEMMT_UTN dgemmt_UTN +#define DGEMMT_UTT dgemmt_UTT +#define DGEMMT_LNN dgemmt_LNN +#define DGEMMT_LNT dgemmt_LNT +#define DGEMMT_LTN dgemmt_LTN +#define DGEMMT_LTT dgemmt_LTT + #define DHERK_UN dsyrk_UN #define DHERK_LN dsyrk_LN #define DHERK_UC dsyrk_UT diff --git a/common_level3.h b/common_level3.h index 3e4edf1950..6ef6cf2331 100644 --- a/common_level3.h +++ b/common_level3.h @@ -1689,6 +1689,151 @@ int ssyrk_thread_LN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLAS int ssyrk_thread_LT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); int dsyrk_thread_UN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); + +int qgemmt_UNN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int qgemmt_UNT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int qgemmt_UTN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int qgemmt_UTT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int qgemmt_LNN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int qgemmt_LNT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int qgemmt_LTN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int qgemmt_LTT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); + +int dgemmt_UNN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int dgemmt_UNT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int dgemmt_UTN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int dgemmt_UTT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int dgemmt_LNN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int dgemmt_LNT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int dgemmt_LTN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int dgemmt_LTT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); + +int sgemmt_UNN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int sgemmt_UNT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int sgemmt_UTN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int sgemmt_UTT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int sgemmt_LNN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int sgemmt_LNT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int sgemmt_LTN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int sgemmt_LTT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); + +int xgemmt_UNN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_UNT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_UNR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_UNC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_UTN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_UTT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_UTR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_UTC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_URN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_URT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_URR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_URC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_UCN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_UCT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_UCR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_UCC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LNN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LNT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LNR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LNC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LTN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LTT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LTR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LTC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LRN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LRT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LRR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LRC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LCN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LCT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LCR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LCC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_kernel_UCN(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); +int xgemmt_kernel_UNC(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); +int xgemmt_kernel_UCC(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); +int xgemmt_kernel_LCN(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); +int xgemmt_kernel_LNC(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); +int xgemmt_kernel_LCC(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); + +int zgemmt_UNN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_UNT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_UNR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_UNC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_UTN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_UTT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_UTR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_UTC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_URN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_URT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_URR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_URC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_UCN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_UCT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_UCR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_UCC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LNN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LNT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LNR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LNC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LTN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LTT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LTR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LTC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LRN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LRT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LRR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LRC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LCN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LCT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LCR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LCC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_kernel_UCN(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); +int zgemmt_kernel_UNC(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); +int zgemmt_kernel_UCC(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); +int zgemmt_kernel_LCN(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); +int zgemmt_kernel_LNC(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); +int zgemmt_kernel_LCC(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); + +int cgemmt_UNN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_UNT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_UNR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_UNC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_UTN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_UTT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_UTR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_UTC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_URN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_URT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_URR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_URC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_UCN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_UCT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_UCR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_UCC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LNN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LNT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LNR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LNC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LTN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LTT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LTR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LTC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LRN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LRT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LRR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LRC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LCN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LCT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LCR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LCC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_kernel_UCN(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); +int cgemmt_kernel_UNC(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); +int cgemmt_kernel_UCC(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); +int cgemmt_kernel_LCN(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); +int cgemmt_kernel_LNC(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); +int cgemmt_kernel_LCC(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); + int dsyrk_thread_UT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); int dsyrk_thread_LN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); int dsyrk_thread_LT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); diff --git a/common_macro.h b/common_macro.h index 84c2473020..2b23f06c71 100644 --- a/common_macro.h +++ b/common_macro.h @@ -211,6 +211,15 @@ #define SYRK_KERNEL_U QSYRK_KERNEL_U #define SYRK_KERNEL_L QSYRK_KERNEL_L +#define GEMMT_UNN QGEMMT_UNN +#define GEMMT_UNT QGEMMT_UNT +#define GEMMT_UTN QGEMMT_UTN +#define GEMMT_UTT QGEMMT_UTT +#define GEMMT_LNN QGEMMT_LNN +#define GEMMT_LNT QGEMMT_LNT +#define GEMMT_LTN QGEMMT_LTN +#define GEMMT_LTT QGEMMT_LTT + #define HERK_UN QSYRK_UN #define HERK_LN QSYRK_LN #define HERK_UC QSYRK_UT @@ -503,6 +512,15 @@ #define SYRK_KERNEL_U DSYRK_KERNEL_U #define SYRK_KERNEL_L DSYRK_KERNEL_L +#define GEMMT_UNN DGEMMT_UNN +#define GEMMT_UNT DGEMMT_UNT +#define GEMMT_UTN DGEMMT_UTN +#define GEMMT_UTT DGEMMT_UTT +#define GEMMT_LNN DGEMMT_LNN +#define GEMMT_LNT DGEMMT_LNT +#define GEMMT_LTN DGEMMT_LTN +#define GEMMT_LTT DGEMMT_LTT + #define HERK_UN DSYRK_UN #define HERK_LN DSYRK_LN #define HERK_UC DSYRK_UT @@ -991,6 +1009,15 @@ #define SYRK_KERNEL_U SSYRK_KERNEL_U #define SYRK_KERNEL_L SSYRK_KERNEL_L +#define GEMMT_UNN SGEMMT_UNN +#define GEMMT_UNT SGEMMT_UNT +#define GEMMT_UTN SGEMMT_UTN +#define GEMMT_UTT SGEMMT_UTT +#define GEMMT_LNN SGEMMT_LNN +#define GEMMT_LNT SGEMMT_LNT +#define GEMMT_LTN SGEMMT_LTN +#define GEMMT_LTT SGEMMT_LTT + #define HERK_UN SSYRK_UN #define HERK_LN SSYRK_LN #define HERK_UC SSYRK_UT @@ -1214,6 +1241,15 @@ #define SYRK_KERNEL_U SSYRK_KERNEL_U #define SYRK_KERNEL_L SSYRK_KERNEL_L +#define GEMMT_UNN SGEMMT_UNN +#define GEMMT_UNT SGEMMT_UNT +#define GEMMT_UTN SGEMMT_UTN +#define GEMMT_UTT SGEMMT_UTT +#define GEMMT_LNN SGEMMT_LNN +#define GEMMT_LNT SGEMMT_LNT +#define GEMMT_LTN SGEMMT_LTN +#define GEMMT_LTT SGEMMT_LTT + #define HERK_UN SSYRK_UN #define HERK_LN SSYRK_LN #define HERK_UC SSYRK_UT @@ -1583,6 +1619,46 @@ #define SYRK_KERNEL_U XSYRK_KERNEL_U #define SYRK_KERNEL_L XSYRK_KERNEL_L +#define GEMMT_UNN XGEMMT_UNN +#define GEMMT_UNT XGEMMT_UNT +#define GEMMT_UNR XGEMMT_UNR +#define GEMMT_UNC XGEMMT_UNC +#define GEMMT_UTN XGEMMT_UTN +#define GEMMT_UTT XGEMMT_UTT +#define GEMMT_UTR XGEMMT_UTR +#define GEMMT_UTC XGEMMT_UTC +#define GEMMT_URN XGEMMT_URN +#define GEMMT_URT XGEMMT_URT +#define GEMMT_URR XGEMMT_URR +#define GEMMT_URC XGEMMT_URC +#define GEMMT_UCN XGEMMT_UCN +#define GEMMT_UCT XGEMMT_UCT +#define GEMMT_UCR XGEMMT_UCR +#define GEMMT_UCC XGEMMT_UCC +#define GEMMT_LNN XGEMMT_LNN +#define GEMMT_LNT XGEMMT_LNT +#define GEMMT_LNR XGEMMT_LNR +#define GEMMT_LNC XGEMMT_LNC +#define GEMMT_LTN XGEMMT_LTN +#define GEMMT_LTT XGEMMT_LTT +#define GEMMT_LTR XGEMMT_LTR +#define GEMMT_LTC XGEMMT_LTC +#define GEMMT_LRN XGEMMT_LRN +#define GEMMT_LRT XGEMMT_LRT +#define GEMMT_LRR XGEMMT_LRR +#define GEMMT_LRC XGEMMT_LRC +#define GEMMT_LCN XGEMMT_LCN +#define GEMMT_LCT XGEMMT_LCT +#define GEMMT_LCR XGEMMT_LCR +#define GEMMT_LCC XGEMMT_LCC + +#define GEMMT_KERNEL_UCN XGEMMT_KERNEL_UCN +#define GEMMT_KERNEL_UNC XGEMMT_KERNEL_UNC +#define GEMMT_KERNEL_UCC XGEMMT_KERNEL_UCC +#define GEMMT_KERNEL_LCN XGEMMT_KERNEL_LCN +#define GEMMT_KERNEL_LNC XGEMMT_KERNEL_LNC +#define GEMMT_KERNEL_LCC XGEMMT_KERNEL_LCC + #define HERK_UN XHERK_UN #define HERK_LN XHERK_LN #define HERK_UC XHERK_UC @@ -1985,6 +2061,46 @@ #define SYRK_KERNEL_U ZSYRK_KERNEL_U #define SYRK_KERNEL_L ZSYRK_KERNEL_L +#define GEMMT_UNN ZGEMMT_UNN +#define GEMMT_UNT ZGEMMT_UNT +#define GEMMT_UNR ZGEMMT_UNR +#define GEMMT_UNC ZGEMMT_UNC +#define GEMMT_UTN ZGEMMT_UTN +#define GEMMT_UTT ZGEMMT_UTT +#define GEMMT_UTR ZGEMMT_UTR +#define GEMMT_UTC ZGEMMT_UTC +#define GEMMT_URN ZGEMMT_URN +#define GEMMT_URT ZGEMMT_URT +#define GEMMT_URR ZGEMMT_URR +#define GEMMT_URC ZGEMMT_URC +#define GEMMT_UCN ZGEMMT_UCN +#define GEMMT_UCT ZGEMMT_UCT +#define GEMMT_UCR ZGEMMT_UCR +#define GEMMT_UCC ZGEMMT_UCC +#define GEMMT_LNN ZGEMMT_LNN +#define GEMMT_LNT ZGEMMT_LNT +#define GEMMT_LNR ZGEMMT_LNR +#define GEMMT_LNC ZGEMMT_LNC +#define GEMMT_LTN ZGEMMT_LTN +#define GEMMT_LTT ZGEMMT_LTT +#define GEMMT_LTR ZGEMMT_LTR +#define GEMMT_LTC ZGEMMT_LTC +#define GEMMT_LRN ZGEMMT_LRN +#define GEMMT_LRT ZGEMMT_LRT +#define GEMMT_LRR ZGEMMT_LRR +#define GEMMT_LRC ZGEMMT_LRC +#define GEMMT_LCN ZGEMMT_LCN +#define GEMMT_LCT ZGEMMT_LCT +#define GEMMT_LCR ZGEMMT_LCR +#define GEMMT_LCC ZGEMMT_LCC + +#define GEMMT_KERNEL_UCN ZGEMMT_KERNEL_UCN +#define GEMMT_KERNEL_UNC ZGEMMT_KERNEL_UNC +#define GEMMT_KERNEL_UCC ZGEMMT_KERNEL_UCC +#define GEMMT_KERNEL_LCN ZGEMMT_KERNEL_LCN +#define GEMMT_KERNEL_LNC ZGEMMT_KERNEL_LNC +#define GEMMT_KERNEL_LCC ZGEMMT_KERNEL_LCC + #define HERK_UN ZHERK_UN #define HERK_LN ZHERK_LN #define HERK_UC ZHERK_UC @@ -2450,6 +2566,46 @@ #define SYRK_KERNEL_U CSYRK_KERNEL_U #define SYRK_KERNEL_L CSYRK_KERNEL_L +#define GEMMT_UNN CGEMMT_UNN +#define GEMMT_UNT CGEMMT_UNT +#define GEMMT_UNR CGEMMT_UNR +#define GEMMT_UNC CGEMMT_UNC +#define GEMMT_UTN CGEMMT_UTN +#define GEMMT_UTT CGEMMT_UTT +#define GEMMT_UTR CGEMMT_UTR +#define GEMMT_UTC CGEMMT_UTC +#define GEMMT_URN CGEMMT_URN +#define GEMMT_URT CGEMMT_URT +#define GEMMT_URR CGEMMT_URR +#define GEMMT_URC CGEMMT_URC +#define GEMMT_UCN CGEMMT_UCN +#define GEMMT_UCT CGEMMT_UCT +#define GEMMT_UCR CGEMMT_UCR +#define GEMMT_UCC CGEMMT_UCC +#define GEMMT_LNN CGEMMT_LNN +#define GEMMT_LNT CGEMMT_LNT +#define GEMMT_LNR CGEMMT_LNR +#define GEMMT_LNC CGEMMT_LNC +#define GEMMT_LTN CGEMMT_LTN +#define GEMMT_LTT CGEMMT_LTT +#define GEMMT_LTR CGEMMT_LTR +#define GEMMT_LTC CGEMMT_LTC +#define GEMMT_LRN CGEMMT_LRN +#define GEMMT_LRT CGEMMT_LRT +#define GEMMT_LRR CGEMMT_LRR +#define GEMMT_LRC CGEMMT_LRC +#define GEMMT_LCN CGEMMT_LCN +#define GEMMT_LCT CGEMMT_LCT +#define GEMMT_LCR CGEMMT_LCR +#define GEMMT_LCC CGEMMT_LCC + +#define GEMMT_KERNEL_UCN CGEMMT_KERNEL_UCN +#define GEMMT_KERNEL_UNC CGEMMT_KERNEL_UNC +#define GEMMT_KERNEL_UCC CGEMMT_KERNEL_UCC +#define GEMMT_KERNEL_LCN CGEMMT_KERNEL_LCN +#define GEMMT_KERNEL_LNC CGEMMT_KERNEL_LNC +#define GEMMT_KERNEL_LCC CGEMMT_KERNEL_LCC + #define HERK_UN CHERK_UN #define HERK_LN CHERK_LN #define HERK_UC CHERK_UC diff --git a/common_q.h b/common_q.h index 1d976f1e8d..52da71c88c 100644 --- a/common_q.h +++ b/common_q.h @@ -299,6 +299,15 @@ #define QSYRK_KERNEL_U qsyrk_kernel_U #define QSYRK_KERNEL_L qsyrk_kernel_L +#define QGEMMT_UNN qgemmt_UNN +#define QGEMMT_UNT qgemmt_UNT +#define QGEMMT_UTN qgemmt_UTN +#define QGEMMT_UTT qgemmt_UTT +#define QGEMMT_LNN qgemmt_LNN +#define QGEMMT_LNT qgemmt_LNT +#define QGEMMT_LTN qgemmt_LTN +#define QGEMMT_LTT qgemmt_LTT + #define QHERK_UN qsyrk_UN #define QHERK_LN qsyrk_LN #define QHERK_UC qsyrk_UT diff --git a/common_s.h b/common_s.h index 6df9ba0f1e..4e5f1af7cb 100644 --- a/common_s.h +++ b/common_s.h @@ -389,6 +389,15 @@ #define SSYRK_KERNEL_U ssyrk_kernel_U #define SSYRK_KERNEL_L ssyrk_kernel_L +#define SGEMMT_UNN sgemmt_UNN +#define SGEMMT_UNT sgemmt_UNT +#define SGEMMT_UTN sgemmt_UTN +#define SGEMMT_UTT sgemmt_UTT +#define SGEMMT_LNN sgemmt_LNN +#define SGEMMT_LNT sgemmt_LNT +#define SGEMMT_LTN sgemmt_LTN +#define SGEMMT_LTT sgemmt_LTT + #define SHERK_UN ssyrk_UN #define SHERK_LN ssyrk_LN #define SHERK_UC ssyrk_UT diff --git a/common_x.h b/common_x.h index 2ed525faa1..12ed909c57 100644 --- a/common_x.h +++ b/common_x.h @@ -423,6 +423,46 @@ #define XSYRK_KERNEL_U xsyrk_kernel_U #define XSYRK_KERNEL_L xsyrk_kernel_L +#define XGEMMT_UNN xgemmt_UNN +#define XGEMMT_UNT xgemmt_UNT +#define XGEMMT_UNR xgemmt_UNR +#define XGEMMT_UNC xgemmt_UNC +#define XGEMMT_UTN xgemmt_UTN +#define XGEMMT_UTT xgemmt_UTT +#define XGEMMT_UTR xgemmt_UTR +#define XGEMMT_UTC xgemmt_UTC +#define XGEMMT_URN xgemmt_URN +#define XGEMMT_URT xgemmt_URT +#define XGEMMT_URR xgemmt_URR +#define XGEMMT_URC xgemmt_URC +#define XGEMMT_UCN xgemmt_UCN +#define XGEMMT_UCT xgemmt_UCT +#define XGEMMT_UCR xgemmt_UCR +#define XGEMMT_UCC xgemmt_UCC +#define XGEMMT_LNN xgemmt_LNN +#define XGEMMT_LNT xgemmt_LNT +#define XGEMMT_LNR xgemmt_LNR +#define XGEMMT_LNC xgemmt_LNC +#define XGEMMT_LTN xgemmt_LTN +#define XGEMMT_LTT xgemmt_LTT +#define XGEMMT_LTR xgemmt_LTR +#define XGEMMT_LTC xgemmt_LTC +#define XGEMMT_LRN xgemmt_LRN +#define XGEMMT_LRT xgemmt_LRT +#define XGEMMT_LRR xgemmt_LRR +#define XGEMMT_LRC xgemmt_LRC +#define XGEMMT_LCN xgemmt_LCN +#define XGEMMT_LCT xgemmt_LCT +#define XGEMMT_LCR xgemmt_LCR +#define XGEMMT_LCC xgemmt_LCC + +#define XGEMMT_KERNEL_UCN xgemmt_kernel_UCN +#define XGEMMT_KERNEL_UNC xgemmt_kernel_UNC +#define XGEMMT_KERNEL_UCC xgemmt_kernel_UCC +#define XGEMMT_KERNEL_LCN xgemmt_kernel_LCN +#define XGEMMT_KERNEL_LNC xgemmt_kernel_LNC +#define XGEMMT_KERNEL_LCC xgemmt_kernel_LCC + #define XHERK_UN xherk_UN #define XHERK_LN xherk_LN #define XHERK_UC xherk_UC diff --git a/common_z.h b/common_z.h index 5a9fb88b77..bd79f49d0f 100644 --- a/common_z.h +++ b/common_z.h @@ -514,6 +514,46 @@ #define ZSYRK_KERNEL_U zsyrk_kernel_U #define ZSYRK_KERNEL_L zsyrk_kernel_L +#define ZGEMMT_UNN zgemmt_UNN +#define ZGEMMT_UNT zgemmt_UNT +#define ZGEMMT_UNR zgemmt_UNR +#define ZGEMMT_UNC zgemmt_UNC +#define ZGEMMT_UTN zgemmt_UTN +#define ZGEMMT_UTT zgemmt_UTT +#define ZGEMMT_UTR zgemmt_UTR +#define ZGEMMT_UTC zgemmt_UTC +#define ZGEMMT_URN zgemmt_URN +#define ZGEMMT_URT zgemmt_URT +#define ZGEMMT_URR zgemmt_URR +#define ZGEMMT_URC zgemmt_URC +#define ZGEMMT_UCN zgemmt_UCN +#define ZGEMMT_UCT zgemmt_UCT +#define ZGEMMT_UCR zgemmt_UCR +#define ZGEMMT_UCC zgemmt_UCC +#define ZGEMMT_LNN zgemmt_LNN +#define ZGEMMT_LNT zgemmt_LNT +#define ZGEMMT_LNR zgemmt_LNR +#define ZGEMMT_LNC zgemmt_LNC +#define ZGEMMT_LTN zgemmt_LTN +#define ZGEMMT_LTT zgemmt_LTT +#define ZGEMMT_LTR zgemmt_LTR +#define ZGEMMT_LTC zgemmt_LTC +#define ZGEMMT_LRN zgemmt_LRN +#define ZGEMMT_LRT zgemmt_LRT +#define ZGEMMT_LRR zgemmt_LRR +#define ZGEMMT_LRC zgemmt_LRC +#define ZGEMMT_LCN zgemmt_LCN +#define ZGEMMT_LCT zgemmt_LCT +#define ZGEMMT_LCR zgemmt_LCR +#define ZGEMMT_LCC zgemmt_LCC + +#define ZGEMMT_KERNEL_UCN zgemmt_kernel_UCN +#define ZGEMMT_KERNEL_UNC zgemmt_kernel_UNC +#define ZGEMMT_KERNEL_UCC zgemmt_kernel_UCC +#define ZGEMMT_KERNEL_LCN zgemmt_kernel_LCN +#define ZGEMMT_KERNEL_LNC zgemmt_kernel_LNC +#define ZGEMMT_KERNEL_LCC zgemmt_kernel_LCC + #define ZHERK_UN zherk_UN #define ZHERK_LN zherk_LN #define ZHERK_UC zherk_UC diff --git a/driver/level3/CMakeLists.txt b/driver/level3/CMakeLists.txt index 5024331dab..58a6d3e38f 100644 --- a/driver/level3/CMakeLists.txt +++ b/driver/level3/CMakeLists.txt @@ -63,6 +63,31 @@ GenerateCombinationObjects("symm_k.c" "RSIDE;LOWER" "L;U" "NN" 1) GenerateCombinationObjects("syrk_k.c" "LOWER;TRANS" "U;N" "" 1) GenerateCombinationObjects("syr2k_k.c" "LOWER;TRANS" "U;N" "" 1) GenerateCombinationObjects("syrk_kernel.c" "LOWER" "U" "" 2) + +# GEMMT / GEMMTR: one driver per uplo x op(A) x op(B) (the same set of +# op combinations as GEMM), plus the conjugating variants of the +# triangular kernel, which only complex GEMMT needs. +set(GEMMT_REAL_TRANS N T) +set(GEMMT_COMPLEX_TRANS N T R C) +foreach (GEMMT_UPLO U L) + set(GEMMT_UPLO_DEFS "") + if (${GEMMT_UPLO} STREQUAL "L") + set(GEMMT_UPLO_DEFS "LOWER") + endif () + foreach (GEMMT_TA ${GEMMT_REAL_TRANS}) + foreach (GEMMT_TB ${GEMMT_REAL_TRANS}) + GenerateNamedObjects("gemmt_k.c" "${GEMMT_TA}${GEMMT_TB};${GEMMT_UPLO_DEFS}" "gemmt_${GEMMT_UPLO}${GEMMT_TA}${GEMMT_TB}" 0 "" "" false 1) + endforeach () + endforeach () + foreach (GEMMT_TA ${GEMMT_COMPLEX_TRANS}) + foreach (GEMMT_TB ${GEMMT_COMPLEX_TRANS}) + GenerateNamedObjects("gemmt_k.c" "${GEMMT_TA}${GEMMT_TB};${GEMMT_UPLO_DEFS}" "gemmt_${GEMMT_UPLO}${GEMMT_TA}${GEMMT_TB}" 0 "" "" false 2) + endforeach () + endforeach () + GenerateNamedObjects("syrk_kernel.c" "CONJA;${GEMMT_UPLO_DEFS}" "gemmt_kernel_${GEMMT_UPLO}CN" 0 "" "" false 2) + GenerateNamedObjects("syrk_kernel.c" "CONJB;${GEMMT_UPLO_DEFS}" "gemmt_kernel_${GEMMT_UPLO}NC" 0 "" "" false 2) + GenerateNamedObjects("syrk_kernel.c" "CONJA;CONJB;${GEMMT_UPLO_DEFS}" "gemmt_kernel_${GEMMT_UPLO}CC" 0 "" "" false 2) +endforeach () GenerateCombinationObjects("syr2k_kernel.c" "LOWER" "U" "" 2) if (USE_THREAD) diff --git a/driver/level3/Makefile b/driver/level3/Makefile index 70655d9cd3..9896fc6983 100644 --- a/driver/level3/Makefile +++ b/driver/level3/Makefile @@ -208,6 +208,57 @@ XBLASOBJS += \ xher2k_kernel_UN.$(SUFFIX) xher2k_kernel_UC.$(SUFFIX) \ xher2k_kernel_LN.$(SUFFIX) xher2k_kernel_LC.$(SUFFIX) + +# GEMMT / GEMMTR: one driver per uplo x op(A) x op(B), plus the +# conjugating variants of the triangular (SYRK) kernel for complex. +SBLASOBJS += \ + sgemmt_UNN.$(SUFFIX) sgemmt_UNT.$(SUFFIX) sgemmt_UTN.$(SUFFIX) sgemmt_UTT.$(SUFFIX) \ + sgemmt_LNN.$(SUFFIX) sgemmt_LNT.$(SUFFIX) sgemmt_LTN.$(SUFFIX) sgemmt_LTT.$(SUFFIX) + +DBLASOBJS += \ + dgemmt_UNN.$(SUFFIX) dgemmt_UNT.$(SUFFIX) dgemmt_UTN.$(SUFFIX) dgemmt_UTT.$(SUFFIX) \ + dgemmt_LNN.$(SUFFIX) dgemmt_LNT.$(SUFFIX) dgemmt_LTN.$(SUFFIX) dgemmt_LTT.$(SUFFIX) + +QBLASOBJS += \ + qgemmt_UNN.$(SUFFIX) qgemmt_UNT.$(SUFFIX) qgemmt_UTN.$(SUFFIX) qgemmt_UTT.$(SUFFIX) \ + qgemmt_LNN.$(SUFFIX) qgemmt_LNT.$(SUFFIX) qgemmt_LTN.$(SUFFIX) qgemmt_LTT.$(SUFFIX) + +CBLASOBJS += \ + cgemmt_UNN.$(SUFFIX) cgemmt_UNT.$(SUFFIX) cgemmt_UNR.$(SUFFIX) cgemmt_UNC.$(SUFFIX) \ + cgemmt_UTN.$(SUFFIX) cgemmt_UTT.$(SUFFIX) cgemmt_UTR.$(SUFFIX) cgemmt_UTC.$(SUFFIX) \ + cgemmt_URN.$(SUFFIX) cgemmt_URT.$(SUFFIX) cgemmt_URR.$(SUFFIX) cgemmt_URC.$(SUFFIX) \ + cgemmt_UCN.$(SUFFIX) cgemmt_UCT.$(SUFFIX) cgemmt_UCR.$(SUFFIX) cgemmt_UCC.$(SUFFIX) \ + cgemmt_LNN.$(SUFFIX) cgemmt_LNT.$(SUFFIX) cgemmt_LNR.$(SUFFIX) cgemmt_LNC.$(SUFFIX) \ + cgemmt_LTN.$(SUFFIX) cgemmt_LTT.$(SUFFIX) cgemmt_LTR.$(SUFFIX) cgemmt_LTC.$(SUFFIX) \ + cgemmt_LRN.$(SUFFIX) cgemmt_LRT.$(SUFFIX) cgemmt_LRR.$(SUFFIX) cgemmt_LRC.$(SUFFIX) \ + cgemmt_LCN.$(SUFFIX) cgemmt_LCT.$(SUFFIX) cgemmt_LCR.$(SUFFIX) cgemmt_LCC.$(SUFFIX) \ + cgemmt_kernel_UCN.$(SUFFIX) cgemmt_kernel_UNC.$(SUFFIX) cgemmt_kernel_UCC.$(SUFFIX) cgemmt_kernel_LCN.$(SUFFIX) \ + cgemmt_kernel_LNC.$(SUFFIX) cgemmt_kernel_LCC.$(SUFFIX) + +ZBLASOBJS += \ + zgemmt_UNN.$(SUFFIX) zgemmt_UNT.$(SUFFIX) zgemmt_UNR.$(SUFFIX) zgemmt_UNC.$(SUFFIX) \ + zgemmt_UTN.$(SUFFIX) zgemmt_UTT.$(SUFFIX) zgemmt_UTR.$(SUFFIX) zgemmt_UTC.$(SUFFIX) \ + zgemmt_URN.$(SUFFIX) zgemmt_URT.$(SUFFIX) zgemmt_URR.$(SUFFIX) zgemmt_URC.$(SUFFIX) \ + zgemmt_UCN.$(SUFFIX) zgemmt_UCT.$(SUFFIX) zgemmt_UCR.$(SUFFIX) zgemmt_UCC.$(SUFFIX) \ + zgemmt_LNN.$(SUFFIX) zgemmt_LNT.$(SUFFIX) zgemmt_LNR.$(SUFFIX) zgemmt_LNC.$(SUFFIX) \ + zgemmt_LTN.$(SUFFIX) zgemmt_LTT.$(SUFFIX) zgemmt_LTR.$(SUFFIX) zgemmt_LTC.$(SUFFIX) \ + zgemmt_LRN.$(SUFFIX) zgemmt_LRT.$(SUFFIX) zgemmt_LRR.$(SUFFIX) zgemmt_LRC.$(SUFFIX) \ + zgemmt_LCN.$(SUFFIX) zgemmt_LCT.$(SUFFIX) zgemmt_LCR.$(SUFFIX) zgemmt_LCC.$(SUFFIX) \ + zgemmt_kernel_UCN.$(SUFFIX) zgemmt_kernel_UNC.$(SUFFIX) zgemmt_kernel_UCC.$(SUFFIX) zgemmt_kernel_LCN.$(SUFFIX) \ + zgemmt_kernel_LNC.$(SUFFIX) zgemmt_kernel_LCC.$(SUFFIX) + +XBLASOBJS += \ + xgemmt_UNN.$(SUFFIX) xgemmt_UNT.$(SUFFIX) xgemmt_UNR.$(SUFFIX) xgemmt_UNC.$(SUFFIX) \ + xgemmt_UTN.$(SUFFIX) xgemmt_UTT.$(SUFFIX) xgemmt_UTR.$(SUFFIX) xgemmt_UTC.$(SUFFIX) \ + xgemmt_URN.$(SUFFIX) xgemmt_URT.$(SUFFIX) xgemmt_URR.$(SUFFIX) xgemmt_URC.$(SUFFIX) \ + xgemmt_UCN.$(SUFFIX) xgemmt_UCT.$(SUFFIX) xgemmt_UCR.$(SUFFIX) xgemmt_UCC.$(SUFFIX) \ + xgemmt_LNN.$(SUFFIX) xgemmt_LNT.$(SUFFIX) xgemmt_LNR.$(SUFFIX) xgemmt_LNC.$(SUFFIX) \ + xgemmt_LTN.$(SUFFIX) xgemmt_LTT.$(SUFFIX) xgemmt_LTR.$(SUFFIX) xgemmt_LTC.$(SUFFIX) \ + xgemmt_LRN.$(SUFFIX) xgemmt_LRT.$(SUFFIX) xgemmt_LRR.$(SUFFIX) xgemmt_LRC.$(SUFFIX) \ + xgemmt_LCN.$(SUFFIX) xgemmt_LCT.$(SUFFIX) xgemmt_LCR.$(SUFFIX) xgemmt_LCC.$(SUFFIX) \ + xgemmt_kernel_UCN.$(SUFFIX) xgemmt_kernel_UNC.$(SUFFIX) xgemmt_kernel_UCC.$(SUFFIX) xgemmt_kernel_LCN.$(SUFFIX) \ + xgemmt_kernel_LNC.$(SUFFIX) xgemmt_kernel_LCC.$(SUFFIX) + ifeq ($(USE_GEMM3M), 1) CBLASOBJS += \ @@ -1587,6 +1638,421 @@ xsyrk_kernel_U.$(SUFFIX) : syrk_kernel.c xsyrk_kernel_L.$(SUFFIX) : syrk_kernel.c $(CC) -c $(CFLAGS) -DXDOUBLE -DCOMPLEX -DLOWER $< -o $(@F) + +sgemmt_UNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -ULOWER -DNN $< -o $(@F) + +sgemmt_UNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -ULOWER -DNT $< -o $(@F) + +sgemmt_UTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -ULOWER -DTN $< -o $(@F) + +sgemmt_UTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -ULOWER -DTT $< -o $(@F) + +sgemmt_LNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -DLOWER -DNN $< -o $(@F) + +sgemmt_LNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -DLOWER -DNT $< -o $(@F) + +sgemmt_LTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -DLOWER -DTN $< -o $(@F) + +sgemmt_LTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -DLOWER -DTT $< -o $(@F) + +dgemmt_UNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -ULOWER -DNN $< -o $(@F) + +dgemmt_UNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -ULOWER -DNT $< -o $(@F) + +dgemmt_UTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -ULOWER -DTN $< -o $(@F) + +dgemmt_UTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -ULOWER -DTT $< -o $(@F) + +dgemmt_LNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -DLOWER -DNN $< -o $(@F) + +dgemmt_LNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -DLOWER -DNT $< -o $(@F) + +dgemmt_LTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -DLOWER -DTN $< -o $(@F) + +dgemmt_LTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -DLOWER -DTT $< -o $(@F) + +qgemmt_UNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -ULOWER -DNN $< -o $(@F) + +qgemmt_UNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -ULOWER -DNT $< -o $(@F) + +qgemmt_UTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -ULOWER -DTN $< -o $(@F) + +qgemmt_UTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -ULOWER -DTT $< -o $(@F) + +qgemmt_LNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -DLOWER -DNN $< -o $(@F) + +qgemmt_LNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -DLOWER -DNT $< -o $(@F) + +qgemmt_LTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -DLOWER -DTN $< -o $(@F) + +qgemmt_LTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -DLOWER -DTT $< -o $(@F) + +cgemmt_UNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DNN $< -o $(@F) + +cgemmt_UNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DNT $< -o $(@F) + +cgemmt_UNR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DNR $< -o $(@F) + +cgemmt_UNC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DNC $< -o $(@F) + +cgemmt_UTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DTN $< -o $(@F) + +cgemmt_UTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DTT $< -o $(@F) + +cgemmt_UTR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DTR $< -o $(@F) + +cgemmt_UTC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DTC $< -o $(@F) + +cgemmt_URN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DRN $< -o $(@F) + +cgemmt_URT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DRT $< -o $(@F) + +cgemmt_URR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DRR $< -o $(@F) + +cgemmt_URC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DRC $< -o $(@F) + +cgemmt_UCN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCN $< -o $(@F) + +cgemmt_UCT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCT $< -o $(@F) + +cgemmt_UCR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCR $< -o $(@F) + +cgemmt_UCC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCC $< -o $(@F) + +cgemmt_LNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DNN $< -o $(@F) + +cgemmt_LNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DNT $< -o $(@F) + +cgemmt_LNR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DNR $< -o $(@F) + +cgemmt_LNC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DNC $< -o $(@F) + +cgemmt_LTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DTN $< -o $(@F) + +cgemmt_LTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DTT $< -o $(@F) + +cgemmt_LTR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DTR $< -o $(@F) + +cgemmt_LTC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DTC $< -o $(@F) + +cgemmt_LRN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DRN $< -o $(@F) + +cgemmt_LRT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DRT $< -o $(@F) + +cgemmt_LRR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DRR $< -o $(@F) + +cgemmt_LRC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DRC $< -o $(@F) + +cgemmt_LCN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCN $< -o $(@F) + +cgemmt_LCT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCT $< -o $(@F) + +cgemmt_LCR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCR $< -o $(@F) + +cgemmt_LCC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCC $< -o $(@F) + +cgemmt_kernel_UCN.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -UDOUBLE -DCOMPLEX -ULOWER -DCONJA -UCONJB $< -o $(@F) + +cgemmt_kernel_UNC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -UDOUBLE -DCOMPLEX -ULOWER -UCONJA -DCONJB $< -o $(@F) + +cgemmt_kernel_UCC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -UDOUBLE -DCOMPLEX -ULOWER -DCONJA -DCONJB $< -o $(@F) + +cgemmt_kernel_LCN.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -UDOUBLE -DCOMPLEX -DLOWER -DCONJA -UCONJB $< -o $(@F) + +cgemmt_kernel_LNC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -UDOUBLE -DCOMPLEX -DLOWER -UCONJA -DCONJB $< -o $(@F) + +cgemmt_kernel_LCC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -UDOUBLE -DCOMPLEX -DLOWER -DCONJA -DCONJB $< -o $(@F) + +zgemmt_UNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DNN $< -o $(@F) + +zgemmt_UNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DNT $< -o $(@F) + +zgemmt_UNR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DNR $< -o $(@F) + +zgemmt_UNC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DNC $< -o $(@F) + +zgemmt_UTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DTN $< -o $(@F) + +zgemmt_UTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DTT $< -o $(@F) + +zgemmt_UTR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DTR $< -o $(@F) + +zgemmt_UTC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DTC $< -o $(@F) + +zgemmt_URN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DRN $< -o $(@F) + +zgemmt_URT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DRT $< -o $(@F) + +zgemmt_URR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DRR $< -o $(@F) + +zgemmt_URC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DRC $< -o $(@F) + +zgemmt_UCN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCN $< -o $(@F) + +zgemmt_UCT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCT $< -o $(@F) + +zgemmt_UCR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCR $< -o $(@F) + +zgemmt_UCC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCC $< -o $(@F) + +zgemmt_LNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DNN $< -o $(@F) + +zgemmt_LNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DNT $< -o $(@F) + +zgemmt_LNR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DNR $< -o $(@F) + +zgemmt_LNC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DNC $< -o $(@F) + +zgemmt_LTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DTN $< -o $(@F) + +zgemmt_LTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DTT $< -o $(@F) + +zgemmt_LTR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DTR $< -o $(@F) + +zgemmt_LTC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DTC $< -o $(@F) + +zgemmt_LRN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DRN $< -o $(@F) + +zgemmt_LRT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DRT $< -o $(@F) + +zgemmt_LRR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DRR $< -o $(@F) + +zgemmt_LRC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DRC $< -o $(@F) + +zgemmt_LCN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCN $< -o $(@F) + +zgemmt_LCT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCT $< -o $(@F) + +zgemmt_LCR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCR $< -o $(@F) + +zgemmt_LCC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCC $< -o $(@F) + +zgemmt_kernel_UCN.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DDOUBLE -DCOMPLEX -ULOWER -DCONJA -UCONJB $< -o $(@F) + +zgemmt_kernel_UNC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DDOUBLE -DCOMPLEX -ULOWER -UCONJA -DCONJB $< -o $(@F) + +zgemmt_kernel_UCC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DDOUBLE -DCOMPLEX -ULOWER -DCONJA -DCONJB $< -o $(@F) + +zgemmt_kernel_LCN.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DDOUBLE -DCOMPLEX -DLOWER -DCONJA -UCONJB $< -o $(@F) + +zgemmt_kernel_LNC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DDOUBLE -DCOMPLEX -DLOWER -UCONJA -DCONJB $< -o $(@F) + +zgemmt_kernel_LCC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DDOUBLE -DCOMPLEX -DLOWER -DCONJA -DCONJB $< -o $(@F) + +xgemmt_UNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DNN $< -o $(@F) + +xgemmt_UNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DNT $< -o $(@F) + +xgemmt_UNR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DNR $< -o $(@F) + +xgemmt_UNC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DNC $< -o $(@F) + +xgemmt_UTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DTN $< -o $(@F) + +xgemmt_UTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DTT $< -o $(@F) + +xgemmt_UTR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DTR $< -o $(@F) + +xgemmt_UTC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DTC $< -o $(@F) + +xgemmt_URN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DRN $< -o $(@F) + +xgemmt_URT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DRT $< -o $(@F) + +xgemmt_URR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DRR $< -o $(@F) + +xgemmt_URC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DRC $< -o $(@F) + +xgemmt_UCN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCN $< -o $(@F) + +xgemmt_UCT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCT $< -o $(@F) + +xgemmt_UCR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCR $< -o $(@F) + +xgemmt_UCC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCC $< -o $(@F) + +xgemmt_LNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DNN $< -o $(@F) + +xgemmt_LNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DNT $< -o $(@F) + +xgemmt_LNR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DNR $< -o $(@F) + +xgemmt_LNC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DNC $< -o $(@F) + +xgemmt_LTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DTN $< -o $(@F) + +xgemmt_LTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DTT $< -o $(@F) + +xgemmt_LTR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DTR $< -o $(@F) + +xgemmt_LTC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DTC $< -o $(@F) + +xgemmt_LRN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DRN $< -o $(@F) + +xgemmt_LRT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DRT $< -o $(@F) + +xgemmt_LRR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DRR $< -o $(@F) + +xgemmt_LRC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DRC $< -o $(@F) + +xgemmt_LCN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCN $< -o $(@F) + +xgemmt_LCT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCT $< -o $(@F) + +xgemmt_LCR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCR $< -o $(@F) + +xgemmt_LCC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCC $< -o $(@F) + +xgemmt_kernel_UCN.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DXDOUBLE -DCOMPLEX -ULOWER -DCONJA -UCONJB $< -o $(@F) + +xgemmt_kernel_UNC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DXDOUBLE -DCOMPLEX -ULOWER -UCONJA -DCONJB $< -o $(@F) + +xgemmt_kernel_UCC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DXDOUBLE -DCOMPLEX -ULOWER -DCONJA -DCONJB $< -o $(@F) + +xgemmt_kernel_LCN.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DXDOUBLE -DCOMPLEX -DLOWER -DCONJA -UCONJB $< -o $(@F) + +xgemmt_kernel_LNC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DXDOUBLE -DCOMPLEX -DLOWER -UCONJA -DCONJB $< -o $(@F) + +xgemmt_kernel_LCC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DXDOUBLE -DCOMPLEX -DLOWER -DCONJA -DCONJB $< -o $(@F) + syrk_thread.$(SUFFIX) : syrk_thread.c ../../common.h $(CC) -c $(CFLAGS) $< -o $(@F) @@ -4022,6 +4488,421 @@ xsyrk_kernel_U.$(PSUFFIX) : syrk_kernel.c xsyrk_kernel_L.$(PSUFFIX) : syrk_kernel.c $(CC) -c $(PFLAGS) -DXDOUBLE -DCOMPLEX -DLOWER $< -o $(@F) + +sgemmt_UNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -ULOWER -DNN $< -o $(@F) + +sgemmt_UNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -ULOWER -DNT $< -o $(@F) + +sgemmt_UTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -ULOWER -DTN $< -o $(@F) + +sgemmt_UTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -ULOWER -DTT $< -o $(@F) + +sgemmt_LNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -DLOWER -DNN $< -o $(@F) + +sgemmt_LNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -DLOWER -DNT $< -o $(@F) + +sgemmt_LTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -DLOWER -DTN $< -o $(@F) + +sgemmt_LTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -DLOWER -DTT $< -o $(@F) + +dgemmt_UNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -ULOWER -DNN $< -o $(@F) + +dgemmt_UNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -ULOWER -DNT $< -o $(@F) + +dgemmt_UTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -ULOWER -DTN $< -o $(@F) + +dgemmt_UTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -ULOWER -DTT $< -o $(@F) + +dgemmt_LNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -DLOWER -DNN $< -o $(@F) + +dgemmt_LNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -DLOWER -DNT $< -o $(@F) + +dgemmt_LTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -DLOWER -DTN $< -o $(@F) + +dgemmt_LTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -DLOWER -DTT $< -o $(@F) + +qgemmt_UNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -ULOWER -DNN $< -o $(@F) + +qgemmt_UNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -ULOWER -DNT $< -o $(@F) + +qgemmt_UTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -ULOWER -DTN $< -o $(@F) + +qgemmt_UTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -ULOWER -DTT $< -o $(@F) + +qgemmt_LNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -DLOWER -DNN $< -o $(@F) + +qgemmt_LNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -DLOWER -DNT $< -o $(@F) + +qgemmt_LTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -DLOWER -DTN $< -o $(@F) + +qgemmt_LTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -DLOWER -DTT $< -o $(@F) + +cgemmt_UNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DNN $< -o $(@F) + +cgemmt_UNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DNT $< -o $(@F) + +cgemmt_UNR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DNR $< -o $(@F) + +cgemmt_UNC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DNC $< -o $(@F) + +cgemmt_UTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DTN $< -o $(@F) + +cgemmt_UTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DTT $< -o $(@F) + +cgemmt_UTR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DTR $< -o $(@F) + +cgemmt_UTC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DTC $< -o $(@F) + +cgemmt_URN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DRN $< -o $(@F) + +cgemmt_URT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DRT $< -o $(@F) + +cgemmt_URR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DRR $< -o $(@F) + +cgemmt_URC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DRC $< -o $(@F) + +cgemmt_UCN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCN $< -o $(@F) + +cgemmt_UCT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCT $< -o $(@F) + +cgemmt_UCR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCR $< -o $(@F) + +cgemmt_UCC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCC $< -o $(@F) + +cgemmt_LNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DNN $< -o $(@F) + +cgemmt_LNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DNT $< -o $(@F) + +cgemmt_LNR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DNR $< -o $(@F) + +cgemmt_LNC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DNC $< -o $(@F) + +cgemmt_LTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DTN $< -o $(@F) + +cgemmt_LTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DTT $< -o $(@F) + +cgemmt_LTR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DTR $< -o $(@F) + +cgemmt_LTC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DTC $< -o $(@F) + +cgemmt_LRN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DRN $< -o $(@F) + +cgemmt_LRT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DRT $< -o $(@F) + +cgemmt_LRR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DRR $< -o $(@F) + +cgemmt_LRC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DRC $< -o $(@F) + +cgemmt_LCN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCN $< -o $(@F) + +cgemmt_LCT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCT $< -o $(@F) + +cgemmt_LCR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCR $< -o $(@F) + +cgemmt_LCC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCC $< -o $(@F) + +cgemmt_kernel_UCN.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -UDOUBLE -DCOMPLEX -ULOWER -DCONJA -UCONJB $< -o $(@F) + +cgemmt_kernel_UNC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -UDOUBLE -DCOMPLEX -ULOWER -UCONJA -DCONJB $< -o $(@F) + +cgemmt_kernel_UCC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -UDOUBLE -DCOMPLEX -ULOWER -DCONJA -DCONJB $< -o $(@F) + +cgemmt_kernel_LCN.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -UDOUBLE -DCOMPLEX -DLOWER -DCONJA -UCONJB $< -o $(@F) + +cgemmt_kernel_LNC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -UDOUBLE -DCOMPLEX -DLOWER -UCONJA -DCONJB $< -o $(@F) + +cgemmt_kernel_LCC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -UDOUBLE -DCOMPLEX -DLOWER -DCONJA -DCONJB $< -o $(@F) + +zgemmt_UNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DNN $< -o $(@F) + +zgemmt_UNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DNT $< -o $(@F) + +zgemmt_UNR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DNR $< -o $(@F) + +zgemmt_UNC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DNC $< -o $(@F) + +zgemmt_UTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DTN $< -o $(@F) + +zgemmt_UTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DTT $< -o $(@F) + +zgemmt_UTR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DTR $< -o $(@F) + +zgemmt_UTC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DTC $< -o $(@F) + +zgemmt_URN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DRN $< -o $(@F) + +zgemmt_URT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DRT $< -o $(@F) + +zgemmt_URR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DRR $< -o $(@F) + +zgemmt_URC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DRC $< -o $(@F) + +zgemmt_UCN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCN $< -o $(@F) + +zgemmt_UCT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCT $< -o $(@F) + +zgemmt_UCR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCR $< -o $(@F) + +zgemmt_UCC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCC $< -o $(@F) + +zgemmt_LNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DNN $< -o $(@F) + +zgemmt_LNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DNT $< -o $(@F) + +zgemmt_LNR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DNR $< -o $(@F) + +zgemmt_LNC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DNC $< -o $(@F) + +zgemmt_LTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DTN $< -o $(@F) + +zgemmt_LTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DTT $< -o $(@F) + +zgemmt_LTR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DTR $< -o $(@F) + +zgemmt_LTC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DTC $< -o $(@F) + +zgemmt_LRN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DRN $< -o $(@F) + +zgemmt_LRT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DRT $< -o $(@F) + +zgemmt_LRR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DRR $< -o $(@F) + +zgemmt_LRC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DRC $< -o $(@F) + +zgemmt_LCN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCN $< -o $(@F) + +zgemmt_LCT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCT $< -o $(@F) + +zgemmt_LCR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCR $< -o $(@F) + +zgemmt_LCC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCC $< -o $(@F) + +zgemmt_kernel_UCN.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DDOUBLE -DCOMPLEX -ULOWER -DCONJA -UCONJB $< -o $(@F) + +zgemmt_kernel_UNC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DDOUBLE -DCOMPLEX -ULOWER -UCONJA -DCONJB $< -o $(@F) + +zgemmt_kernel_UCC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DDOUBLE -DCOMPLEX -ULOWER -DCONJA -DCONJB $< -o $(@F) + +zgemmt_kernel_LCN.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DDOUBLE -DCOMPLEX -DLOWER -DCONJA -UCONJB $< -o $(@F) + +zgemmt_kernel_LNC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DDOUBLE -DCOMPLEX -DLOWER -UCONJA -DCONJB $< -o $(@F) + +zgemmt_kernel_LCC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DDOUBLE -DCOMPLEX -DLOWER -DCONJA -DCONJB $< -o $(@F) + +xgemmt_UNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DNN $< -o $(@F) + +xgemmt_UNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DNT $< -o $(@F) + +xgemmt_UNR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DNR $< -o $(@F) + +xgemmt_UNC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DNC $< -o $(@F) + +xgemmt_UTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DTN $< -o $(@F) + +xgemmt_UTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DTT $< -o $(@F) + +xgemmt_UTR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DTR $< -o $(@F) + +xgemmt_UTC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DTC $< -o $(@F) + +xgemmt_URN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DRN $< -o $(@F) + +xgemmt_URT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DRT $< -o $(@F) + +xgemmt_URR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DRR $< -o $(@F) + +xgemmt_URC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DRC $< -o $(@F) + +xgemmt_UCN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCN $< -o $(@F) + +xgemmt_UCT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCT $< -o $(@F) + +xgemmt_UCR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCR $< -o $(@F) + +xgemmt_UCC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCC $< -o $(@F) + +xgemmt_LNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DNN $< -o $(@F) + +xgemmt_LNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DNT $< -o $(@F) + +xgemmt_LNR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DNR $< -o $(@F) + +xgemmt_LNC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DNC $< -o $(@F) + +xgemmt_LTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DTN $< -o $(@F) + +xgemmt_LTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DTT $< -o $(@F) + +xgemmt_LTR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DTR $< -o $(@F) + +xgemmt_LTC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DTC $< -o $(@F) + +xgemmt_LRN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DRN $< -o $(@F) + +xgemmt_LRT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DRT $< -o $(@F) + +xgemmt_LRR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DRR $< -o $(@F) + +xgemmt_LRC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DRC $< -o $(@F) + +xgemmt_LCN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCN $< -o $(@F) + +xgemmt_LCT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCT $< -o $(@F) + +xgemmt_LCR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCR $< -o $(@F) + +xgemmt_LCC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCC $< -o $(@F) + +xgemmt_kernel_UCN.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DXDOUBLE -DCOMPLEX -ULOWER -DCONJA -UCONJB $< -o $(@F) + +xgemmt_kernel_UNC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DXDOUBLE -DCOMPLEX -ULOWER -UCONJA -DCONJB $< -o $(@F) + +xgemmt_kernel_UCC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DXDOUBLE -DCOMPLEX -ULOWER -DCONJA -DCONJB $< -o $(@F) + +xgemmt_kernel_LCN.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DXDOUBLE -DCOMPLEX -DLOWER -DCONJA -UCONJB $< -o $(@F) + +xgemmt_kernel_LNC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DXDOUBLE -DCOMPLEX -DLOWER -UCONJA -DCONJB $< -o $(@F) + +xgemmt_kernel_LCC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DXDOUBLE -DCOMPLEX -DLOWER -DCONJA -DCONJB $< -o $(@F) + syrk_thread.$(PSUFFIX) : syrk_thread.c ../../common.h $(CC) -c $(PFLAGS) $< -o $(@F) diff --git a/driver/level3/gemmt_k.c b/driver/level3/gemmt_k.c new file mode 100644 index 0000000000..e94168bf6c --- /dev/null +++ b/driver/level3/gemmt_k.c @@ -0,0 +1,127 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ + +#include +#include "common.h" + +/* Scale the part of C that GEMMT actually references by beta. This is + the same operation as syrk_beta() in syrk_k.c. */ +static __inline int gemmt_beta(BLASLONG m_from, BLASLONG m_to, BLASLONG n_from, BLASLONG n_to, + FLOAT *alpha, FLOAT *c, BLASLONG ldc) { + + BLASLONG i; + +#ifndef LOWER + if (m_from > n_from) n_from = m_from; + if (m_to > n_to ) m_to = n_to; +#else + if (m_from < n_from) m_from = n_from; + if (m_to < n_to ) n_to = m_to; +#endif + + if ((m_to <= m_from) || (n_to <= n_from)) return 0; + + c += (m_from + n_from * ldc) * COMPSIZE; + + m_to -= m_from; + n_to -= n_from; + + for (i = 0; i < n_to; i++){ + +#ifndef LOWER + + SCAL_K(MIN(i + n_from - m_from + 1, m_to), 0, 0, alpha[0], +#ifdef COMPLEX + alpha[1], +#endif + c, 1, NULL, 0, NULL, 0); + + c += ldc * COMPSIZE; + +#else + + SCAL_K(MIN(m_to - i + m_from - n_from, m_to), 0, 0, alpha[0], +#ifdef COMPLEX + alpha[1], +#endif + c, 1, NULL, 0, NULL, 0); + + if (i < m_from - n_from) { + c += ldc * COMPSIZE; + } else { + c += (1 + ldc) * COMPSIZE; + } +#endif + + } + + return 0; +} + +#define GEMMT_BETA(M_FROM, M_TO, N_FROM, N_TO, BETA, C, LDC) \ + gemmt_beta(M_FROM, M_TO, N_FROM, N_TO, BETA, C, LDC) + +/* The blocks on the diagonal are computed by the SYRK kernel, which + clips a block against the diagonal; the conjugation variants needed + for complex GEMMT are the same file built with CONJA / CONJB. */ +#ifndef KERNEL_FUNC +#ifndef LOWER +#if defined(NN) || defined(NT) || defined(TN) || defined(TT) +#define KERNEL_FUNC SYRK_KERNEL_U +#elif defined(CN) || defined(CT) || defined(RN) || defined(RT) +#define KERNEL_FUNC GEMMT_KERNEL_UCN +#elif defined(NC) || defined(TC) || defined(NR) || defined(TR) +#define KERNEL_FUNC GEMMT_KERNEL_UNC +#else +#define KERNEL_FUNC GEMMT_KERNEL_UCC +#endif +#else +#if defined(NN) || defined(NT) || defined(TN) || defined(TT) +#define KERNEL_FUNC SYRK_KERNEL_L +#elif defined(CN) || defined(CT) || defined(RN) || defined(RT) +#define KERNEL_FUNC GEMMT_KERNEL_LCN +#elif defined(NC) || defined(TC) || defined(NR) || defined(TR) +#define KERNEL_FUNC GEMMT_KERNEL_LNC +#else +#define KERNEL_FUNC GEMMT_KERNEL_LCC +#endif +#endif +#endif + +/* GEMMT is threaded by splitting the triangle into equal-area column + ranges (syrk_thread) and running this driver on each of them, so there + is no separate THREADED_LEVEL3 variant. */ +#include "level3_gemmt.c" diff --git a/driver/level3/level3_gemmt.c b/driver/level3/level3_gemmt.c new file mode 100644 index 0000000000..ad42228cad --- /dev/null +++ b/driver/level3/level3_gemmt.c @@ -0,0 +1,508 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#ifndef KERNEL_OPERATION +#ifndef COMPLEX +#define KERNEL_OPERATION(M, N, K, ALPHA, SA, SB, C, LDC, X, Y) \ + KERNEL_FUNC(M, N, K, ALPHA[0], SA, SB, (FLOAT *)(C) + ((X) + (Y) * LDC) * COMPSIZE, LDC, (X) - (Y)) +#else +#define KERNEL_OPERATION(M, N, K, ALPHA, SA, SB, C, LDC, X, Y) \ + KERNEL_FUNC(M, N, K, ALPHA[0], ALPHA[1], SA, SB, (FLOAT *)(C) + ((X) + (Y) * LDC) * COMPSIZE, LDC, (X) - (Y)) +#endif +#endif + +#ifndef ICOPY_OPERATION +#if defined(NN) || defined(NT) || defined(NC) || defined(NR) || \ + defined(RN) || defined(RT) || defined(RC) || defined(RR) +#define ICOPY_OPERATION(M, N, A, LDA, X, Y, BUFFER) GEMM_ITCOPY(M, N, (FLOAT *)(A) + ((Y) + (X) * (LDA)) * COMPSIZE, LDA, BUFFER); +#else +#define ICOPY_OPERATION(M, N, A, LDA, X, Y, BUFFER) GEMM_INCOPY(M, N, (FLOAT *)(A) + ((X) + (Y) * (LDA)) * COMPSIZE, LDA, BUFFER); +#endif +#endif + +#ifndef OCOPY_OPERATION +#if defined(NN) || defined(TN) || defined(CN) || defined(RN) || \ + defined(NR) || defined(TR) || defined(CR) || defined(RR) +#define OCOPY_OPERATION(M, N, A, LDA, X, Y, BUFFER) GEMM_ONCOPY(M, N, (FLOAT *)(A) + ((X) + (Y) * (LDA)) * COMPSIZE, LDA, BUFFER); +#else +#define OCOPY_OPERATION(M, N, A, LDA, X, Y, BUFFER) GEMM_OTCOPY(M, N, (FLOAT *)(A) + ((Y) + (X) * (LDA)) * COMPSIZE, LDA, BUFFER); +#endif +#endif + +#ifndef M +#define M args -> n +#endif + +#ifndef N +#define N args -> n +#endif + +#ifndef K +#define K args -> k +#endif + +#ifndef A +#define A args -> a +#endif + +#ifndef B +#define B args -> b +#endif + +#ifndef C +#define C args -> c +#endif + +#ifndef LDA +#define LDA args -> lda +#endif + +#ifndef LDB +#define LDB args -> ldb +#endif + +#ifndef LDC +#define LDC args -> ldc +#endif + +#ifdef TIMING +#define START_RPCC() rpcc_counter = rpcc() +#define STOP_RPCC(COUNTER) COUNTER += rpcc() - rpcc_counter +#else +#define START_RPCC() +#define STOP_RPCC(COUNTER) +#endif + +/* Single-threaded GEMMT / GEMMTR driver: + + C := alpha * op(A) * op(B) + beta * C + + with only the triangle of the n by n matrix C selected by LOWER + referenced and updated. Derived from level3_syrk.c: the blocking, + the diagonal handling (SYRK_KERNEL clips the blocks that cross the + diagonal) and the loop structure are the same, but the packed B + buffer is filled from a second matrix instead of from A, so the + buffer sharing that SYRK can do is not available here. */ + +int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLOAT *sb, BLASLONG dummy) { + + BLASLONG m_from, m_to, n_from, n_to, k, lda, ldb, ldc; + FLOAT *a, *b, *c, *alpha, *beta; + + BLASLONG ls, is, js; + BLASLONG min_l, min_i, min_j; + BLASLONG jjs, min_jj; + BLASLONG m_start, m_end; + +#ifdef LOWER + FLOAT *aa; +#endif + +#ifdef TIMING + unsigned long long rpcc_counter; + unsigned long long innercost = 0; + unsigned long long outercost = 0; + unsigned long long kernelcost = 0; + double total; +#endif + + k = K; + + a = (FLOAT *)A; + b = (FLOAT *)B; + c = (FLOAT *)C; + + lda = LDA; + ldb = LDB; + ldc = LDC; + + alpha = (FLOAT *)args -> alpha; + beta = (FLOAT *)args -> beta; + + m_from = 0; + m_to = M; + + if (range_m) { + m_from = *(((BLASLONG *)range_m) + 0); + m_to = *(((BLASLONG *)range_m) + 1); + } + + n_from = 0; + n_to = N; + + if (range_n) { + n_from = *(((BLASLONG *)range_n) + 0); + n_to = *(((BLASLONG *)range_n) + 1); + } + + if (beta) { +#ifndef COMPLEX + if (beta[0] != ONE) +#else + if ((beta[0] != ONE) || (beta[1] != ZERO)) +#endif + GEMMT_BETA(m_from, m_to, n_from, n_to, beta, c, ldc); + } + + if ((k == 0) || (alpha == NULL)) return 0; + + if (alpha[0] == ZERO +#if defined(COMPLEX) && !defined(HERK) + && alpha[1] == ZERO +#endif + ) return 0; + +#if 0 + fprintf(stderr, "m_from : %ld m_to : %ld n_from : %ld n_to : %ld\n", + m_from, m_to, n_from, n_to); +#endif + + for(js = n_from; js < n_to; js += GEMM_R){ + min_j = n_to - js; + if (min_j > GEMM_R) min_j = GEMM_R; + +#ifndef LOWER + m_start = m_from; + m_end = js + min_j; + if (m_end > m_to) m_end = m_to; +#else + m_start = m_from; + m_end = m_to; + if (m_start < js) m_start = js; +#endif + + for(ls = 0; ls < k; ls += min_l){ + min_l = k - ls; + if (min_l >= GEMM_Q * 2) { + min_l = GEMM_Q; + } else + if (min_l > GEMM_Q) { + min_l = (min_l + 1) / 2; + } + + min_i = m_end - m_start; + + if (min_i >= GEMM_P * 2) { + min_i = GEMM_P; + } else + if (min_i > GEMM_P) { + min_i = ((min_i / 2 + GEMM_UNROLL_MN - 1)/GEMM_UNROLL_MN) * GEMM_UNROLL_MN; + } + +#ifndef LOWER + + if (m_end >= js) { + + /* A is packed in the same GEMM_UNROLL_MN chunks in which the kernel + consumes it: a block that crosses the diagonal is cut down to + min_jj rows there, and the packed layout of a panel depends on + how many rows were handed to the copy routine. Columns to the + left of this row band are below the diagonal and stay unpacked; + the row bands further down skip them via the kernel offset. */ + for(jjs = MAX(m_start, js); jjs < js + min_j; jjs += min_jj){ + min_jj = js + min_j - jjs; + if (min_jj > GEMM_UNROLL_MN) min_jj = GEMM_UNROLL_MN; + + if (jjs - MAX(m_start, js) < min_i) { + START_RPCC(); + + ICOPY_OPERATION(min_l, min_jj, a, lda, ls, jjs, sa + min_l * (jjs - MAX(m_start, js)) * COMPSIZE); + + STOP_RPCC(innercost); + } + + START_RPCC(); + + OCOPY_OPERATION(min_l, min_jj, b, ldb, ls, jjs, sb + min_l * (jjs - js) * COMPSIZE); + + STOP_RPCC(outercost); + + START_RPCC(); + + KERNEL_OPERATION(min_i, min_jj, min_l, alpha, sa, sb + min_l * (jjs - js) * COMPSIZE, c, ldc, MAX(m_start, js), jjs); + + STOP_RPCC(kernelcost); + } + + for(is = MAX(m_start, js) + min_i; is < m_end; is += min_i){ + min_i = m_end - is; + if (min_i >= GEMM_P * 2) { + min_i = GEMM_P; + } else + if (min_i > GEMM_P) { + min_i = ((min_i / 2 + GEMM_UNROLL_MN - 1)/GEMM_UNROLL_MN) * GEMM_UNROLL_MN; + } + + START_RPCC(); + + ICOPY_OPERATION(min_l, min_i, a, lda, ls, is, sa); + + STOP_RPCC(innercost); + + START_RPCC(); + + KERNEL_OPERATION(min_i, min_j, min_l, alpha, sa, sb, c, ldc, is, js); + + STOP_RPCC(kernelcost); + + } + + } + + if (m_start < js) { + + if (m_end < js) { + + START_RPCC(); + + ICOPY_OPERATION(min_l, min_i, a, lda, ls, m_start, sa); + + STOP_RPCC(innercost); + + for(jjs = js; jjs < js + min_j; jjs += GEMM_UNROLL_MN){ + min_jj = min_j + js - jjs; + if (min_jj > GEMM_UNROLL_MN) min_jj = GEMM_UNROLL_MN; + + START_RPCC(); + + OCOPY_OPERATION(min_l, min_jj, b, ldb, ls, jjs, sb + min_l * (jjs - js) * COMPSIZE); + + STOP_RPCC(outercost); + + START_RPCC(); + + KERNEL_OPERATION(min_i, min_jj, min_l, alpha, sa, sb + min_l * (jjs - js) * COMPSIZE, c, ldc, m_start, jjs); + + STOP_RPCC(kernelcost); + + } + } else { + min_i = 0; + } + + for(is = m_start + min_i; is < MIN(m_end, js); is += min_i){ + + min_i = MIN(m_end, js)- is; + if (min_i >= GEMM_P * 2) { + min_i = GEMM_P; + } else + if (min_i > GEMM_P) { + min_i = ((min_i / 2 + GEMM_UNROLL_MN - 1)/GEMM_UNROLL_MN) * GEMM_UNROLL_MN; + } + + START_RPCC(); + + ICOPY_OPERATION(min_l, min_i, a, lda, ls, is, sa); + + STOP_RPCC(innercost); + + START_RPCC(); + + KERNEL_OPERATION(min_i, min_j, min_l, alpha, sa, sb, c, ldc, is, js); + + STOP_RPCC(kernelcost); + + } + } + +#else + + if (m_start < js + min_j) { + + aa = sb + min_l * (m_start - js) * COMPSIZE; + + START_RPCC(); + + ICOPY_OPERATION(min_l, min_i, a, lda, ls, m_start, sa); + + STOP_RPCC(innercost); + + START_RPCC(); + + OCOPY_OPERATION(min_l, MIN(min_i, min_j + js - m_start), b, ldb, ls, m_start, aa); + + STOP_RPCC(outercost); + + START_RPCC(); + + KERNEL_OPERATION(min_i, MIN(min_i, min_j + js - m_start), min_l, alpha, sa, aa, c, ldc, m_start, m_start); + + STOP_RPCC(kernelcost); + + for(jjs = js; jjs < m_start; jjs += GEMM_UNROLL_N){ + min_jj = m_start - jjs; + if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N; + + START_RPCC(); + + OCOPY_OPERATION(min_l, min_jj, b, ldb, ls, jjs, sb + min_l * (jjs - js) * COMPSIZE); + + STOP_RPCC(outercost); + + START_RPCC(); + + KERNEL_OPERATION(min_i, min_jj, min_l, alpha, sa, sb + min_l * (jjs - js) * COMPSIZE, c, ldc, m_start, jjs); + + STOP_RPCC(kernelcost); + + } + + for(is = m_start + min_i; is < m_end; is += min_i){ + + min_i = m_end - is; + + if (min_i >= GEMM_P * 2) { + min_i = GEMM_P; + } else + if (min_i > GEMM_P) { + min_i = ((min_i / 2 + GEMM_UNROLL_MN - 1)/GEMM_UNROLL_MN) * GEMM_UNROLL_MN; + } + + if (is < js + min_j) { + + START_RPCC(); + + ICOPY_OPERATION(min_l, min_i, a, lda, ls, is, sa); + + STOP_RPCC(innercost); + + aa = sb + min_l * (is - js) * COMPSIZE; + + START_RPCC(); + + OCOPY_OPERATION(min_l, MIN(min_i, min_j - is + js), b, ldb, ls, is, aa); + + STOP_RPCC(outercost); + + START_RPCC(); + + KERNEL_OPERATION(min_i, MIN(min_i, min_j - is + js), min_l, alpha, sa, aa, c, ldc, is, is); + + STOP_RPCC(kernelcost); + + START_RPCC(); + + KERNEL_OPERATION(min_i, is - js, min_l, alpha, sa, sb, c, ldc, is, js); + + STOP_RPCC(kernelcost); + + } else { + + START_RPCC(); + + ICOPY_OPERATION(min_l, min_i, a, lda, ls, is, sa); + + STOP_RPCC(innercost); + + START_RPCC(); + + KERNEL_OPERATION(min_i, min_j, min_l, alpha, sa, sb, c, ldc, is, js); + + STOP_RPCC(kernelcost); + + } + + } + + } else { + + START_RPCC(); + + ICOPY_OPERATION(min_l, min_i, a, lda, ls, m_start, sa); + + STOP_RPCC(innercost); + + for(jjs = js; jjs < js + min_j; jjs += GEMM_UNROLL_N){ + min_jj = js + min_j - jjs; + if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N; + + START_RPCC(); + + OCOPY_OPERATION(min_l, min_jj, b, ldb, ls, jjs, sb + min_l * (jjs - js) * COMPSIZE); + + STOP_RPCC(outercost); + + START_RPCC(); + + KERNEL_OPERATION(min_i, min_jj, min_l, alpha, sa, sb + min_l * (jjs - js) * COMPSIZE, c, ldc, m_start, jjs); + + STOP_RPCC(kernelcost); + + } + + for(is = m_start + min_i; is < m_end; is += min_i){ + + min_i = m_end - is; + + if (min_i >= GEMM_P * 2) { + min_i = GEMM_P; + } else + if (min_i > GEMM_P) { + min_i = ((min_i / 2 + GEMM_UNROLL_MN - 1)/GEMM_UNROLL_MN) * GEMM_UNROLL_MN; + } + + START_RPCC(); + + ICOPY_OPERATION(min_l, min_i, a, lda, ls, is, sa); + + STOP_RPCC(innercost); + + START_RPCC(); + + KERNEL_OPERATION(min_i, min_j, min_l, alpha, sa, sb, c, ldc, is, js); + + STOP_RPCC(kernelcost); + + } + } +#endif + } + } + +#ifdef TIMING + total = (double)outercost + (double)innercost + (double)kernelcost; + + printf( "Copy A : %5.2f Copy B: %5.2f Kernel : %5.2f kernel Effi. : %5.2f Total Effi. : %5.2f\n", + innercost / total * 100., outercost / total * 100., kernelcost / total * 100., + (double)(m_to - m_from) * (double)(n_to - n_from) * (double)k / (double)kernelcost * 100. * (double)COMPSIZE / (double)DNUMOPT, + (double)(m_to - m_from) * (double)(n_to - n_from) * (double)k / total * 100. * (double)COMPSIZE / (double)DNUMOPT); + +#endif + + return 0; +} diff --git a/interface/gemmt.c b/interface/gemmt.c index 39f41ae1c2..847d36ad3d 100644 --- a/interface/gemmt.c +++ b/interface/gemmt.c @@ -102,15 +102,7 @@ void NAME(char *UPLO, char *TRANSA, char *TRANSB, char transA, transB, Uplo; blasint nrowa, nrowb; -#if defined(COMPLEX) - blasint ncolb; -#endif - IFLOAT *buffer; - IFLOAT *aa, *bb; - FLOAT *cc; -#if defined(COMPLEX) - FLOAT alpha_r, alpha_i, beta_r, beta_i; -#else +#if !defined(COMPLEX) FLOAT alpha, beta; #endif @@ -121,11 +113,7 @@ void NAME(char *UPLO, char *TRANSA, char *TRANSB, #if defined(COMPLEX) FLOAT *alpha = Alpha; - alpha_r = *(Alpha + 0); - alpha_i = *(Alpha + 1); - - beta_r = *(Beta + 0); - beta_i = *(Beta + 1); + FLOAT *beta = Beta; #else alpha = *Alpha; beta = *Beta; @@ -186,15 +174,7 @@ void NAME(char *UPLO, char *TRANSA, char *TRANSB, nrowa = m; if (transa & 1) nrowa = k; nrowb = k; -#if defined(COMPLEX) - ncolb = m; -#endif - if (transb & 1) { - nrowb = m; -#if defined(COMPLEX) - ncolb = k; -#endif - } + if (transb & 1) nrowb = m; info = 0; @@ -240,16 +220,10 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_UPLO Uplo, FLOAT *B = (FLOAT *) vb; FLOAT *c = (FLOAT *) vc; #endif - FLOAT *aa, *bb, *cc; - int transa, transb, uplo; blasint info; blasint lda, ldb; FLOAT *a, *b; -#if defined(COMPLEX) - blasint nrowb, ncolb; -#endif - XFLOAT *buffer; PRINT_DEBUG_CNAME; @@ -300,22 +274,12 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_UPLO Uplo, info = -1; - blasint nrowa; -#if !defined(COMPLEX) - blasint nrowb; -#endif + blasint nrowa, nrowb; + nrowa = m; if (transa & 1) nrowa = k; nrowb = k; -#if defined(COMPLEX) - ncolb = m; -#endif - if (transb & 1) { - nrowb = m; -#if defined(COMPLEX) - ncolb = k; -#endif - } + if (transb & 1) nrowb = m; if (ldc < MAX(1, m)) info = 13; @@ -379,23 +343,12 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_UPLO Uplo, info = -1; - blasint ncola; -#if !defined(COMPLEX) - blasint ncolb; -#endif + blasint ncola, ncolb; + ncola = m; if (transa & 1) ncola = k; ncolb = k; -#if defined(COMPLEX) - nrowb = m; -#endif - - if (transb & 1) { -#if defined(COMPLEX) - nrowb = k; -#endif - ncolb = m; - } + if (transb & 1) ncolb = m; if (ldc < MAX(1,m)) info = 13; @@ -419,290 +372,129 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_UPLO Uplo, BLASFUNC(xerbla) (ERROR_NAME, &info, sizeof(ERROR_NAME)); return; } -#if defined(COMPLEX) - FLOAT alpha_r = *(alpha + 0); - FLOAT alpha_i = *(alpha + 1); - - FLOAT beta_r = *(beta + 0); - FLOAT beta_i = *(beta + 1); #endif - -#endif - int buffer_size; - blasint l; - blasint i, j; - -#ifdef SMP - int nthreads; +#ifndef COMPLEX +#define GEMMT_TABLE_INDEX ((uplo << 2) | (transb << 1) | transa) +#else +#define GEMMT_TABLE_INDEX ((uplo << 4) | (transb << 2) | transa) #endif -#if defined(COMPLEX) + blas_arg_t args; + FLOAT *buffer, *sa, *sb; #ifdef SMP - static int (*gemv_thread[]) (BLASLONG, BLASLONG, FLOAT *, FLOAT *, - BLASLONG, FLOAT *, BLASLONG, FLOAT *, - BLASLONG, FLOAT *, int) = { + double MMK; +#ifndef COMPLEX #ifdef XDOUBLE - xgemv_thread_n, xgemv_thread_t, xgemv_thread_r, xgemv_thread_c, - xgemv_thread_o, xgemv_thread_u, xgemv_thread_s, - xgemv_thread_d, -#elif defined DOUBLE - zgemv_thread_n, zgemv_thread_t, zgemv_thread_r, zgemv_thread_c, - zgemv_thread_o, zgemv_thread_u, zgemv_thread_s, - zgemv_thread_d, + int mode = BLAS_XDOUBLE | BLAS_REAL; +#elif defined(DOUBLE) + int mode = BLAS_DOUBLE | BLAS_REAL; #else - cgemv_thread_n, cgemv_thread_t, cgemv_thread_r, cgemv_thread_c, - cgemv_thread_o, cgemv_thread_u, cgemv_thread_s, - cgemv_thread_d, -#endif - }; + int mode = BLAS_SINGLE | BLAS_REAL; #endif - - int (*gemv[]) (BLASLONG, BLASLONG, BLASLONG, FLOAT, FLOAT, FLOAT *, - BLASLONG, FLOAT *, BLASLONG, FLOAT *, BLASLONG, - FLOAT *) = { - GEMV_N, GEMV_T, GEMV_R, GEMV_C, GEMV_O, GEMV_U, GEMV_S, GEMV_D,}; - #else - -#ifdef SMP - static int (*gemv_thread[]) (BLASLONG, BLASLONG, FLOAT, FLOAT *, - BLASLONG, FLOAT *, BLASLONG, FLOAT *, - BLASLONG, FLOAT *, int) = { #ifdef XDOUBLE - qgemv_thread_n, qgemv_thread_t, -#elif defined DOUBLE - dgemv_thread_n, dgemv_thread_t, + int mode = BLAS_XDOUBLE | BLAS_COMPLEX; +#elif defined(DOUBLE) + int mode = BLAS_DOUBLE | BLAS_COMPLEX; #else - sgemv_thread_n, sgemv_thread_t, + int mode = BLAS_SINGLE | BLAS_COMPLEX; +#endif #endif - }; #endif - int (*gemv[]) (BLASLONG, BLASLONG, BLASLONG, FLOAT, FLOAT *, BLASLONG, - FLOAT *, BLASLONG, FLOAT *, BLASLONG, FLOAT *) = { - GEMV_N, GEMV_T,}; + /* [uplo][op(B)][op(A)], as in interface/gemm.c */ + static int (*gemmt[])(blas_arg_t *, BLASLONG *, BLASLONG *, FLOAT *, FLOAT *, BLASLONG) = { +#ifndef COMPLEX + GEMMT_UNN, GEMMT_UTN, + GEMMT_UNT, GEMMT_UTT, + GEMMT_LNN, GEMMT_LTN, + GEMMT_LNT, GEMMT_LTT, +#else + GEMMT_UNN, GEMMT_UTN, GEMMT_URN, GEMMT_UCN, + GEMMT_UNT, GEMMT_UTT, GEMMT_URT, GEMMT_UCT, + GEMMT_UNR, GEMMT_UTR, GEMMT_URR, GEMMT_UCR, + GEMMT_UNC, GEMMT_UTC, GEMMT_URC, GEMMT_UCC, + GEMMT_LNN, GEMMT_LTN, GEMMT_LRN, GEMMT_LCN, + GEMMT_LNT, GEMMT_LTT, GEMMT_LRT, GEMMT_LCT, + GEMMT_LNR, GEMMT_LTR, GEMMT_LRR, GEMMT_LCR, + GEMMT_LNC, GEMMT_LTC, GEMMT_LRC, GEMMT_LCC, #endif + }; if (m == 0) return; IDEBUG_START; -#if defined(COMPLEX) - if (transb > 1){ -#ifndef CBLAS - IMATCOPY_K_CNC(nrowb, ncolb, (FLOAT)(1.0), (FLOAT)(0.0), b, ldb); -#else - if (order == CblasColMajor) - IMATCOPY_K_CNC(nrowb, ncolb, (FLOAT)(1.0), (FLOAT)(0.0), b, ldb); - if (order == CblasRowMajor) - IMATCOPY_K_RNC(nrowb, ncolb, (FLOAT)(1.0), (FLOAT)(0.0), b, ldb); -#endif - } -#endif + FUNCTION_PROFILE_START(); - const blasint incb = ((transb & 1) == 0) ? 1 : ldb; + args.m = m; + args.n = m; + args.k = k; - if (uplo == 1) { - for (i = 0; i < m; i++) { - j = m - i; + args.a = (void *)a; + args.b = (void *)b; + args.c = (void *)c; - l = j; -#if defined(COMPLEX) - aa = a + i * 2; - bb = b + i * ldb * 2; - if (transa & 1) { - aa = a + lda * i * 2; - } - if (transb & 1) - bb = b + i * 2; - cc = c + i * 2 * ldc + i * 2; -#else - aa = a + i; - bb = b + i * ldb; - if (transa & 1) { - aa = a + lda * i; - } - if (transb & 1) - bb = b + i; - cc = c + i * ldc + i; -#endif + args.lda = lda; + args.ldb = ldb; + args.ldc = ldc; #if defined(COMPLEX) - if (beta_r != ONE || beta_i != ZERO) - SCAL_K(l, 0, 0, beta_r, beta_i, cc, 1, NULL, 0, - NULL, 0); - - if (alpha_r == ZERO && alpha_i == ZERO) - continue; + args.alpha = (void *)alpha; + args.beta = (void *)beta; #else - if (beta != ONE) - SCAL_K(l, 0, 0, beta, cc, 1, NULL, 0, NULL, 0); - - if (alpha == ZERO) - continue; -#endif - - IDEBUG_START; - - buffer_size = 2 * (j + k) + 128 / sizeof(FLOAT); -#ifdef WINDOWS_ABI - buffer_size += 160 / sizeof(FLOAT); + args.alpha = (void *)α + args.beta = (void *)β #endif - // for alignment - buffer_size = (buffer_size + 3) & ~3; - STACK_ALLOC(buffer_size, IFLOAT, buffer); #ifdef SMP + args.common = NULL; + args.nthreads = 1; +#endif - if (1L * j * k < 2304L * GEMM_MULTITHREAD_THRESHOLD) - nthreads = 1; - else - nthreads = num_cpu_avail(2); + buffer = (FLOAT *)blas_memory_alloc(0); + if (!buffer) { + info = -999; + BLASFUNC(xerbla) (ERROR_NAME, &info, sizeof(ERROR_NAME)); + return; + } - if (nthreads == 1) { -#endif + sa = (FLOAT *)((BLASLONG)buffer + GEMM_OFFSET_A); + sb = (FLOAT *)(((BLASLONG)sa + ((GEMM_P * GEMM_Q * COMPSIZE * SIZE + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); -#if defined(COMPLEX) - if (!(transa & 1)) - (gemv[(int)transa]) (j, k, 0, alpha_r, alpha_i, - aa, lda, bb, incb, cc, 1, - buffer); - else - (gemv[(int)transa]) (k, j, 0, alpha_r, alpha_i, - aa, lda, bb, incb, cc, 1, - buffer); -#else - if (!(transa & 1)) - (gemv[(int)transa]) (j, k, 0, alpha, aa, lda, - bb, incb, cc, 1, buffer); - else - (gemv[(int)transa]) (k, j, 0, alpha, aa, lda, - bb, incb, cc, 1, buffer); -#endif #ifdef SMP - } else { - if (!(transa & 1)) - (gemv_thread[(int)transa]) (j, k, alpha, aa, - lda, bb, incb, cc, - 1, buffer, - nthreads); - else - (gemv_thread[(int)transa]) (k, j, alpha, aa, - lda, bb, incb, cc, - 1, buffer, - nthreads); - - } -#endif - - STACK_FREE(buffer); - } + MMK = (double)(m + 1) * (double)m * (double)k; + if (MMK <= (SMP_THRESHOLD_MIN * GEMM_MULTITHREAD_THRESHOLD)) { + args.nthreads = 1; } else { + args.nthreads = num_cpu_avail(3); + } - for (i = 0; i < m; i++) { - j = i + 1; - - l = j; -#if defined COMPLEX - bb = b + i * ldb * 2; - if (transb & 1) { - bb = b + i * 2; - } - cc = c + i * 2 * ldc; -#else - bb = b + i * ldb; - if (transb & 1) { - bb = b + i; - } - cc = c + i * ldc; -#endif - -#if defined(COMPLEX) - if (beta_r != ONE || beta_i != ZERO) - SCAL_K(l, 0, 0, beta_r, beta_i, cc, 1, NULL, 0, - NULL, 0); - - if (alpha_r == ZERO && alpha_i == ZERO) - continue; -#else - if (beta != ONE) - SCAL_K(l, 0, 0, beta, cc, 1, NULL, 0, NULL, 0); - - if (alpha == ZERO) - continue; + if (args.nthreads == 1) { #endif - IDEBUG_START; - buffer_size = 2 * (j + k) + 128 / sizeof(FLOAT); -#ifdef WINDOWS_ABI - buffer_size += 160 / sizeof(FLOAT); -#endif - // for alignment - buffer_size = (buffer_size + 3) & ~3; - STACK_ALLOC(buffer_size, IFLOAT, buffer); + (gemmt[GEMMT_TABLE_INDEX])(&args, NULL, NULL, sa, sb, 0); #ifdef SMP + } else { - if (1L * j * k < 2304L * GEMM_MULTITHREAD_THRESHOLD) - nthreads = 1; - else - nthreads = num_cpu_avail(2); - - if (nthreads == 1) { -#endif + /* Split the triangle into column ranges of equal area; every + thread then runs the single-threaded driver on its own range + and writes a disjoint set of columns of C. */ + mode |= (uplo << BLAS_UPLO_SHIFT); -#if defined(COMPLEX) - if (!(transa & 1)) - (gemv[(int)transa]) (j, k, 0, alpha_r, alpha_i, - a, lda, bb, incb, cc, 1, - buffer); - else - (gemv[(int)transa]) (k, j, 0, alpha_r, alpha_i, - a, lda, bb, incb, cc, 1, - buffer); -#else - if (!(transa & 1)) - (gemv[(int)transa]) (j, k, 0, alpha, a, lda, bb, - incb, cc, 1, buffer); - else - (gemv[(int)transa]) (k, j, 0, alpha, a, lda, bb, - incb, cc, 1, buffer); + syrk_thread(mode, &args, NULL, NULL, gemmt[GEMMT_TABLE_INDEX], sa, sb, args.nthreads); + } #endif -#ifdef SMP - } else { - if (!(transa & 1)) - (gemv_thread[(int)transa]) (j, k, alpha, a, lda, - bb, incb, cc, 1, - buffer, nthreads); - else - (gemv_thread[(int)transa]) (k, j, alpha, a, lda, - bb, incb, cc, 1, - buffer, nthreads); - } -#endif + blas_memory_free(buffer); - STACK_FREE(buffer); - } - } + FUNCTION_PROFILE_END(COMPSIZE * COMPSIZE, args.m * args.k + args.k * args.n + args.m * args.n / 2, + args.m * args.n * args.k); IDEBUG_END; -/* transform B back if necessary */ -#if defined(COMPLEX) - if (transb > 1){ -#ifndef CBLAS - IMATCOPY_K_CNC(nrowb, ncolb, (FLOAT)(1.0), (FLOAT)(0.0), b, ldb); -#else - if (order == CblasColMajor) - IMATCOPY_K_CNC(nrowb, ncolb, (FLOAT)(1.0), (FLOAT)(0.0), b, ldb); - if (order == CblasRowMajor) - IMATCOPY_K_RNC(nrowb, ncolb, (FLOAT)(1.0), (FLOAT)(0.0), b, ldb); -#endif - } -#endif - return; } diff --git a/utest/test_extensions/test_cgemmt.c b/utest/test_extensions/test_cgemmt.c index 92590b1e8b..0f6fa76690 100644 --- a/utest/test_extensions/test_cgemmt.c +++ b/utest/test_extensions/test_cgemmt.c @@ -33,6 +33,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "utest/openblas_utest.h" #include "common.h" +#include +#include #define DATASIZE 100 @@ -2038,5 +2040,59 @@ CTEST(cgemmt, xerbla_c_api_rowmajor_ldc_invalid) M, K, lda, ldb, ldc, expected_info); ASSERT_EQUAL(TRUE, passed); } + +/** + * Test a size that is not a multiple of the register blocking and large + * enough to be split over several threads: the last block column of C then + * starts at an unaligned offset and its diagonal block has to be cut down + * by the kernel. Verified against GEMM, which may only differ by rounding, + * and the untouched triangle must be bit-identical. + */ +CTEST(cgemmt, upper_M_257_K_64_a_notrans_b_notrans) +{ + blasint M = 257, K = 64, i, j, e; + blasint lda = M, ldb = K, ldc = M; + blasint size_c = 2 * M * ldc; + char transa = 'N', transb = 'N', uplo = 'U'; + float alpha[] = {1.5, 0.5}, beta[] = {2.0, 1.0}; + float *a = (float *) malloc(2 * M * K * sizeof(float)); + float *b = (float *) malloc(2 * K * M * sizeof(float)); + float *c = (float *) malloc(size_c * sizeof(float)); + float *c_gemm = (float *) malloc(size_c * sizeof(float)); + float *c_orig = (float *) malloc(size_c * sizeof(float)); + double sumsq = 0.0, norm; + int untouched_ok = 1; + + ASSERT_TRUE(a != NULL && b != NULL && c != NULL && c_gemm != NULL && c_orig != NULL); + + srand_generate(a, 2 * M * K); + srand_generate(b, 2 * K * M); + srand_generate(c, size_c); + for (i = 0; i < size_c; i++) + c_gemm[i] = c_orig[i] = c[i]; + + BLASFUNC(cgemmt)(&uplo, &transa, &transb, &M, &K, alpha, a, &lda, b, &ldb, + beta, c, &ldc); + BLASFUNC(cgemm)(&transa, &transb, &M, &M, &K, alpha, a, &lda, b, &ldb, + beta, c_gemm, &ldc); + + for (j = 0; j < M; j++) /* column */ + for (i = 0; i < M; i++) /* row */ + for (e = 0; e < 2; e++) { + blasint idx = 2 * (i + j * ldc) + e; + if (i <= j) { + double d = (double)(c[idx] - c_gemm[idx]); + sumsq += d * d; + } else if (c[idx] != c_orig[idx]) { + untouched_ok = 0; + } + } + + norm = sqrt(sumsq) / size_c; + free(a); free(b); free(c); free(c_gemm); free(c_orig); + ASSERT_EQUAL(1, untouched_ok); + ASSERT_DBL_NEAR_TOL(0.0, norm, SINGLE_EPS); +} + #endif #endif diff --git a/utest/test_extensions/test_dgemmt.c b/utest/test_extensions/test_dgemmt.c index 9ce8859cca..57dba0b0e2 100644 --- a/utest/test_extensions/test_dgemmt.c +++ b/utest/test_extensions/test_dgemmt.c @@ -33,6 +33,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "utest/openblas_utest.h" #include "common.h" +#include +#include #define DATASIZE 100 @@ -1463,5 +1465,59 @@ CTEST(dgemmt, xerbla_c_api_rowmajor_ldc_invalid) M, K, lda, ldb, ldc, expected_info); ASSERT_EQUAL(TRUE, passed); } + +/** + * Test a size that is not a multiple of the register blocking and large + * enough to be split over several threads: the last block column of C then + * starts at an unaligned offset and its diagonal block has to be cut down + * by the kernel. Verified against GEMM, which may only differ by rounding, + * and the untouched triangle must be bit-identical. + */ +CTEST(dgemmt, upper_M_257_K_64_a_notrans_b_notrans) +{ + blasint M = 257, K = 64, i, j, e; + blasint lda = M, ldb = K, ldc = M; + blasint size_c = 1 * M * ldc; + char transa = 'N', transb = 'N', uplo = 'U'; + double alpha = 1.5, beta = 2.0; + double *a = (double *) malloc(1 * M * K * sizeof(double)); + double *b = (double *) malloc(1 * K * M * sizeof(double)); + double *c = (double *) malloc(size_c * sizeof(double)); + double *c_gemm = (double *) malloc(size_c * sizeof(double)); + double *c_orig = (double *) malloc(size_c * sizeof(double)); + double sumsq = 0.0, norm; + int untouched_ok = 1; + + ASSERT_TRUE(a != NULL && b != NULL && c != NULL && c_gemm != NULL && c_orig != NULL); + + drand_generate(a, 1 * M * K); + drand_generate(b, 1 * K * M); + drand_generate(c, size_c); + for (i = 0; i < size_c; i++) + c_gemm[i] = c_orig[i] = c[i]; + + BLASFUNC(dgemmt)(&uplo, &transa, &transb, &M, &K, &alpha, a, &lda, b, &ldb, + &beta, c, &ldc); + BLASFUNC(dgemm)(&transa, &transb, &M, &M, &K, &alpha, a, &lda, b, &ldb, + &beta, c_gemm, &ldc); + + for (j = 0; j < M; j++) /* column */ + for (i = 0; i < M; i++) /* row */ + for (e = 0; e < 1; e++) { + blasint idx = 1 * (i + j * ldc) + e; + if (i <= j) { + double d = (double)(c[idx] - c_gemm[idx]); + sumsq += d * d; + } else if (c[idx] != c_orig[idx]) { + untouched_ok = 0; + } + } + + norm = sqrt(sumsq) / size_c; + free(a); free(b); free(c); free(c_gemm); free(c_orig); + ASSERT_EQUAL(1, untouched_ok); + ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); +} + #endif #endif diff --git a/utest/test_extensions/test_sgemmt.c b/utest/test_extensions/test_sgemmt.c index f61fcf5fca..32c81aa875 100644 --- a/utest/test_extensions/test_sgemmt.c +++ b/utest/test_extensions/test_sgemmt.c @@ -33,6 +33,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "utest/openblas_utest.h" #include "common.h" +#include +#include #define DATASIZE 100 @@ -1463,5 +1465,59 @@ CTEST(sgemmt, xerbla_c_api_rowmajor_ldc_invalid) M, K, lda, ldb, ldc, expected_info); ASSERT_EQUAL(TRUE, passed); } + +/** + * Test a size that is not a multiple of the register blocking and large + * enough to be split over several threads: the last block column of C then + * starts at an unaligned offset and its diagonal block has to be cut down + * by the kernel. Verified against GEMM, which may only differ by rounding, + * and the untouched triangle must be bit-identical. + */ +CTEST(sgemmt, upper_M_257_K_64_a_notrans_b_notrans) +{ + blasint M = 257, K = 64, i, j, e; + blasint lda = M, ldb = K, ldc = M; + blasint size_c = 1 * M * ldc; + char transa = 'N', transb = 'N', uplo = 'U'; + float alpha = 1.5, beta = 2.0; + float *a = (float *) malloc(1 * M * K * sizeof(float)); + float *b = (float *) malloc(1 * K * M * sizeof(float)); + float *c = (float *) malloc(size_c * sizeof(float)); + float *c_gemm = (float *) malloc(size_c * sizeof(float)); + float *c_orig = (float *) malloc(size_c * sizeof(float)); + double sumsq = 0.0, norm; + int untouched_ok = 1; + + ASSERT_TRUE(a != NULL && b != NULL && c != NULL && c_gemm != NULL && c_orig != NULL); + + srand_generate(a, 1 * M * K); + srand_generate(b, 1 * K * M); + srand_generate(c, size_c); + for (i = 0; i < size_c; i++) + c_gemm[i] = c_orig[i] = c[i]; + + BLASFUNC(sgemmt)(&uplo, &transa, &transb, &M, &K, &alpha, a, &lda, b, &ldb, + &beta, c, &ldc); + BLASFUNC(sgemm)(&transa, &transb, &M, &M, &K, &alpha, a, &lda, b, &ldb, + &beta, c_gemm, &ldc); + + for (j = 0; j < M; j++) /* column */ + for (i = 0; i < M; i++) /* row */ + for (e = 0; e < 1; e++) { + blasint idx = 1 * (i + j * ldc) + e; + if (i <= j) { + double d = (double)(c[idx] - c_gemm[idx]); + sumsq += d * d; + } else if (c[idx] != c_orig[idx]) { + untouched_ok = 0; + } + } + + norm = sqrt(sumsq) / size_c; + free(a); free(b); free(c); free(c_gemm); free(c_orig); + ASSERT_EQUAL(1, untouched_ok); + ASSERT_DBL_NEAR_TOL(0.0, norm, SINGLE_EPS); +} + #endif #endif diff --git a/utest/test_extensions/test_zgemmt.c b/utest/test_extensions/test_zgemmt.c index dcd70e9e36..0f915ad2e8 100644 --- a/utest/test_extensions/test_zgemmt.c +++ b/utest/test_extensions/test_zgemmt.c @@ -33,6 +33,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "utest/openblas_utest.h" #include "common.h" +#include +#include #define DATASIZE 100 @@ -2036,5 +2038,59 @@ CTEST(zgemmt, xerbla_c_api_rowmajor_ldc_invalid) M, K, lda, ldb, ldc, expected_info); ASSERT_EQUAL(TRUE, passed); } + +/** + * Test a size that is not a multiple of the register blocking and large + * enough to be split over several threads: the last block column of C then + * starts at an unaligned offset and its diagonal block has to be cut down + * by the kernel. Verified against GEMM, which may only differ by rounding, + * and the untouched triangle must be bit-identical. + */ +CTEST(zgemmt, upper_M_257_K_64_a_notrans_b_notrans) +{ + blasint M = 257, K = 64, i, j, e; + blasint lda = M, ldb = K, ldc = M; + blasint size_c = 2 * M * ldc; + char transa = 'N', transb = 'N', uplo = 'U'; + double alpha[] = {1.5, 0.5}, beta[] = {2.0, 1.0}; + double *a = (double *) malloc(2 * M * K * sizeof(double)); + double *b = (double *) malloc(2 * K * M * sizeof(double)); + double *c = (double *) malloc(size_c * sizeof(double)); + double *c_gemm = (double *) malloc(size_c * sizeof(double)); + double *c_orig = (double *) malloc(size_c * sizeof(double)); + double sumsq = 0.0, norm; + int untouched_ok = 1; + + ASSERT_TRUE(a != NULL && b != NULL && c != NULL && c_gemm != NULL && c_orig != NULL); + + drand_generate(a, 2 * M * K); + drand_generate(b, 2 * K * M); + drand_generate(c, size_c); + for (i = 0; i < size_c; i++) + c_gemm[i] = c_orig[i] = c[i]; + + BLASFUNC(zgemmt)(&uplo, &transa, &transb, &M, &K, alpha, a, &lda, b, &ldb, + beta, c, &ldc); + BLASFUNC(zgemm)(&transa, &transb, &M, &M, &K, alpha, a, &lda, b, &ldb, + beta, c_gemm, &ldc); + + for (j = 0; j < M; j++) /* column */ + for (i = 0; i < M; i++) /* row */ + for (e = 0; e < 2; e++) { + blasint idx = 2 * (i + j * ldc) + e; + if (i <= j) { + double d = (double)(c[idx] - c_gemm[idx]); + sumsq += d * d; + } else if (c[idx] != c_orig[idx]) { + untouched_ok = 0; + } + } + + norm = sqrt(sumsq) / size_c; + free(a); free(b); free(c); free(c_gemm); free(c_orig); + ASSERT_EQUAL(1, untouched_ok); + ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); +} + #endif #endif From 49f817ac5a56749ee400d97a6a26098b4260bf5f Mon Sep 17 00:00:00 2001 From: Joris Gillis Date: Fri, 25 Sep 2026 22:21:31 +0200 Subject: [PATCH 2/2] Pass CR and RC as -DCR=CR / -DRC=RC when building GEMMT objects with make A bare -DCR expands every CR token in the MinGW system headers to 1 and breaks the complex ?gemmt_?CR objects ("expected identifier or '(' before numeric constant"). The GEMM rules and cmake/utils.cmake already use the self-referencing form for exactly this reason. Co-Authored-By: Claude Opus 5.5 --- driver/level3/Makefile | 48 +++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/driver/level3/Makefile b/driver/level3/Makefile index 9896fc6983..7866c12eb5 100644 --- a/driver/level3/Makefile +++ b/driver/level3/Makefile @@ -1745,7 +1745,7 @@ cgemmt_URR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DRR $< -o $(@F) cgemmt_URC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DRC $< -o $(@F) + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DRC=RC $< -o $(@F) cgemmt_UCN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCN $< -o $(@F) @@ -1754,7 +1754,7 @@ cgemmt_UCT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCT $< -o $(@F) cgemmt_UCR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCR $< -o $(@F) + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCR=CR $< -o $(@F) cgemmt_UCC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCC $< -o $(@F) @@ -1793,7 +1793,7 @@ cgemmt_LRR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DRR $< -o $(@F) cgemmt_LRC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DRC $< -o $(@F) + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DRC=RC $< -o $(@F) cgemmt_LCN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCN $< -o $(@F) @@ -1802,7 +1802,7 @@ cgemmt_LCT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCT $< -o $(@F) cgemmt_LCR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCR $< -o $(@F) + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCR=CR $< -o $(@F) cgemmt_LCC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCC $< -o $(@F) @@ -1859,7 +1859,7 @@ zgemmt_URR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DRR $< -o $(@F) zgemmt_URC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DRC $< -o $(@F) + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DRC=RC $< -o $(@F) zgemmt_UCN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCN $< -o $(@F) @@ -1868,7 +1868,7 @@ zgemmt_UCT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCT $< -o $(@F) zgemmt_UCR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCR $< -o $(@F) + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCR=CR $< -o $(@F) zgemmt_UCC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCC $< -o $(@F) @@ -1907,7 +1907,7 @@ zgemmt_LRR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DRR $< -o $(@F) zgemmt_LRC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DRC $< -o $(@F) + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DRC=RC $< -o $(@F) zgemmt_LCN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCN $< -o $(@F) @@ -1916,7 +1916,7 @@ zgemmt_LCT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCT $< -o $(@F) zgemmt_LCR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCR $< -o $(@F) + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCR=CR $< -o $(@F) zgemmt_LCC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCC $< -o $(@F) @@ -1973,7 +1973,7 @@ xgemmt_URR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DRR $< -o $(@F) xgemmt_URC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DRC $< -o $(@F) + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DRC=RC $< -o $(@F) xgemmt_UCN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCN $< -o $(@F) @@ -1982,7 +1982,7 @@ xgemmt_UCT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCT $< -o $(@F) xgemmt_UCR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCR $< -o $(@F) + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCR=CR $< -o $(@F) xgemmt_UCC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCC $< -o $(@F) @@ -2021,7 +2021,7 @@ xgemmt_LRR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DRR $< -o $(@F) xgemmt_LRC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DRC $< -o $(@F) + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DRC=RC $< -o $(@F) xgemmt_LCN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCN $< -o $(@F) @@ -2030,7 +2030,7 @@ xgemmt_LCT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCT $< -o $(@F) xgemmt_LCR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCR $< -o $(@F) + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCR=CR $< -o $(@F) xgemmt_LCC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCC $< -o $(@F) @@ -4595,7 +4595,7 @@ cgemmt_URR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DRR $< -o $(@F) cgemmt_URC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DRC $< -o $(@F) + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DRC=RC $< -o $(@F) cgemmt_UCN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCN $< -o $(@F) @@ -4604,7 +4604,7 @@ cgemmt_UCT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCT $< -o $(@F) cgemmt_UCR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCR $< -o $(@F) + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCR=CR $< -o $(@F) cgemmt_UCC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCC $< -o $(@F) @@ -4643,7 +4643,7 @@ cgemmt_LRR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DRR $< -o $(@F) cgemmt_LRC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DRC $< -o $(@F) + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DRC=RC $< -o $(@F) cgemmt_LCN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCN $< -o $(@F) @@ -4652,7 +4652,7 @@ cgemmt_LCT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCT $< -o $(@F) cgemmt_LCR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCR $< -o $(@F) + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCR=CR $< -o $(@F) cgemmt_LCC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCC $< -o $(@F) @@ -4709,7 +4709,7 @@ zgemmt_URR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DRR $< -o $(@F) zgemmt_URC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DRC $< -o $(@F) + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DRC=RC $< -o $(@F) zgemmt_UCN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCN $< -o $(@F) @@ -4718,7 +4718,7 @@ zgemmt_UCT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCT $< -o $(@F) zgemmt_UCR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCR $< -o $(@F) + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCR=CR $< -o $(@F) zgemmt_UCC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCC $< -o $(@F) @@ -4757,7 +4757,7 @@ zgemmt_LRR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DRR $< -o $(@F) zgemmt_LRC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DRC $< -o $(@F) + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DRC=RC $< -o $(@F) zgemmt_LCN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCN $< -o $(@F) @@ -4766,7 +4766,7 @@ zgemmt_LCT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCT $< -o $(@F) zgemmt_LCR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCR $< -o $(@F) + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCR=CR $< -o $(@F) zgemmt_LCC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCC $< -o $(@F) @@ -4823,7 +4823,7 @@ xgemmt_URR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DRR $< -o $(@F) xgemmt_URC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DRC $< -o $(@F) + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DRC=RC $< -o $(@F) xgemmt_UCN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCN $< -o $(@F) @@ -4832,7 +4832,7 @@ xgemmt_UCT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCT $< -o $(@F) xgemmt_UCR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCR $< -o $(@F) + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCR=CR $< -o $(@F) xgemmt_UCC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCC $< -o $(@F) @@ -4871,7 +4871,7 @@ xgemmt_LRR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DRR $< -o $(@F) xgemmt_LRC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DRC $< -o $(@F) + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DRC=RC $< -o $(@F) xgemmt_LCN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCN $< -o $(@F) @@ -4880,7 +4880,7 @@ xgemmt_LCT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCT $< -o $(@F) xgemmt_LCR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h - $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCR $< -o $(@F) + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCR=CR $< -o $(@F) xgemmt_LCC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCC $< -o $(@F)