diff --git a/common_c.h b/common_c.h index 88f8f5dcdf..3aa9349d0a 100644 --- a/common_c.h +++ b/common_c.h @@ -514,6 +514,46 @@ #define CSYRK_KERNEL_U csyrk_kernel_U #define CSYRK_KERNEL_L csyrk_kernel_L +#define CGEMMT_UNN cgemmt_UNN +#define CGEMMT_UNT cgemmt_UNT +#define CGEMMT_UNR cgemmt_UNR +#define CGEMMT_UNC cgemmt_UNC +#define CGEMMT_UTN cgemmt_UTN +#define CGEMMT_UTT cgemmt_UTT +#define CGEMMT_UTR cgemmt_UTR +#define CGEMMT_UTC cgemmt_UTC +#define CGEMMT_URN cgemmt_URN +#define CGEMMT_URT cgemmt_URT +#define CGEMMT_URR cgemmt_URR +#define CGEMMT_URC cgemmt_URC +#define CGEMMT_UCN cgemmt_UCN +#define CGEMMT_UCT cgemmt_UCT +#define CGEMMT_UCR cgemmt_UCR +#define CGEMMT_UCC cgemmt_UCC +#define CGEMMT_LNN cgemmt_LNN +#define CGEMMT_LNT cgemmt_LNT +#define CGEMMT_LNR cgemmt_LNR +#define CGEMMT_LNC cgemmt_LNC +#define CGEMMT_LTN cgemmt_LTN +#define CGEMMT_LTT cgemmt_LTT +#define CGEMMT_LTR cgemmt_LTR +#define CGEMMT_LTC cgemmt_LTC +#define CGEMMT_LRN cgemmt_LRN +#define CGEMMT_LRT cgemmt_LRT +#define CGEMMT_LRR cgemmt_LRR +#define CGEMMT_LRC cgemmt_LRC +#define CGEMMT_LCN cgemmt_LCN +#define CGEMMT_LCT cgemmt_LCT +#define CGEMMT_LCR cgemmt_LCR +#define CGEMMT_LCC cgemmt_LCC + +#define CGEMMT_KERNEL_UCN cgemmt_kernel_UCN +#define CGEMMT_KERNEL_UNC cgemmt_kernel_UNC +#define CGEMMT_KERNEL_UCC cgemmt_kernel_UCC +#define CGEMMT_KERNEL_LCN cgemmt_kernel_LCN +#define CGEMMT_KERNEL_LNC cgemmt_kernel_LNC +#define CGEMMT_KERNEL_LCC cgemmt_kernel_LCC + #define CHERK_UN cherk_UN #define CHERK_LN cherk_LN #define CHERK_UC cherk_UC diff --git a/common_d.h b/common_d.h index a7a1e1cd14..200d840b0d 100644 --- a/common_d.h +++ b/common_d.h @@ -341,6 +341,15 @@ #define DSYRK_KERNEL_U dsyrk_kernel_U #define DSYRK_KERNEL_L dsyrk_kernel_L +#define DGEMMT_UNN dgemmt_UNN +#define DGEMMT_UNT dgemmt_UNT +#define DGEMMT_UTN dgemmt_UTN +#define DGEMMT_UTT dgemmt_UTT +#define DGEMMT_LNN dgemmt_LNN +#define DGEMMT_LNT dgemmt_LNT +#define DGEMMT_LTN dgemmt_LTN +#define DGEMMT_LTT dgemmt_LTT + #define DHERK_UN dsyrk_UN #define DHERK_LN dsyrk_LN #define DHERK_UC dsyrk_UT diff --git a/common_level3.h b/common_level3.h index 3e4edf1950..6ef6cf2331 100644 --- a/common_level3.h +++ b/common_level3.h @@ -1689,6 +1689,151 @@ int ssyrk_thread_LN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLAS int ssyrk_thread_LT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); int dsyrk_thread_UN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); + +int qgemmt_UNN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int qgemmt_UNT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int qgemmt_UTN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int qgemmt_UTT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int qgemmt_LNN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int qgemmt_LNT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int qgemmt_LTN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int qgemmt_LTT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); + +int dgemmt_UNN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int dgemmt_UNT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int dgemmt_UTN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int dgemmt_UTT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int dgemmt_LNN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int dgemmt_LNT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int dgemmt_LTN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int dgemmt_LTT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); + +int sgemmt_UNN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int sgemmt_UNT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int sgemmt_UTN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int sgemmt_UTT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int sgemmt_LNN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int sgemmt_LNT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int sgemmt_LTN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int sgemmt_LTT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); + +int xgemmt_UNN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_UNT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_UNR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_UNC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_UTN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_UTT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_UTR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_UTC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_URN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_URT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_URR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_URC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_UCN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_UCT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_UCR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_UCC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LNN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LNT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LNR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LNC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LTN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LTT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LTR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LTC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LRN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LRT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LRR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LRC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LCN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LCT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LCR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_LCC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG); +int xgemmt_kernel_UCN(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); +int xgemmt_kernel_UNC(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); +int xgemmt_kernel_UCC(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); +int xgemmt_kernel_LCN(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); +int xgemmt_kernel_LNC(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); +int xgemmt_kernel_LCC(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); + +int zgemmt_UNN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_UNT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_UNR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_UNC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_UTN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_UTT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_UTR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_UTC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_URN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_URT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_URR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_URC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_UCN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_UCT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_UCR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_UCC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LNN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LNT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LNR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LNC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LTN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LTT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LTR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LTC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LRN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LRT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LRR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LRC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LCN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LCT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LCR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_LCC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); +int zgemmt_kernel_UCN(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); +int zgemmt_kernel_UNC(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); +int zgemmt_kernel_UCC(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); +int zgemmt_kernel_LCN(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); +int zgemmt_kernel_LNC(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); +int zgemmt_kernel_LCC(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); + +int cgemmt_UNN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_UNT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_UNR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_UNC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_UTN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_UTT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_UTR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_UTC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_URN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_URT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_URR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_URC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_UCN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_UCT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_UCR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_UCC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LNN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LNT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LNR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LNC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LTN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LTT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LTR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LTC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LRN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LRT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LRR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LRC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LCN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LCT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LCR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_LCC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG); +int cgemmt_kernel_UCN(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); +int cgemmt_kernel_UNC(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); +int cgemmt_kernel_UCC(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); +int cgemmt_kernel_LCN(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); +int cgemmt_kernel_LNC(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); +int cgemmt_kernel_LCC(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); + int dsyrk_thread_UT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); int dsyrk_thread_LN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); int dsyrk_thread_LT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG); diff --git a/common_macro.h b/common_macro.h index 84c2473020..2b23f06c71 100644 --- a/common_macro.h +++ b/common_macro.h @@ -211,6 +211,15 @@ #define SYRK_KERNEL_U QSYRK_KERNEL_U #define SYRK_KERNEL_L QSYRK_KERNEL_L +#define GEMMT_UNN QGEMMT_UNN +#define GEMMT_UNT QGEMMT_UNT +#define GEMMT_UTN QGEMMT_UTN +#define GEMMT_UTT QGEMMT_UTT +#define GEMMT_LNN QGEMMT_LNN +#define GEMMT_LNT QGEMMT_LNT +#define GEMMT_LTN QGEMMT_LTN +#define GEMMT_LTT QGEMMT_LTT + #define HERK_UN QSYRK_UN #define HERK_LN QSYRK_LN #define HERK_UC QSYRK_UT @@ -503,6 +512,15 @@ #define SYRK_KERNEL_U DSYRK_KERNEL_U #define SYRK_KERNEL_L DSYRK_KERNEL_L +#define GEMMT_UNN DGEMMT_UNN +#define GEMMT_UNT DGEMMT_UNT +#define GEMMT_UTN DGEMMT_UTN +#define GEMMT_UTT DGEMMT_UTT +#define GEMMT_LNN DGEMMT_LNN +#define GEMMT_LNT DGEMMT_LNT +#define GEMMT_LTN DGEMMT_LTN +#define GEMMT_LTT DGEMMT_LTT + #define HERK_UN DSYRK_UN #define HERK_LN DSYRK_LN #define HERK_UC DSYRK_UT @@ -991,6 +1009,15 @@ #define SYRK_KERNEL_U SSYRK_KERNEL_U #define SYRK_KERNEL_L SSYRK_KERNEL_L +#define GEMMT_UNN SGEMMT_UNN +#define GEMMT_UNT SGEMMT_UNT +#define GEMMT_UTN SGEMMT_UTN +#define GEMMT_UTT SGEMMT_UTT +#define GEMMT_LNN SGEMMT_LNN +#define GEMMT_LNT SGEMMT_LNT +#define GEMMT_LTN SGEMMT_LTN +#define GEMMT_LTT SGEMMT_LTT + #define HERK_UN SSYRK_UN #define HERK_LN SSYRK_LN #define HERK_UC SSYRK_UT @@ -1214,6 +1241,15 @@ #define SYRK_KERNEL_U SSYRK_KERNEL_U #define SYRK_KERNEL_L SSYRK_KERNEL_L +#define GEMMT_UNN SGEMMT_UNN +#define GEMMT_UNT SGEMMT_UNT +#define GEMMT_UTN SGEMMT_UTN +#define GEMMT_UTT SGEMMT_UTT +#define GEMMT_LNN SGEMMT_LNN +#define GEMMT_LNT SGEMMT_LNT +#define GEMMT_LTN SGEMMT_LTN +#define GEMMT_LTT SGEMMT_LTT + #define HERK_UN SSYRK_UN #define HERK_LN SSYRK_LN #define HERK_UC SSYRK_UT @@ -1583,6 +1619,46 @@ #define SYRK_KERNEL_U XSYRK_KERNEL_U #define SYRK_KERNEL_L XSYRK_KERNEL_L +#define GEMMT_UNN XGEMMT_UNN +#define GEMMT_UNT XGEMMT_UNT +#define GEMMT_UNR XGEMMT_UNR +#define GEMMT_UNC XGEMMT_UNC +#define GEMMT_UTN XGEMMT_UTN +#define GEMMT_UTT XGEMMT_UTT +#define GEMMT_UTR XGEMMT_UTR +#define GEMMT_UTC XGEMMT_UTC +#define GEMMT_URN XGEMMT_URN +#define GEMMT_URT XGEMMT_URT +#define GEMMT_URR XGEMMT_URR +#define GEMMT_URC XGEMMT_URC +#define GEMMT_UCN XGEMMT_UCN +#define GEMMT_UCT XGEMMT_UCT +#define GEMMT_UCR XGEMMT_UCR +#define GEMMT_UCC XGEMMT_UCC +#define GEMMT_LNN XGEMMT_LNN +#define GEMMT_LNT XGEMMT_LNT +#define GEMMT_LNR XGEMMT_LNR +#define GEMMT_LNC XGEMMT_LNC +#define GEMMT_LTN XGEMMT_LTN +#define GEMMT_LTT XGEMMT_LTT +#define GEMMT_LTR XGEMMT_LTR +#define GEMMT_LTC XGEMMT_LTC +#define GEMMT_LRN XGEMMT_LRN +#define GEMMT_LRT XGEMMT_LRT +#define GEMMT_LRR XGEMMT_LRR +#define GEMMT_LRC XGEMMT_LRC +#define GEMMT_LCN XGEMMT_LCN +#define GEMMT_LCT XGEMMT_LCT +#define GEMMT_LCR XGEMMT_LCR +#define GEMMT_LCC XGEMMT_LCC + +#define GEMMT_KERNEL_UCN XGEMMT_KERNEL_UCN +#define GEMMT_KERNEL_UNC XGEMMT_KERNEL_UNC +#define GEMMT_KERNEL_UCC XGEMMT_KERNEL_UCC +#define GEMMT_KERNEL_LCN XGEMMT_KERNEL_LCN +#define GEMMT_KERNEL_LNC XGEMMT_KERNEL_LNC +#define GEMMT_KERNEL_LCC XGEMMT_KERNEL_LCC + #define HERK_UN XHERK_UN #define HERK_LN XHERK_LN #define HERK_UC XHERK_UC @@ -1985,6 +2061,46 @@ #define SYRK_KERNEL_U ZSYRK_KERNEL_U #define SYRK_KERNEL_L ZSYRK_KERNEL_L +#define GEMMT_UNN ZGEMMT_UNN +#define GEMMT_UNT ZGEMMT_UNT +#define GEMMT_UNR ZGEMMT_UNR +#define GEMMT_UNC ZGEMMT_UNC +#define GEMMT_UTN ZGEMMT_UTN +#define GEMMT_UTT ZGEMMT_UTT +#define GEMMT_UTR ZGEMMT_UTR +#define GEMMT_UTC ZGEMMT_UTC +#define GEMMT_URN ZGEMMT_URN +#define GEMMT_URT ZGEMMT_URT +#define GEMMT_URR ZGEMMT_URR +#define GEMMT_URC ZGEMMT_URC +#define GEMMT_UCN ZGEMMT_UCN +#define GEMMT_UCT ZGEMMT_UCT +#define GEMMT_UCR ZGEMMT_UCR +#define GEMMT_UCC ZGEMMT_UCC +#define GEMMT_LNN ZGEMMT_LNN +#define GEMMT_LNT ZGEMMT_LNT +#define GEMMT_LNR ZGEMMT_LNR +#define GEMMT_LNC ZGEMMT_LNC +#define GEMMT_LTN ZGEMMT_LTN +#define GEMMT_LTT ZGEMMT_LTT +#define GEMMT_LTR ZGEMMT_LTR +#define GEMMT_LTC ZGEMMT_LTC +#define GEMMT_LRN ZGEMMT_LRN +#define GEMMT_LRT ZGEMMT_LRT +#define GEMMT_LRR ZGEMMT_LRR +#define GEMMT_LRC ZGEMMT_LRC +#define GEMMT_LCN ZGEMMT_LCN +#define GEMMT_LCT ZGEMMT_LCT +#define GEMMT_LCR ZGEMMT_LCR +#define GEMMT_LCC ZGEMMT_LCC + +#define GEMMT_KERNEL_UCN ZGEMMT_KERNEL_UCN +#define GEMMT_KERNEL_UNC ZGEMMT_KERNEL_UNC +#define GEMMT_KERNEL_UCC ZGEMMT_KERNEL_UCC +#define GEMMT_KERNEL_LCN ZGEMMT_KERNEL_LCN +#define GEMMT_KERNEL_LNC ZGEMMT_KERNEL_LNC +#define GEMMT_KERNEL_LCC ZGEMMT_KERNEL_LCC + #define HERK_UN ZHERK_UN #define HERK_LN ZHERK_LN #define HERK_UC ZHERK_UC @@ -2450,6 +2566,46 @@ #define SYRK_KERNEL_U CSYRK_KERNEL_U #define SYRK_KERNEL_L CSYRK_KERNEL_L +#define GEMMT_UNN CGEMMT_UNN +#define GEMMT_UNT CGEMMT_UNT +#define GEMMT_UNR CGEMMT_UNR +#define GEMMT_UNC CGEMMT_UNC +#define GEMMT_UTN CGEMMT_UTN +#define GEMMT_UTT CGEMMT_UTT +#define GEMMT_UTR CGEMMT_UTR +#define GEMMT_UTC CGEMMT_UTC +#define GEMMT_URN CGEMMT_URN +#define GEMMT_URT CGEMMT_URT +#define GEMMT_URR CGEMMT_URR +#define GEMMT_URC CGEMMT_URC +#define GEMMT_UCN CGEMMT_UCN +#define GEMMT_UCT CGEMMT_UCT +#define GEMMT_UCR CGEMMT_UCR +#define GEMMT_UCC CGEMMT_UCC +#define GEMMT_LNN CGEMMT_LNN +#define GEMMT_LNT CGEMMT_LNT +#define GEMMT_LNR CGEMMT_LNR +#define GEMMT_LNC CGEMMT_LNC +#define GEMMT_LTN CGEMMT_LTN +#define GEMMT_LTT CGEMMT_LTT +#define GEMMT_LTR CGEMMT_LTR +#define GEMMT_LTC CGEMMT_LTC +#define GEMMT_LRN CGEMMT_LRN +#define GEMMT_LRT CGEMMT_LRT +#define GEMMT_LRR CGEMMT_LRR +#define GEMMT_LRC CGEMMT_LRC +#define GEMMT_LCN CGEMMT_LCN +#define GEMMT_LCT CGEMMT_LCT +#define GEMMT_LCR CGEMMT_LCR +#define GEMMT_LCC CGEMMT_LCC + +#define GEMMT_KERNEL_UCN CGEMMT_KERNEL_UCN +#define GEMMT_KERNEL_UNC CGEMMT_KERNEL_UNC +#define GEMMT_KERNEL_UCC CGEMMT_KERNEL_UCC +#define GEMMT_KERNEL_LCN CGEMMT_KERNEL_LCN +#define GEMMT_KERNEL_LNC CGEMMT_KERNEL_LNC +#define GEMMT_KERNEL_LCC CGEMMT_KERNEL_LCC + #define HERK_UN CHERK_UN #define HERK_LN CHERK_LN #define HERK_UC CHERK_UC diff --git a/common_q.h b/common_q.h index 1d976f1e8d..52da71c88c 100644 --- a/common_q.h +++ b/common_q.h @@ -299,6 +299,15 @@ #define QSYRK_KERNEL_U qsyrk_kernel_U #define QSYRK_KERNEL_L qsyrk_kernel_L +#define QGEMMT_UNN qgemmt_UNN +#define QGEMMT_UNT qgemmt_UNT +#define QGEMMT_UTN qgemmt_UTN +#define QGEMMT_UTT qgemmt_UTT +#define QGEMMT_LNN qgemmt_LNN +#define QGEMMT_LNT qgemmt_LNT +#define QGEMMT_LTN qgemmt_LTN +#define QGEMMT_LTT qgemmt_LTT + #define QHERK_UN qsyrk_UN #define QHERK_LN qsyrk_LN #define QHERK_UC qsyrk_UT diff --git a/common_s.h b/common_s.h index 6df9ba0f1e..4e5f1af7cb 100644 --- a/common_s.h +++ b/common_s.h @@ -389,6 +389,15 @@ #define SSYRK_KERNEL_U ssyrk_kernel_U #define SSYRK_KERNEL_L ssyrk_kernel_L +#define SGEMMT_UNN sgemmt_UNN +#define SGEMMT_UNT sgemmt_UNT +#define SGEMMT_UTN sgemmt_UTN +#define SGEMMT_UTT sgemmt_UTT +#define SGEMMT_LNN sgemmt_LNN +#define SGEMMT_LNT sgemmt_LNT +#define SGEMMT_LTN sgemmt_LTN +#define SGEMMT_LTT sgemmt_LTT + #define SHERK_UN ssyrk_UN #define SHERK_LN ssyrk_LN #define SHERK_UC ssyrk_UT diff --git a/common_x.h b/common_x.h index 2ed525faa1..12ed909c57 100644 --- a/common_x.h +++ b/common_x.h @@ -423,6 +423,46 @@ #define XSYRK_KERNEL_U xsyrk_kernel_U #define XSYRK_KERNEL_L xsyrk_kernel_L +#define XGEMMT_UNN xgemmt_UNN +#define XGEMMT_UNT xgemmt_UNT +#define XGEMMT_UNR xgemmt_UNR +#define XGEMMT_UNC xgemmt_UNC +#define XGEMMT_UTN xgemmt_UTN +#define XGEMMT_UTT xgemmt_UTT +#define XGEMMT_UTR xgemmt_UTR +#define XGEMMT_UTC xgemmt_UTC +#define XGEMMT_URN xgemmt_URN +#define XGEMMT_URT xgemmt_URT +#define XGEMMT_URR xgemmt_URR +#define XGEMMT_URC xgemmt_URC +#define XGEMMT_UCN xgemmt_UCN +#define XGEMMT_UCT xgemmt_UCT +#define XGEMMT_UCR xgemmt_UCR +#define XGEMMT_UCC xgemmt_UCC +#define XGEMMT_LNN xgemmt_LNN +#define XGEMMT_LNT xgemmt_LNT +#define XGEMMT_LNR xgemmt_LNR +#define XGEMMT_LNC xgemmt_LNC +#define XGEMMT_LTN xgemmt_LTN +#define XGEMMT_LTT xgemmt_LTT +#define XGEMMT_LTR xgemmt_LTR +#define XGEMMT_LTC xgemmt_LTC +#define XGEMMT_LRN xgemmt_LRN +#define XGEMMT_LRT xgemmt_LRT +#define XGEMMT_LRR xgemmt_LRR +#define XGEMMT_LRC xgemmt_LRC +#define XGEMMT_LCN xgemmt_LCN +#define XGEMMT_LCT xgemmt_LCT +#define XGEMMT_LCR xgemmt_LCR +#define XGEMMT_LCC xgemmt_LCC + +#define XGEMMT_KERNEL_UCN xgemmt_kernel_UCN +#define XGEMMT_KERNEL_UNC xgemmt_kernel_UNC +#define XGEMMT_KERNEL_UCC xgemmt_kernel_UCC +#define XGEMMT_KERNEL_LCN xgemmt_kernel_LCN +#define XGEMMT_KERNEL_LNC xgemmt_kernel_LNC +#define XGEMMT_KERNEL_LCC xgemmt_kernel_LCC + #define XHERK_UN xherk_UN #define XHERK_LN xherk_LN #define XHERK_UC xherk_UC diff --git a/common_z.h b/common_z.h index 5a9fb88b77..bd79f49d0f 100644 --- a/common_z.h +++ b/common_z.h @@ -514,6 +514,46 @@ #define ZSYRK_KERNEL_U zsyrk_kernel_U #define ZSYRK_KERNEL_L zsyrk_kernel_L +#define ZGEMMT_UNN zgemmt_UNN +#define ZGEMMT_UNT zgemmt_UNT +#define ZGEMMT_UNR zgemmt_UNR +#define ZGEMMT_UNC zgemmt_UNC +#define ZGEMMT_UTN zgemmt_UTN +#define ZGEMMT_UTT zgemmt_UTT +#define ZGEMMT_UTR zgemmt_UTR +#define ZGEMMT_UTC zgemmt_UTC +#define ZGEMMT_URN zgemmt_URN +#define ZGEMMT_URT zgemmt_URT +#define ZGEMMT_URR zgemmt_URR +#define ZGEMMT_URC zgemmt_URC +#define ZGEMMT_UCN zgemmt_UCN +#define ZGEMMT_UCT zgemmt_UCT +#define ZGEMMT_UCR zgemmt_UCR +#define ZGEMMT_UCC zgemmt_UCC +#define ZGEMMT_LNN zgemmt_LNN +#define ZGEMMT_LNT zgemmt_LNT +#define ZGEMMT_LNR zgemmt_LNR +#define ZGEMMT_LNC zgemmt_LNC +#define ZGEMMT_LTN zgemmt_LTN +#define ZGEMMT_LTT zgemmt_LTT +#define ZGEMMT_LTR zgemmt_LTR +#define ZGEMMT_LTC zgemmt_LTC +#define ZGEMMT_LRN zgemmt_LRN +#define ZGEMMT_LRT zgemmt_LRT +#define ZGEMMT_LRR zgemmt_LRR +#define ZGEMMT_LRC zgemmt_LRC +#define ZGEMMT_LCN zgemmt_LCN +#define ZGEMMT_LCT zgemmt_LCT +#define ZGEMMT_LCR zgemmt_LCR +#define ZGEMMT_LCC zgemmt_LCC + +#define ZGEMMT_KERNEL_UCN zgemmt_kernel_UCN +#define ZGEMMT_KERNEL_UNC zgemmt_kernel_UNC +#define ZGEMMT_KERNEL_UCC zgemmt_kernel_UCC +#define ZGEMMT_KERNEL_LCN zgemmt_kernel_LCN +#define ZGEMMT_KERNEL_LNC zgemmt_kernel_LNC +#define ZGEMMT_KERNEL_LCC zgemmt_kernel_LCC + #define ZHERK_UN zherk_UN #define ZHERK_LN zherk_LN #define ZHERK_UC zherk_UC diff --git a/driver/level3/CMakeLists.txt b/driver/level3/CMakeLists.txt index 5024331dab..58a6d3e38f 100644 --- a/driver/level3/CMakeLists.txt +++ b/driver/level3/CMakeLists.txt @@ -63,6 +63,31 @@ GenerateCombinationObjects("symm_k.c" "RSIDE;LOWER" "L;U" "NN" 1) GenerateCombinationObjects("syrk_k.c" "LOWER;TRANS" "U;N" "" 1) GenerateCombinationObjects("syr2k_k.c" "LOWER;TRANS" "U;N" "" 1) GenerateCombinationObjects("syrk_kernel.c" "LOWER" "U" "" 2) + +# GEMMT / GEMMTR: one driver per uplo x op(A) x op(B) (the same set of +# op combinations as GEMM), plus the conjugating variants of the +# triangular kernel, which only complex GEMMT needs. +set(GEMMT_REAL_TRANS N T) +set(GEMMT_COMPLEX_TRANS N T R C) +foreach (GEMMT_UPLO U L) + set(GEMMT_UPLO_DEFS "") + if (${GEMMT_UPLO} STREQUAL "L") + set(GEMMT_UPLO_DEFS "LOWER") + endif () + foreach (GEMMT_TA ${GEMMT_REAL_TRANS}) + foreach (GEMMT_TB ${GEMMT_REAL_TRANS}) + GenerateNamedObjects("gemmt_k.c" "${GEMMT_TA}${GEMMT_TB};${GEMMT_UPLO_DEFS}" "gemmt_${GEMMT_UPLO}${GEMMT_TA}${GEMMT_TB}" 0 "" "" false 1) + endforeach () + endforeach () + foreach (GEMMT_TA ${GEMMT_COMPLEX_TRANS}) + foreach (GEMMT_TB ${GEMMT_COMPLEX_TRANS}) + GenerateNamedObjects("gemmt_k.c" "${GEMMT_TA}${GEMMT_TB};${GEMMT_UPLO_DEFS}" "gemmt_${GEMMT_UPLO}${GEMMT_TA}${GEMMT_TB}" 0 "" "" false 2) + endforeach () + endforeach () + GenerateNamedObjects("syrk_kernel.c" "CONJA;${GEMMT_UPLO_DEFS}" "gemmt_kernel_${GEMMT_UPLO}CN" 0 "" "" false 2) + GenerateNamedObjects("syrk_kernel.c" "CONJB;${GEMMT_UPLO_DEFS}" "gemmt_kernel_${GEMMT_UPLO}NC" 0 "" "" false 2) + GenerateNamedObjects("syrk_kernel.c" "CONJA;CONJB;${GEMMT_UPLO_DEFS}" "gemmt_kernel_${GEMMT_UPLO}CC" 0 "" "" false 2) +endforeach () GenerateCombinationObjects("syr2k_kernel.c" "LOWER" "U" "" 2) if (USE_THREAD) diff --git a/driver/level3/Makefile b/driver/level3/Makefile index 70655d9cd3..7866c12eb5 100644 --- a/driver/level3/Makefile +++ b/driver/level3/Makefile @@ -208,6 +208,57 @@ XBLASOBJS += \ xher2k_kernel_UN.$(SUFFIX) xher2k_kernel_UC.$(SUFFIX) \ xher2k_kernel_LN.$(SUFFIX) xher2k_kernel_LC.$(SUFFIX) + +# GEMMT / GEMMTR: one driver per uplo x op(A) x op(B), plus the +# conjugating variants of the triangular (SYRK) kernel for complex. +SBLASOBJS += \ + sgemmt_UNN.$(SUFFIX) sgemmt_UNT.$(SUFFIX) sgemmt_UTN.$(SUFFIX) sgemmt_UTT.$(SUFFIX) \ + sgemmt_LNN.$(SUFFIX) sgemmt_LNT.$(SUFFIX) sgemmt_LTN.$(SUFFIX) sgemmt_LTT.$(SUFFIX) + +DBLASOBJS += \ + dgemmt_UNN.$(SUFFIX) dgemmt_UNT.$(SUFFIX) dgemmt_UTN.$(SUFFIX) dgemmt_UTT.$(SUFFIX) \ + dgemmt_LNN.$(SUFFIX) dgemmt_LNT.$(SUFFIX) dgemmt_LTN.$(SUFFIX) dgemmt_LTT.$(SUFFIX) + +QBLASOBJS += \ + qgemmt_UNN.$(SUFFIX) qgemmt_UNT.$(SUFFIX) qgemmt_UTN.$(SUFFIX) qgemmt_UTT.$(SUFFIX) \ + qgemmt_LNN.$(SUFFIX) qgemmt_LNT.$(SUFFIX) qgemmt_LTN.$(SUFFIX) qgemmt_LTT.$(SUFFIX) + +CBLASOBJS += \ + cgemmt_UNN.$(SUFFIX) cgemmt_UNT.$(SUFFIX) cgemmt_UNR.$(SUFFIX) cgemmt_UNC.$(SUFFIX) \ + cgemmt_UTN.$(SUFFIX) cgemmt_UTT.$(SUFFIX) cgemmt_UTR.$(SUFFIX) cgemmt_UTC.$(SUFFIX) \ + cgemmt_URN.$(SUFFIX) cgemmt_URT.$(SUFFIX) cgemmt_URR.$(SUFFIX) cgemmt_URC.$(SUFFIX) \ + cgemmt_UCN.$(SUFFIX) cgemmt_UCT.$(SUFFIX) cgemmt_UCR.$(SUFFIX) cgemmt_UCC.$(SUFFIX) \ + cgemmt_LNN.$(SUFFIX) cgemmt_LNT.$(SUFFIX) cgemmt_LNR.$(SUFFIX) cgemmt_LNC.$(SUFFIX) \ + cgemmt_LTN.$(SUFFIX) cgemmt_LTT.$(SUFFIX) cgemmt_LTR.$(SUFFIX) cgemmt_LTC.$(SUFFIX) \ + cgemmt_LRN.$(SUFFIX) cgemmt_LRT.$(SUFFIX) cgemmt_LRR.$(SUFFIX) cgemmt_LRC.$(SUFFIX) \ + cgemmt_LCN.$(SUFFIX) cgemmt_LCT.$(SUFFIX) cgemmt_LCR.$(SUFFIX) cgemmt_LCC.$(SUFFIX) \ + cgemmt_kernel_UCN.$(SUFFIX) cgemmt_kernel_UNC.$(SUFFIX) cgemmt_kernel_UCC.$(SUFFIX) cgemmt_kernel_LCN.$(SUFFIX) \ + cgemmt_kernel_LNC.$(SUFFIX) cgemmt_kernel_LCC.$(SUFFIX) + +ZBLASOBJS += \ + zgemmt_UNN.$(SUFFIX) zgemmt_UNT.$(SUFFIX) zgemmt_UNR.$(SUFFIX) zgemmt_UNC.$(SUFFIX) \ + zgemmt_UTN.$(SUFFIX) zgemmt_UTT.$(SUFFIX) zgemmt_UTR.$(SUFFIX) zgemmt_UTC.$(SUFFIX) \ + zgemmt_URN.$(SUFFIX) zgemmt_URT.$(SUFFIX) zgemmt_URR.$(SUFFIX) zgemmt_URC.$(SUFFIX) \ + zgemmt_UCN.$(SUFFIX) zgemmt_UCT.$(SUFFIX) zgemmt_UCR.$(SUFFIX) zgemmt_UCC.$(SUFFIX) \ + zgemmt_LNN.$(SUFFIX) zgemmt_LNT.$(SUFFIX) zgemmt_LNR.$(SUFFIX) zgemmt_LNC.$(SUFFIX) \ + zgemmt_LTN.$(SUFFIX) zgemmt_LTT.$(SUFFIX) zgemmt_LTR.$(SUFFIX) zgemmt_LTC.$(SUFFIX) \ + zgemmt_LRN.$(SUFFIX) zgemmt_LRT.$(SUFFIX) zgemmt_LRR.$(SUFFIX) zgemmt_LRC.$(SUFFIX) \ + zgemmt_LCN.$(SUFFIX) zgemmt_LCT.$(SUFFIX) zgemmt_LCR.$(SUFFIX) zgemmt_LCC.$(SUFFIX) \ + zgemmt_kernel_UCN.$(SUFFIX) zgemmt_kernel_UNC.$(SUFFIX) zgemmt_kernel_UCC.$(SUFFIX) zgemmt_kernel_LCN.$(SUFFIX) \ + zgemmt_kernel_LNC.$(SUFFIX) zgemmt_kernel_LCC.$(SUFFIX) + +XBLASOBJS += \ + xgemmt_UNN.$(SUFFIX) xgemmt_UNT.$(SUFFIX) xgemmt_UNR.$(SUFFIX) xgemmt_UNC.$(SUFFIX) \ + xgemmt_UTN.$(SUFFIX) xgemmt_UTT.$(SUFFIX) xgemmt_UTR.$(SUFFIX) xgemmt_UTC.$(SUFFIX) \ + xgemmt_URN.$(SUFFIX) xgemmt_URT.$(SUFFIX) xgemmt_URR.$(SUFFIX) xgemmt_URC.$(SUFFIX) \ + xgemmt_UCN.$(SUFFIX) xgemmt_UCT.$(SUFFIX) xgemmt_UCR.$(SUFFIX) xgemmt_UCC.$(SUFFIX) \ + xgemmt_LNN.$(SUFFIX) xgemmt_LNT.$(SUFFIX) xgemmt_LNR.$(SUFFIX) xgemmt_LNC.$(SUFFIX) \ + xgemmt_LTN.$(SUFFIX) xgemmt_LTT.$(SUFFIX) xgemmt_LTR.$(SUFFIX) xgemmt_LTC.$(SUFFIX) \ + xgemmt_LRN.$(SUFFIX) xgemmt_LRT.$(SUFFIX) xgemmt_LRR.$(SUFFIX) xgemmt_LRC.$(SUFFIX) \ + xgemmt_LCN.$(SUFFIX) xgemmt_LCT.$(SUFFIX) xgemmt_LCR.$(SUFFIX) xgemmt_LCC.$(SUFFIX) \ + xgemmt_kernel_UCN.$(SUFFIX) xgemmt_kernel_UNC.$(SUFFIX) xgemmt_kernel_UCC.$(SUFFIX) xgemmt_kernel_LCN.$(SUFFIX) \ + xgemmt_kernel_LNC.$(SUFFIX) xgemmt_kernel_LCC.$(SUFFIX) + ifeq ($(USE_GEMM3M), 1) CBLASOBJS += \ @@ -1587,6 +1638,421 @@ xsyrk_kernel_U.$(SUFFIX) : syrk_kernel.c xsyrk_kernel_L.$(SUFFIX) : syrk_kernel.c $(CC) -c $(CFLAGS) -DXDOUBLE -DCOMPLEX -DLOWER $< -o $(@F) + +sgemmt_UNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -ULOWER -DNN $< -o $(@F) + +sgemmt_UNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -ULOWER -DNT $< -o $(@F) + +sgemmt_UTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -ULOWER -DTN $< -o $(@F) + +sgemmt_UTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -ULOWER -DTT $< -o $(@F) + +sgemmt_LNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -DLOWER -DNN $< -o $(@F) + +sgemmt_LNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -DLOWER -DNT $< -o $(@F) + +sgemmt_LTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -DLOWER -DTN $< -o $(@F) + +sgemmt_LTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -DLOWER -DTT $< -o $(@F) + +dgemmt_UNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -ULOWER -DNN $< -o $(@F) + +dgemmt_UNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -ULOWER -DNT $< -o $(@F) + +dgemmt_UTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -ULOWER -DTN $< -o $(@F) + +dgemmt_UTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -ULOWER -DTT $< -o $(@F) + +dgemmt_LNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -DLOWER -DNN $< -o $(@F) + +dgemmt_LNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -DLOWER -DNT $< -o $(@F) + +dgemmt_LTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -DLOWER -DTN $< -o $(@F) + +dgemmt_LTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -DLOWER -DTT $< -o $(@F) + +qgemmt_UNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -ULOWER -DNN $< -o $(@F) + +qgemmt_UNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -ULOWER -DNT $< -o $(@F) + +qgemmt_UTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -ULOWER -DTN $< -o $(@F) + +qgemmt_UTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -ULOWER -DTT $< -o $(@F) + +qgemmt_LNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -DLOWER -DNN $< -o $(@F) + +qgemmt_LNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -DLOWER -DNT $< -o $(@F) + +qgemmt_LTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -DLOWER -DTN $< -o $(@F) + +qgemmt_LTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -DLOWER -DTT $< -o $(@F) + +cgemmt_UNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DNN $< -o $(@F) + +cgemmt_UNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DNT $< -o $(@F) + +cgemmt_UNR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DNR $< -o $(@F) + +cgemmt_UNC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DNC $< -o $(@F) + +cgemmt_UTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DTN $< -o $(@F) + +cgemmt_UTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DTT $< -o $(@F) + +cgemmt_UTR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DTR $< -o $(@F) + +cgemmt_UTC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DTC $< -o $(@F) + +cgemmt_URN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DRN $< -o $(@F) + +cgemmt_URT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DRT $< -o $(@F) + +cgemmt_URR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DRR $< -o $(@F) + +cgemmt_URC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DRC=RC $< -o $(@F) + +cgemmt_UCN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCN $< -o $(@F) + +cgemmt_UCT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCT $< -o $(@F) + +cgemmt_UCR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCR=CR $< -o $(@F) + +cgemmt_UCC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCC $< -o $(@F) + +cgemmt_LNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DNN $< -o $(@F) + +cgemmt_LNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DNT $< -o $(@F) + +cgemmt_LNR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DNR $< -o $(@F) + +cgemmt_LNC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DNC $< -o $(@F) + +cgemmt_LTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DTN $< -o $(@F) + +cgemmt_LTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DTT $< -o $(@F) + +cgemmt_LTR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DTR $< -o $(@F) + +cgemmt_LTC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DTC $< -o $(@F) + +cgemmt_LRN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DRN $< -o $(@F) + +cgemmt_LRT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DRT $< -o $(@F) + +cgemmt_LRR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DRR $< -o $(@F) + +cgemmt_LRC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DRC=RC $< -o $(@F) + +cgemmt_LCN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCN $< -o $(@F) + +cgemmt_LCT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCT $< -o $(@F) + +cgemmt_LCR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCR=CR $< -o $(@F) + +cgemmt_LCC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCC $< -o $(@F) + +cgemmt_kernel_UCN.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -UDOUBLE -DCOMPLEX -ULOWER -DCONJA -UCONJB $< -o $(@F) + +cgemmt_kernel_UNC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -UDOUBLE -DCOMPLEX -ULOWER -UCONJA -DCONJB $< -o $(@F) + +cgemmt_kernel_UCC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -UDOUBLE -DCOMPLEX -ULOWER -DCONJA -DCONJB $< -o $(@F) + +cgemmt_kernel_LCN.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -UDOUBLE -DCOMPLEX -DLOWER -DCONJA -UCONJB $< -o $(@F) + +cgemmt_kernel_LNC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -UDOUBLE -DCOMPLEX -DLOWER -UCONJA -DCONJB $< -o $(@F) + +cgemmt_kernel_LCC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -UDOUBLE -DCOMPLEX -DLOWER -DCONJA -DCONJB $< -o $(@F) + +zgemmt_UNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DNN $< -o $(@F) + +zgemmt_UNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DNT $< -o $(@F) + +zgemmt_UNR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DNR $< -o $(@F) + +zgemmt_UNC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DNC $< -o $(@F) + +zgemmt_UTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DTN $< -o $(@F) + +zgemmt_UTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DTT $< -o $(@F) + +zgemmt_UTR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DTR $< -o $(@F) + +zgemmt_UTC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DTC $< -o $(@F) + +zgemmt_URN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DRN $< -o $(@F) + +zgemmt_URT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DRT $< -o $(@F) + +zgemmt_URR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DRR $< -o $(@F) + +zgemmt_URC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DRC=RC $< -o $(@F) + +zgemmt_UCN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCN $< -o $(@F) + +zgemmt_UCT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCT $< -o $(@F) + +zgemmt_UCR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCR=CR $< -o $(@F) + +zgemmt_UCC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCC $< -o $(@F) + +zgemmt_LNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DNN $< -o $(@F) + +zgemmt_LNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DNT $< -o $(@F) + +zgemmt_LNR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DNR $< -o $(@F) + +zgemmt_LNC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DNC $< -o $(@F) + +zgemmt_LTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DTN $< -o $(@F) + +zgemmt_LTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DTT $< -o $(@F) + +zgemmt_LTR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DTR $< -o $(@F) + +zgemmt_LTC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DTC $< -o $(@F) + +zgemmt_LRN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DRN $< -o $(@F) + +zgemmt_LRT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DRT $< -o $(@F) + +zgemmt_LRR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DRR $< -o $(@F) + +zgemmt_LRC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DRC=RC $< -o $(@F) + +zgemmt_LCN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCN $< -o $(@F) + +zgemmt_LCT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCT $< -o $(@F) + +zgemmt_LCR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCR=CR $< -o $(@F) + +zgemmt_LCC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCC $< -o $(@F) + +zgemmt_kernel_UCN.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DDOUBLE -DCOMPLEX -ULOWER -DCONJA -UCONJB $< -o $(@F) + +zgemmt_kernel_UNC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DDOUBLE -DCOMPLEX -ULOWER -UCONJA -DCONJB $< -o $(@F) + +zgemmt_kernel_UCC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DDOUBLE -DCOMPLEX -ULOWER -DCONJA -DCONJB $< -o $(@F) + +zgemmt_kernel_LCN.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DDOUBLE -DCOMPLEX -DLOWER -DCONJA -UCONJB $< -o $(@F) + +zgemmt_kernel_LNC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DDOUBLE -DCOMPLEX -DLOWER -UCONJA -DCONJB $< -o $(@F) + +zgemmt_kernel_LCC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DDOUBLE -DCOMPLEX -DLOWER -DCONJA -DCONJB $< -o $(@F) + +xgemmt_UNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DNN $< -o $(@F) + +xgemmt_UNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DNT $< -o $(@F) + +xgemmt_UNR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DNR $< -o $(@F) + +xgemmt_UNC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DNC $< -o $(@F) + +xgemmt_UTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DTN $< -o $(@F) + +xgemmt_UTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DTT $< -o $(@F) + +xgemmt_UTR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DTR $< -o $(@F) + +xgemmt_UTC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DTC $< -o $(@F) + +xgemmt_URN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DRN $< -o $(@F) + +xgemmt_URT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DRT $< -o $(@F) + +xgemmt_URR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DRR $< -o $(@F) + +xgemmt_URC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DRC=RC $< -o $(@F) + +xgemmt_UCN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCN $< -o $(@F) + +xgemmt_UCT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCT $< -o $(@F) + +xgemmt_UCR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCR=CR $< -o $(@F) + +xgemmt_UCC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCC $< -o $(@F) + +xgemmt_LNN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DNN $< -o $(@F) + +xgemmt_LNT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DNT $< -o $(@F) + +xgemmt_LNR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DNR $< -o $(@F) + +xgemmt_LNC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DNC $< -o $(@F) + +xgemmt_LTN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DTN $< -o $(@F) + +xgemmt_LTT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DTT $< -o $(@F) + +xgemmt_LTR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DTR $< -o $(@F) + +xgemmt_LTC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DTC $< -o $(@F) + +xgemmt_LRN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DRN $< -o $(@F) + +xgemmt_LRT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DRT $< -o $(@F) + +xgemmt_LRR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DRR $< -o $(@F) + +xgemmt_LRC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DRC=RC $< -o $(@F) + +xgemmt_LCN.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCN $< -o $(@F) + +xgemmt_LCT.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCT $< -o $(@F) + +xgemmt_LCR.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCR=CR $< -o $(@F) + +xgemmt_LCC.$(SUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(CFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCC $< -o $(@F) + +xgemmt_kernel_UCN.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DXDOUBLE -DCOMPLEX -ULOWER -DCONJA -UCONJB $< -o $(@F) + +xgemmt_kernel_UNC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DXDOUBLE -DCOMPLEX -ULOWER -UCONJA -DCONJB $< -o $(@F) + +xgemmt_kernel_UCC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DXDOUBLE -DCOMPLEX -ULOWER -DCONJA -DCONJB $< -o $(@F) + +xgemmt_kernel_LCN.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DXDOUBLE -DCOMPLEX -DLOWER -DCONJA -UCONJB $< -o $(@F) + +xgemmt_kernel_LNC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DXDOUBLE -DCOMPLEX -DLOWER -UCONJA -DCONJB $< -o $(@F) + +xgemmt_kernel_LCC.$(SUFFIX) : syrk_kernel.c + $(CC) -c $(CFLAGS) -DXDOUBLE -DCOMPLEX -DLOWER -DCONJA -DCONJB $< -o $(@F) + syrk_thread.$(SUFFIX) : syrk_thread.c ../../common.h $(CC) -c $(CFLAGS) $< -o $(@F) @@ -4022,6 +4488,421 @@ xsyrk_kernel_U.$(PSUFFIX) : syrk_kernel.c xsyrk_kernel_L.$(PSUFFIX) : syrk_kernel.c $(CC) -c $(PFLAGS) -DXDOUBLE -DCOMPLEX -DLOWER $< -o $(@F) + +sgemmt_UNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -ULOWER -DNN $< -o $(@F) + +sgemmt_UNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -ULOWER -DNT $< -o $(@F) + +sgemmt_UTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -ULOWER -DTN $< -o $(@F) + +sgemmt_UTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -ULOWER -DTT $< -o $(@F) + +sgemmt_LNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -DLOWER -DNN $< -o $(@F) + +sgemmt_LNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -DLOWER -DNT $< -o $(@F) + +sgemmt_LTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -DLOWER -DTN $< -o $(@F) + +sgemmt_LTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -UCOMPLEX -DLOWER -DTT $< -o $(@F) + +dgemmt_UNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -ULOWER -DNN $< -o $(@F) + +dgemmt_UNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -ULOWER -DNT $< -o $(@F) + +dgemmt_UTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -ULOWER -DTN $< -o $(@F) + +dgemmt_UTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -ULOWER -DTT $< -o $(@F) + +dgemmt_LNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -DLOWER -DNN $< -o $(@F) + +dgemmt_LNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -DLOWER -DNT $< -o $(@F) + +dgemmt_LTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -DLOWER -DTN $< -o $(@F) + +dgemmt_LTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -UCOMPLEX -DLOWER -DTT $< -o $(@F) + +qgemmt_UNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -ULOWER -DNN $< -o $(@F) + +qgemmt_UNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -ULOWER -DNT $< -o $(@F) + +qgemmt_UTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -ULOWER -DTN $< -o $(@F) + +qgemmt_UTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -ULOWER -DTT $< -o $(@F) + +qgemmt_LNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -DLOWER -DNN $< -o $(@F) + +qgemmt_LNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -DLOWER -DNT $< -o $(@F) + +qgemmt_LTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -DLOWER -DTN $< -o $(@F) + +qgemmt_LTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -UCOMPLEX -DLOWER -DTT $< -o $(@F) + +cgemmt_UNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DNN $< -o $(@F) + +cgemmt_UNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DNT $< -o $(@F) + +cgemmt_UNR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DNR $< -o $(@F) + +cgemmt_UNC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DNC $< -o $(@F) + +cgemmt_UTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DTN $< -o $(@F) + +cgemmt_UTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DTT $< -o $(@F) + +cgemmt_UTR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DTR $< -o $(@F) + +cgemmt_UTC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DTC $< -o $(@F) + +cgemmt_URN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DRN $< -o $(@F) + +cgemmt_URT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DRT $< -o $(@F) + +cgemmt_URR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DRR $< -o $(@F) + +cgemmt_URC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DRC=RC $< -o $(@F) + +cgemmt_UCN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCN $< -o $(@F) + +cgemmt_UCT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCT $< -o $(@F) + +cgemmt_UCR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCR=CR $< -o $(@F) + +cgemmt_UCC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -ULOWER -DCC $< -o $(@F) + +cgemmt_LNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DNN $< -o $(@F) + +cgemmt_LNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DNT $< -o $(@F) + +cgemmt_LNR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DNR $< -o $(@F) + +cgemmt_LNC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DNC $< -o $(@F) + +cgemmt_LTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DTN $< -o $(@F) + +cgemmt_LTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DTT $< -o $(@F) + +cgemmt_LTR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DTR $< -o $(@F) + +cgemmt_LTC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DTC $< -o $(@F) + +cgemmt_LRN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DRN $< -o $(@F) + +cgemmt_LRT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DRT $< -o $(@F) + +cgemmt_LRR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DRR $< -o $(@F) + +cgemmt_LRC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DRC=RC $< -o $(@F) + +cgemmt_LCN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCN $< -o $(@F) + +cgemmt_LCT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCT $< -o $(@F) + +cgemmt_LCR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCR=CR $< -o $(@F) + +cgemmt_LCC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -UDOUBLE -DCOMPLEX -DLOWER -DCC $< -o $(@F) + +cgemmt_kernel_UCN.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -UDOUBLE -DCOMPLEX -ULOWER -DCONJA -UCONJB $< -o $(@F) + +cgemmt_kernel_UNC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -UDOUBLE -DCOMPLEX -ULOWER -UCONJA -DCONJB $< -o $(@F) + +cgemmt_kernel_UCC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -UDOUBLE -DCOMPLEX -ULOWER -DCONJA -DCONJB $< -o $(@F) + +cgemmt_kernel_LCN.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -UDOUBLE -DCOMPLEX -DLOWER -DCONJA -UCONJB $< -o $(@F) + +cgemmt_kernel_LNC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -UDOUBLE -DCOMPLEX -DLOWER -UCONJA -DCONJB $< -o $(@F) + +cgemmt_kernel_LCC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -UDOUBLE -DCOMPLEX -DLOWER -DCONJA -DCONJB $< -o $(@F) + +zgemmt_UNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DNN $< -o $(@F) + +zgemmt_UNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DNT $< -o $(@F) + +zgemmt_UNR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DNR $< -o $(@F) + +zgemmt_UNC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DNC $< -o $(@F) + +zgemmt_UTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DTN $< -o $(@F) + +zgemmt_UTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DTT $< -o $(@F) + +zgemmt_UTR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DTR $< -o $(@F) + +zgemmt_UTC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DTC $< -o $(@F) + +zgemmt_URN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DRN $< -o $(@F) + +zgemmt_URT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DRT $< -o $(@F) + +zgemmt_URR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DRR $< -o $(@F) + +zgemmt_URC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DRC=RC $< -o $(@F) + +zgemmt_UCN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCN $< -o $(@F) + +zgemmt_UCT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCT $< -o $(@F) + +zgemmt_UCR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCR=CR $< -o $(@F) + +zgemmt_UCC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -ULOWER -DCC $< -o $(@F) + +zgemmt_LNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DNN $< -o $(@F) + +zgemmt_LNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DNT $< -o $(@F) + +zgemmt_LNR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DNR $< -o $(@F) + +zgemmt_LNC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DNC $< -o $(@F) + +zgemmt_LTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DTN $< -o $(@F) + +zgemmt_LTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DTT $< -o $(@F) + +zgemmt_LTR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DTR $< -o $(@F) + +zgemmt_LTC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DTC $< -o $(@F) + +zgemmt_LRN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DRN $< -o $(@F) + +zgemmt_LRT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DRT $< -o $(@F) + +zgemmt_LRR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DRR $< -o $(@F) + +zgemmt_LRC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DRC=RC $< -o $(@F) + +zgemmt_LCN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCN $< -o $(@F) + +zgemmt_LCT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCT $< -o $(@F) + +zgemmt_LCR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCR=CR $< -o $(@F) + +zgemmt_LCC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DDOUBLE -DCOMPLEX -DLOWER -DCC $< -o $(@F) + +zgemmt_kernel_UCN.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DDOUBLE -DCOMPLEX -ULOWER -DCONJA -UCONJB $< -o $(@F) + +zgemmt_kernel_UNC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DDOUBLE -DCOMPLEX -ULOWER -UCONJA -DCONJB $< -o $(@F) + +zgemmt_kernel_UCC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DDOUBLE -DCOMPLEX -ULOWER -DCONJA -DCONJB $< -o $(@F) + +zgemmt_kernel_LCN.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DDOUBLE -DCOMPLEX -DLOWER -DCONJA -UCONJB $< -o $(@F) + +zgemmt_kernel_LNC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DDOUBLE -DCOMPLEX -DLOWER -UCONJA -DCONJB $< -o $(@F) + +zgemmt_kernel_LCC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DDOUBLE -DCOMPLEX -DLOWER -DCONJA -DCONJB $< -o $(@F) + +xgemmt_UNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DNN $< -o $(@F) + +xgemmt_UNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DNT $< -o $(@F) + +xgemmt_UNR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DNR $< -o $(@F) + +xgemmt_UNC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DNC $< -o $(@F) + +xgemmt_UTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DTN $< -o $(@F) + +xgemmt_UTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DTT $< -o $(@F) + +xgemmt_UTR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DTR $< -o $(@F) + +xgemmt_UTC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DTC $< -o $(@F) + +xgemmt_URN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DRN $< -o $(@F) + +xgemmt_URT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DRT $< -o $(@F) + +xgemmt_URR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DRR $< -o $(@F) + +xgemmt_URC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DRC=RC $< -o $(@F) + +xgemmt_UCN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCN $< -o $(@F) + +xgemmt_UCT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCT $< -o $(@F) + +xgemmt_UCR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCR=CR $< -o $(@F) + +xgemmt_UCC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -ULOWER -DCC $< -o $(@F) + +xgemmt_LNN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DNN $< -o $(@F) + +xgemmt_LNT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DNT $< -o $(@F) + +xgemmt_LNR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DNR $< -o $(@F) + +xgemmt_LNC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DNC $< -o $(@F) + +xgemmt_LTN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DTN $< -o $(@F) + +xgemmt_LTT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DTT $< -o $(@F) + +xgemmt_LTR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DTR $< -o $(@F) + +xgemmt_LTC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DTC $< -o $(@F) + +xgemmt_LRN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DRN $< -o $(@F) + +xgemmt_LRT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DRT $< -o $(@F) + +xgemmt_LRR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DRR $< -o $(@F) + +xgemmt_LRC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DRC=RC $< -o $(@F) + +xgemmt_LCN.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCN $< -o $(@F) + +xgemmt_LCT.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCT $< -o $(@F) + +xgemmt_LCR.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCR=CR $< -o $(@F) + +xgemmt_LCC.$(PSUFFIX) : gemmt_k.c level3_gemmt.c ../../param.h + $(CC) -c $(PFLAGS) $(BLOCKS) -DXDOUBLE -DCOMPLEX -DLOWER -DCC $< -o $(@F) + +xgemmt_kernel_UCN.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DXDOUBLE -DCOMPLEX -ULOWER -DCONJA -UCONJB $< -o $(@F) + +xgemmt_kernel_UNC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DXDOUBLE -DCOMPLEX -ULOWER -UCONJA -DCONJB $< -o $(@F) + +xgemmt_kernel_UCC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DXDOUBLE -DCOMPLEX -ULOWER -DCONJA -DCONJB $< -o $(@F) + +xgemmt_kernel_LCN.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DXDOUBLE -DCOMPLEX -DLOWER -DCONJA -UCONJB $< -o $(@F) + +xgemmt_kernel_LNC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DXDOUBLE -DCOMPLEX -DLOWER -UCONJA -DCONJB $< -o $(@F) + +xgemmt_kernel_LCC.$(PSUFFIX) : syrk_kernel.c + $(CC) -c $(PFLAGS) -DXDOUBLE -DCOMPLEX -DLOWER -DCONJA -DCONJB $< -o $(@F) + syrk_thread.$(PSUFFIX) : syrk_thread.c ../../common.h $(CC) -c $(PFLAGS) $< -o $(@F) diff --git a/driver/level3/gemmt_k.c b/driver/level3/gemmt_k.c new file mode 100644 index 0000000000..e94168bf6c --- /dev/null +++ b/driver/level3/gemmt_k.c @@ -0,0 +1,127 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ + +#include +#include "common.h" + +/* Scale the part of C that GEMMT actually references by beta. This is + the same operation as syrk_beta() in syrk_k.c. */ +static __inline int gemmt_beta(BLASLONG m_from, BLASLONG m_to, BLASLONG n_from, BLASLONG n_to, + FLOAT *alpha, FLOAT *c, BLASLONG ldc) { + + BLASLONG i; + +#ifndef LOWER + if (m_from > n_from) n_from = m_from; + if (m_to > n_to ) m_to = n_to; +#else + if (m_from < n_from) m_from = n_from; + if (m_to < n_to ) n_to = m_to; +#endif + + if ((m_to <= m_from) || (n_to <= n_from)) return 0; + + c += (m_from + n_from * ldc) * COMPSIZE; + + m_to -= m_from; + n_to -= n_from; + + for (i = 0; i < n_to; i++){ + +#ifndef LOWER + + SCAL_K(MIN(i + n_from - m_from + 1, m_to), 0, 0, alpha[0], +#ifdef COMPLEX + alpha[1], +#endif + c, 1, NULL, 0, NULL, 0); + + c += ldc * COMPSIZE; + +#else + + SCAL_K(MIN(m_to - i + m_from - n_from, m_to), 0, 0, alpha[0], +#ifdef COMPLEX + alpha[1], +#endif + c, 1, NULL, 0, NULL, 0); + + if (i < m_from - n_from) { + c += ldc * COMPSIZE; + } else { + c += (1 + ldc) * COMPSIZE; + } +#endif + + } + + return 0; +} + +#define GEMMT_BETA(M_FROM, M_TO, N_FROM, N_TO, BETA, C, LDC) \ + gemmt_beta(M_FROM, M_TO, N_FROM, N_TO, BETA, C, LDC) + +/* The blocks on the diagonal are computed by the SYRK kernel, which + clips a block against the diagonal; the conjugation variants needed + for complex GEMMT are the same file built with CONJA / CONJB. */ +#ifndef KERNEL_FUNC +#ifndef LOWER +#if defined(NN) || defined(NT) || defined(TN) || defined(TT) +#define KERNEL_FUNC SYRK_KERNEL_U +#elif defined(CN) || defined(CT) || defined(RN) || defined(RT) +#define KERNEL_FUNC GEMMT_KERNEL_UCN +#elif defined(NC) || defined(TC) || defined(NR) || defined(TR) +#define KERNEL_FUNC GEMMT_KERNEL_UNC +#else +#define KERNEL_FUNC GEMMT_KERNEL_UCC +#endif +#else +#if defined(NN) || defined(NT) || defined(TN) || defined(TT) +#define KERNEL_FUNC SYRK_KERNEL_L +#elif defined(CN) || defined(CT) || defined(RN) || defined(RT) +#define KERNEL_FUNC GEMMT_KERNEL_LCN +#elif defined(NC) || defined(TC) || defined(NR) || defined(TR) +#define KERNEL_FUNC GEMMT_KERNEL_LNC +#else +#define KERNEL_FUNC GEMMT_KERNEL_LCC +#endif +#endif +#endif + +/* GEMMT is threaded by splitting the triangle into equal-area column + ranges (syrk_thread) and running this driver on each of them, so there + is no separate THREADED_LEVEL3 variant. */ +#include "level3_gemmt.c" diff --git a/driver/level3/level3_gemmt.c b/driver/level3/level3_gemmt.c new file mode 100644 index 0000000000..ad42228cad --- /dev/null +++ b/driver/level3/level3_gemmt.c @@ -0,0 +1,508 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#ifndef KERNEL_OPERATION +#ifndef COMPLEX +#define KERNEL_OPERATION(M, N, K, ALPHA, SA, SB, C, LDC, X, Y) \ + KERNEL_FUNC(M, N, K, ALPHA[0], SA, SB, (FLOAT *)(C) + ((X) + (Y) * LDC) * COMPSIZE, LDC, (X) - (Y)) +#else +#define KERNEL_OPERATION(M, N, K, ALPHA, SA, SB, C, LDC, X, Y) \ + KERNEL_FUNC(M, N, K, ALPHA[0], ALPHA[1], SA, SB, (FLOAT *)(C) + ((X) + (Y) * LDC) * COMPSIZE, LDC, (X) - (Y)) +#endif +#endif + +#ifndef ICOPY_OPERATION +#if defined(NN) || defined(NT) || defined(NC) || defined(NR) || \ + defined(RN) || defined(RT) || defined(RC) || defined(RR) +#define ICOPY_OPERATION(M, N, A, LDA, X, Y, BUFFER) GEMM_ITCOPY(M, N, (FLOAT *)(A) + ((Y) + (X) * (LDA)) * COMPSIZE, LDA, BUFFER); +#else +#define ICOPY_OPERATION(M, N, A, LDA, X, Y, BUFFER) GEMM_INCOPY(M, N, (FLOAT *)(A) + ((X) + (Y) * (LDA)) * COMPSIZE, LDA, BUFFER); +#endif +#endif + +#ifndef OCOPY_OPERATION +#if defined(NN) || defined(TN) || defined(CN) || defined(RN) || \ + defined(NR) || defined(TR) || defined(CR) || defined(RR) +#define OCOPY_OPERATION(M, N, A, LDA, X, Y, BUFFER) GEMM_ONCOPY(M, N, (FLOAT *)(A) + ((X) + (Y) * (LDA)) * COMPSIZE, LDA, BUFFER); +#else +#define OCOPY_OPERATION(M, N, A, LDA, X, Y, BUFFER) GEMM_OTCOPY(M, N, (FLOAT *)(A) + ((Y) + (X) * (LDA)) * COMPSIZE, LDA, BUFFER); +#endif +#endif + +#ifndef M +#define M args -> n +#endif + +#ifndef N +#define N args -> n +#endif + +#ifndef K +#define K args -> k +#endif + +#ifndef A +#define A args -> a +#endif + +#ifndef B +#define B args -> b +#endif + +#ifndef C +#define C args -> c +#endif + +#ifndef LDA +#define LDA args -> lda +#endif + +#ifndef LDB +#define LDB args -> ldb +#endif + +#ifndef LDC +#define LDC args -> ldc +#endif + +#ifdef TIMING +#define START_RPCC() rpcc_counter = rpcc() +#define STOP_RPCC(COUNTER) COUNTER += rpcc() - rpcc_counter +#else +#define START_RPCC() +#define STOP_RPCC(COUNTER) +#endif + +/* Single-threaded GEMMT / GEMMTR driver: + + C := alpha * op(A) * op(B) + beta * C + + with only the triangle of the n by n matrix C selected by LOWER + referenced and updated. Derived from level3_syrk.c: the blocking, + the diagonal handling (SYRK_KERNEL clips the blocks that cross the + diagonal) and the loop structure are the same, but the packed B + buffer is filled from a second matrix instead of from A, so the + buffer sharing that SYRK can do is not available here. */ + +int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLOAT *sb, BLASLONG dummy) { + + BLASLONG m_from, m_to, n_from, n_to, k, lda, ldb, ldc; + FLOAT *a, *b, *c, *alpha, *beta; + + BLASLONG ls, is, js; + BLASLONG min_l, min_i, min_j; + BLASLONG jjs, min_jj; + BLASLONG m_start, m_end; + +#ifdef LOWER + FLOAT *aa; +#endif + +#ifdef TIMING + unsigned long long rpcc_counter; + unsigned long long innercost = 0; + unsigned long long outercost = 0; + unsigned long long kernelcost = 0; + double total; +#endif + + k = K; + + a = (FLOAT *)A; + b = (FLOAT *)B; + c = (FLOAT *)C; + + lda = LDA; + ldb = LDB; + ldc = LDC; + + alpha = (FLOAT *)args -> alpha; + beta = (FLOAT *)args -> beta; + + m_from = 0; + m_to = M; + + if (range_m) { + m_from = *(((BLASLONG *)range_m) + 0); + m_to = *(((BLASLONG *)range_m) + 1); + } + + n_from = 0; + n_to = N; + + if (range_n) { + n_from = *(((BLASLONG *)range_n) + 0); + n_to = *(((BLASLONG *)range_n) + 1); + } + + if (beta) { +#ifndef COMPLEX + if (beta[0] != ONE) +#else + if ((beta[0] != ONE) || (beta[1] != ZERO)) +#endif + GEMMT_BETA(m_from, m_to, n_from, n_to, beta, c, ldc); + } + + if ((k == 0) || (alpha == NULL)) return 0; + + if (alpha[0] == ZERO +#if defined(COMPLEX) && !defined(HERK) + && alpha[1] == ZERO +#endif + ) return 0; + +#if 0 + fprintf(stderr, "m_from : %ld m_to : %ld n_from : %ld n_to : %ld\n", + m_from, m_to, n_from, n_to); +#endif + + for(js = n_from; js < n_to; js += GEMM_R){ + min_j = n_to - js; + if (min_j > GEMM_R) min_j = GEMM_R; + +#ifndef LOWER + m_start = m_from; + m_end = js + min_j; + if (m_end > m_to) m_end = m_to; +#else + m_start = m_from; + m_end = m_to; + if (m_start < js) m_start = js; +#endif + + for(ls = 0; ls < k; ls += min_l){ + min_l = k - ls; + if (min_l >= GEMM_Q * 2) { + min_l = GEMM_Q; + } else + if (min_l > GEMM_Q) { + min_l = (min_l + 1) / 2; + } + + min_i = m_end - m_start; + + if (min_i >= GEMM_P * 2) { + min_i = GEMM_P; + } else + if (min_i > GEMM_P) { + min_i = ((min_i / 2 + GEMM_UNROLL_MN - 1)/GEMM_UNROLL_MN) * GEMM_UNROLL_MN; + } + +#ifndef LOWER + + if (m_end >= js) { + + /* A is packed in the same GEMM_UNROLL_MN chunks in which the kernel + consumes it: a block that crosses the diagonal is cut down to + min_jj rows there, and the packed layout of a panel depends on + how many rows were handed to the copy routine. Columns to the + left of this row band are below the diagonal and stay unpacked; + the row bands further down skip them via the kernel offset. */ + for(jjs = MAX(m_start, js); jjs < js + min_j; jjs += min_jj){ + min_jj = js + min_j - jjs; + if (min_jj > GEMM_UNROLL_MN) min_jj = GEMM_UNROLL_MN; + + if (jjs - MAX(m_start, js) < min_i) { + START_RPCC(); + + ICOPY_OPERATION(min_l, min_jj, a, lda, ls, jjs, sa + min_l * (jjs - MAX(m_start, js)) * COMPSIZE); + + STOP_RPCC(innercost); + } + + START_RPCC(); + + OCOPY_OPERATION(min_l, min_jj, b, ldb, ls, jjs, sb + min_l * (jjs - js) * COMPSIZE); + + STOP_RPCC(outercost); + + START_RPCC(); + + KERNEL_OPERATION(min_i, min_jj, min_l, alpha, sa, sb + min_l * (jjs - js) * COMPSIZE, c, ldc, MAX(m_start, js), jjs); + + STOP_RPCC(kernelcost); + } + + for(is = MAX(m_start, js) + min_i; is < m_end; is += min_i){ + min_i = m_end - is; + if (min_i >= GEMM_P * 2) { + min_i = GEMM_P; + } else + if (min_i > GEMM_P) { + min_i = ((min_i / 2 + GEMM_UNROLL_MN - 1)/GEMM_UNROLL_MN) * GEMM_UNROLL_MN; + } + + START_RPCC(); + + ICOPY_OPERATION(min_l, min_i, a, lda, ls, is, sa); + + STOP_RPCC(innercost); + + START_RPCC(); + + KERNEL_OPERATION(min_i, min_j, min_l, alpha, sa, sb, c, ldc, is, js); + + STOP_RPCC(kernelcost); + + } + + } + + if (m_start < js) { + + if (m_end < js) { + + START_RPCC(); + + ICOPY_OPERATION(min_l, min_i, a, lda, ls, m_start, sa); + + STOP_RPCC(innercost); + + for(jjs = js; jjs < js + min_j; jjs += GEMM_UNROLL_MN){ + min_jj = min_j + js - jjs; + if (min_jj > GEMM_UNROLL_MN) min_jj = GEMM_UNROLL_MN; + + START_RPCC(); + + OCOPY_OPERATION(min_l, min_jj, b, ldb, ls, jjs, sb + min_l * (jjs - js) * COMPSIZE); + + STOP_RPCC(outercost); + + START_RPCC(); + + KERNEL_OPERATION(min_i, min_jj, min_l, alpha, sa, sb + min_l * (jjs - js) * COMPSIZE, c, ldc, m_start, jjs); + + STOP_RPCC(kernelcost); + + } + } else { + min_i = 0; + } + + for(is = m_start + min_i; is < MIN(m_end, js); is += min_i){ + + min_i = MIN(m_end, js)- is; + if (min_i >= GEMM_P * 2) { + min_i = GEMM_P; + } else + if (min_i > GEMM_P) { + min_i = ((min_i / 2 + GEMM_UNROLL_MN - 1)/GEMM_UNROLL_MN) * GEMM_UNROLL_MN; + } + + START_RPCC(); + + ICOPY_OPERATION(min_l, min_i, a, lda, ls, is, sa); + + STOP_RPCC(innercost); + + START_RPCC(); + + KERNEL_OPERATION(min_i, min_j, min_l, alpha, sa, sb, c, ldc, is, js); + + STOP_RPCC(kernelcost); + + } + } + +#else + + if (m_start < js + min_j) { + + aa = sb + min_l * (m_start - js) * COMPSIZE; + + START_RPCC(); + + ICOPY_OPERATION(min_l, min_i, a, lda, ls, m_start, sa); + + STOP_RPCC(innercost); + + START_RPCC(); + + OCOPY_OPERATION(min_l, MIN(min_i, min_j + js - m_start), b, ldb, ls, m_start, aa); + + STOP_RPCC(outercost); + + START_RPCC(); + + KERNEL_OPERATION(min_i, MIN(min_i, min_j + js - m_start), min_l, alpha, sa, aa, c, ldc, m_start, m_start); + + STOP_RPCC(kernelcost); + + for(jjs = js; jjs < m_start; jjs += GEMM_UNROLL_N){ + min_jj = m_start - jjs; + if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N; + + START_RPCC(); + + OCOPY_OPERATION(min_l, min_jj, b, ldb, ls, jjs, sb + min_l * (jjs - js) * COMPSIZE); + + STOP_RPCC(outercost); + + START_RPCC(); + + KERNEL_OPERATION(min_i, min_jj, min_l, alpha, sa, sb + min_l * (jjs - js) * COMPSIZE, c, ldc, m_start, jjs); + + STOP_RPCC(kernelcost); + + } + + for(is = m_start + min_i; is < m_end; is += min_i){ + + min_i = m_end - is; + + if (min_i >= GEMM_P * 2) { + min_i = GEMM_P; + } else + if (min_i > GEMM_P) { + min_i = ((min_i / 2 + GEMM_UNROLL_MN - 1)/GEMM_UNROLL_MN) * GEMM_UNROLL_MN; + } + + if (is < js + min_j) { + + START_RPCC(); + + ICOPY_OPERATION(min_l, min_i, a, lda, ls, is, sa); + + STOP_RPCC(innercost); + + aa = sb + min_l * (is - js) * COMPSIZE; + + START_RPCC(); + + OCOPY_OPERATION(min_l, MIN(min_i, min_j - is + js), b, ldb, ls, is, aa); + + STOP_RPCC(outercost); + + START_RPCC(); + + KERNEL_OPERATION(min_i, MIN(min_i, min_j - is + js), min_l, alpha, sa, aa, c, ldc, is, is); + + STOP_RPCC(kernelcost); + + START_RPCC(); + + KERNEL_OPERATION(min_i, is - js, min_l, alpha, sa, sb, c, ldc, is, js); + + STOP_RPCC(kernelcost); + + } else { + + START_RPCC(); + + ICOPY_OPERATION(min_l, min_i, a, lda, ls, is, sa); + + STOP_RPCC(innercost); + + START_RPCC(); + + KERNEL_OPERATION(min_i, min_j, min_l, alpha, sa, sb, c, ldc, is, js); + + STOP_RPCC(kernelcost); + + } + + } + + } else { + + START_RPCC(); + + ICOPY_OPERATION(min_l, min_i, a, lda, ls, m_start, sa); + + STOP_RPCC(innercost); + + for(jjs = js; jjs < js + min_j; jjs += GEMM_UNROLL_N){ + min_jj = js + min_j - jjs; + if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N; + + START_RPCC(); + + OCOPY_OPERATION(min_l, min_jj, b, ldb, ls, jjs, sb + min_l * (jjs - js) * COMPSIZE); + + STOP_RPCC(outercost); + + START_RPCC(); + + KERNEL_OPERATION(min_i, min_jj, min_l, alpha, sa, sb + min_l * (jjs - js) * COMPSIZE, c, ldc, m_start, jjs); + + STOP_RPCC(kernelcost); + + } + + for(is = m_start + min_i; is < m_end; is += min_i){ + + min_i = m_end - is; + + if (min_i >= GEMM_P * 2) { + min_i = GEMM_P; + } else + if (min_i > GEMM_P) { + min_i = ((min_i / 2 + GEMM_UNROLL_MN - 1)/GEMM_UNROLL_MN) * GEMM_UNROLL_MN; + } + + START_RPCC(); + + ICOPY_OPERATION(min_l, min_i, a, lda, ls, is, sa); + + STOP_RPCC(innercost); + + START_RPCC(); + + KERNEL_OPERATION(min_i, min_j, min_l, alpha, sa, sb, c, ldc, is, js); + + STOP_RPCC(kernelcost); + + } + } +#endif + } + } + +#ifdef TIMING + total = (double)outercost + (double)innercost + (double)kernelcost; + + printf( "Copy A : %5.2f Copy B: %5.2f Kernel : %5.2f kernel Effi. : %5.2f Total Effi. : %5.2f\n", + innercost / total * 100., outercost / total * 100., kernelcost / total * 100., + (double)(m_to - m_from) * (double)(n_to - n_from) * (double)k / (double)kernelcost * 100. * (double)COMPSIZE / (double)DNUMOPT, + (double)(m_to - m_from) * (double)(n_to - n_from) * (double)k / total * 100. * (double)COMPSIZE / (double)DNUMOPT); + +#endif + + return 0; +} diff --git a/interface/gemmt.c b/interface/gemmt.c index 39f41ae1c2..847d36ad3d 100644 --- a/interface/gemmt.c +++ b/interface/gemmt.c @@ -102,15 +102,7 @@ void NAME(char *UPLO, char *TRANSA, char *TRANSB, char transA, transB, Uplo; blasint nrowa, nrowb; -#if defined(COMPLEX) - blasint ncolb; -#endif - IFLOAT *buffer; - IFLOAT *aa, *bb; - FLOAT *cc; -#if defined(COMPLEX) - FLOAT alpha_r, alpha_i, beta_r, beta_i; -#else +#if !defined(COMPLEX) FLOAT alpha, beta; #endif @@ -121,11 +113,7 @@ void NAME(char *UPLO, char *TRANSA, char *TRANSB, #if defined(COMPLEX) FLOAT *alpha = Alpha; - alpha_r = *(Alpha + 0); - alpha_i = *(Alpha + 1); - - beta_r = *(Beta + 0); - beta_i = *(Beta + 1); + FLOAT *beta = Beta; #else alpha = *Alpha; beta = *Beta; @@ -186,15 +174,7 @@ void NAME(char *UPLO, char *TRANSA, char *TRANSB, nrowa = m; if (transa & 1) nrowa = k; nrowb = k; -#if defined(COMPLEX) - ncolb = m; -#endif - if (transb & 1) { - nrowb = m; -#if defined(COMPLEX) - ncolb = k; -#endif - } + if (transb & 1) nrowb = m; info = 0; @@ -240,16 +220,10 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_UPLO Uplo, FLOAT *B = (FLOAT *) vb; FLOAT *c = (FLOAT *) vc; #endif - FLOAT *aa, *bb, *cc; - int transa, transb, uplo; blasint info; blasint lda, ldb; FLOAT *a, *b; -#if defined(COMPLEX) - blasint nrowb, ncolb; -#endif - XFLOAT *buffer; PRINT_DEBUG_CNAME; @@ -300,22 +274,12 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_UPLO Uplo, info = -1; - blasint nrowa; -#if !defined(COMPLEX) - blasint nrowb; -#endif + blasint nrowa, nrowb; + nrowa = m; if (transa & 1) nrowa = k; nrowb = k; -#if defined(COMPLEX) - ncolb = m; -#endif - if (transb & 1) { - nrowb = m; -#if defined(COMPLEX) - ncolb = k; -#endif - } + if (transb & 1) nrowb = m; if (ldc < MAX(1, m)) info = 13; @@ -379,23 +343,12 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_UPLO Uplo, info = -1; - blasint ncola; -#if !defined(COMPLEX) - blasint ncolb; -#endif + blasint ncola, ncolb; + ncola = m; if (transa & 1) ncola = k; ncolb = k; -#if defined(COMPLEX) - nrowb = m; -#endif - - if (transb & 1) { -#if defined(COMPLEX) - nrowb = k; -#endif - ncolb = m; - } + if (transb & 1) ncolb = m; if (ldc < MAX(1,m)) info = 13; @@ -419,290 +372,129 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_UPLO Uplo, BLASFUNC(xerbla) (ERROR_NAME, &info, sizeof(ERROR_NAME)); return; } -#if defined(COMPLEX) - FLOAT alpha_r = *(alpha + 0); - FLOAT alpha_i = *(alpha + 1); - - FLOAT beta_r = *(beta + 0); - FLOAT beta_i = *(beta + 1); #endif - -#endif - int buffer_size; - blasint l; - blasint i, j; - -#ifdef SMP - int nthreads; +#ifndef COMPLEX +#define GEMMT_TABLE_INDEX ((uplo << 2) | (transb << 1) | transa) +#else +#define GEMMT_TABLE_INDEX ((uplo << 4) | (transb << 2) | transa) #endif -#if defined(COMPLEX) + blas_arg_t args; + FLOAT *buffer, *sa, *sb; #ifdef SMP - static int (*gemv_thread[]) (BLASLONG, BLASLONG, FLOAT *, FLOAT *, - BLASLONG, FLOAT *, BLASLONG, FLOAT *, - BLASLONG, FLOAT *, int) = { + double MMK; +#ifndef COMPLEX #ifdef XDOUBLE - xgemv_thread_n, xgemv_thread_t, xgemv_thread_r, xgemv_thread_c, - xgemv_thread_o, xgemv_thread_u, xgemv_thread_s, - xgemv_thread_d, -#elif defined DOUBLE - zgemv_thread_n, zgemv_thread_t, zgemv_thread_r, zgemv_thread_c, - zgemv_thread_o, zgemv_thread_u, zgemv_thread_s, - zgemv_thread_d, + int mode = BLAS_XDOUBLE | BLAS_REAL; +#elif defined(DOUBLE) + int mode = BLAS_DOUBLE | BLAS_REAL; #else - cgemv_thread_n, cgemv_thread_t, cgemv_thread_r, cgemv_thread_c, - cgemv_thread_o, cgemv_thread_u, cgemv_thread_s, - cgemv_thread_d, -#endif - }; + int mode = BLAS_SINGLE | BLAS_REAL; #endif - - int (*gemv[]) (BLASLONG, BLASLONG, BLASLONG, FLOAT, FLOAT, FLOAT *, - BLASLONG, FLOAT *, BLASLONG, FLOAT *, BLASLONG, - FLOAT *) = { - GEMV_N, GEMV_T, GEMV_R, GEMV_C, GEMV_O, GEMV_U, GEMV_S, GEMV_D,}; - #else - -#ifdef SMP - static int (*gemv_thread[]) (BLASLONG, BLASLONG, FLOAT, FLOAT *, - BLASLONG, FLOAT *, BLASLONG, FLOAT *, - BLASLONG, FLOAT *, int) = { #ifdef XDOUBLE - qgemv_thread_n, qgemv_thread_t, -#elif defined DOUBLE - dgemv_thread_n, dgemv_thread_t, + int mode = BLAS_XDOUBLE | BLAS_COMPLEX; +#elif defined(DOUBLE) + int mode = BLAS_DOUBLE | BLAS_COMPLEX; #else - sgemv_thread_n, sgemv_thread_t, + int mode = BLAS_SINGLE | BLAS_COMPLEX; +#endif #endif - }; #endif - int (*gemv[]) (BLASLONG, BLASLONG, BLASLONG, FLOAT, FLOAT *, BLASLONG, - FLOAT *, BLASLONG, FLOAT *, BLASLONG, FLOAT *) = { - GEMV_N, GEMV_T,}; + /* [uplo][op(B)][op(A)], as in interface/gemm.c */ + static int (*gemmt[])(blas_arg_t *, BLASLONG *, BLASLONG *, FLOAT *, FLOAT *, BLASLONG) = { +#ifndef COMPLEX + GEMMT_UNN, GEMMT_UTN, + GEMMT_UNT, GEMMT_UTT, + GEMMT_LNN, GEMMT_LTN, + GEMMT_LNT, GEMMT_LTT, +#else + GEMMT_UNN, GEMMT_UTN, GEMMT_URN, GEMMT_UCN, + GEMMT_UNT, GEMMT_UTT, GEMMT_URT, GEMMT_UCT, + GEMMT_UNR, GEMMT_UTR, GEMMT_URR, GEMMT_UCR, + GEMMT_UNC, GEMMT_UTC, GEMMT_URC, GEMMT_UCC, + GEMMT_LNN, GEMMT_LTN, GEMMT_LRN, GEMMT_LCN, + GEMMT_LNT, GEMMT_LTT, GEMMT_LRT, GEMMT_LCT, + GEMMT_LNR, GEMMT_LTR, GEMMT_LRR, GEMMT_LCR, + GEMMT_LNC, GEMMT_LTC, GEMMT_LRC, GEMMT_LCC, #endif + }; if (m == 0) return; IDEBUG_START; -#if defined(COMPLEX) - if (transb > 1){ -#ifndef CBLAS - IMATCOPY_K_CNC(nrowb, ncolb, (FLOAT)(1.0), (FLOAT)(0.0), b, ldb); -#else - if (order == CblasColMajor) - IMATCOPY_K_CNC(nrowb, ncolb, (FLOAT)(1.0), (FLOAT)(0.0), b, ldb); - if (order == CblasRowMajor) - IMATCOPY_K_RNC(nrowb, ncolb, (FLOAT)(1.0), (FLOAT)(0.0), b, ldb); -#endif - } -#endif + FUNCTION_PROFILE_START(); - const blasint incb = ((transb & 1) == 0) ? 1 : ldb; + args.m = m; + args.n = m; + args.k = k; - if (uplo == 1) { - for (i = 0; i < m; i++) { - j = m - i; + args.a = (void *)a; + args.b = (void *)b; + args.c = (void *)c; - l = j; -#if defined(COMPLEX) - aa = a + i * 2; - bb = b + i * ldb * 2; - if (transa & 1) { - aa = a + lda * i * 2; - } - if (transb & 1) - bb = b + i * 2; - cc = c + i * 2 * ldc + i * 2; -#else - aa = a + i; - bb = b + i * ldb; - if (transa & 1) { - aa = a + lda * i; - } - if (transb & 1) - bb = b + i; - cc = c + i * ldc + i; -#endif + args.lda = lda; + args.ldb = ldb; + args.ldc = ldc; #if defined(COMPLEX) - if (beta_r != ONE || beta_i != ZERO) - SCAL_K(l, 0, 0, beta_r, beta_i, cc, 1, NULL, 0, - NULL, 0); - - if (alpha_r == ZERO && alpha_i == ZERO) - continue; + args.alpha = (void *)alpha; + args.beta = (void *)beta; #else - if (beta != ONE) - SCAL_K(l, 0, 0, beta, cc, 1, NULL, 0, NULL, 0); - - if (alpha == ZERO) - continue; -#endif - - IDEBUG_START; - - buffer_size = 2 * (j + k) + 128 / sizeof(FLOAT); -#ifdef WINDOWS_ABI - buffer_size += 160 / sizeof(FLOAT); + args.alpha = (void *)α + args.beta = (void *)β #endif - // for alignment - buffer_size = (buffer_size + 3) & ~3; - STACK_ALLOC(buffer_size, IFLOAT, buffer); #ifdef SMP + args.common = NULL; + args.nthreads = 1; +#endif - if (1L * j * k < 2304L * GEMM_MULTITHREAD_THRESHOLD) - nthreads = 1; - else - nthreads = num_cpu_avail(2); + buffer = (FLOAT *)blas_memory_alloc(0); + if (!buffer) { + info = -999; + BLASFUNC(xerbla) (ERROR_NAME, &info, sizeof(ERROR_NAME)); + return; + } - if (nthreads == 1) { -#endif + sa = (FLOAT *)((BLASLONG)buffer + GEMM_OFFSET_A); + sb = (FLOAT *)(((BLASLONG)sa + ((GEMM_P * GEMM_Q * COMPSIZE * SIZE + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); -#if defined(COMPLEX) - if (!(transa & 1)) - (gemv[(int)transa]) (j, k, 0, alpha_r, alpha_i, - aa, lda, bb, incb, cc, 1, - buffer); - else - (gemv[(int)transa]) (k, j, 0, alpha_r, alpha_i, - aa, lda, bb, incb, cc, 1, - buffer); -#else - if (!(transa & 1)) - (gemv[(int)transa]) (j, k, 0, alpha, aa, lda, - bb, incb, cc, 1, buffer); - else - (gemv[(int)transa]) (k, j, 0, alpha, aa, lda, - bb, incb, cc, 1, buffer); -#endif #ifdef SMP - } else { - if (!(transa & 1)) - (gemv_thread[(int)transa]) (j, k, alpha, aa, - lda, bb, incb, cc, - 1, buffer, - nthreads); - else - (gemv_thread[(int)transa]) (k, j, alpha, aa, - lda, bb, incb, cc, - 1, buffer, - nthreads); - - } -#endif - - STACK_FREE(buffer); - } + MMK = (double)(m + 1) * (double)m * (double)k; + if (MMK <= (SMP_THRESHOLD_MIN * GEMM_MULTITHREAD_THRESHOLD)) { + args.nthreads = 1; } else { + args.nthreads = num_cpu_avail(3); + } - for (i = 0; i < m; i++) { - j = i + 1; - - l = j; -#if defined COMPLEX - bb = b + i * ldb * 2; - if (transb & 1) { - bb = b + i * 2; - } - cc = c + i * 2 * ldc; -#else - bb = b + i * ldb; - if (transb & 1) { - bb = b + i; - } - cc = c + i * ldc; -#endif - -#if defined(COMPLEX) - if (beta_r != ONE || beta_i != ZERO) - SCAL_K(l, 0, 0, beta_r, beta_i, cc, 1, NULL, 0, - NULL, 0); - - if (alpha_r == ZERO && alpha_i == ZERO) - continue; -#else - if (beta != ONE) - SCAL_K(l, 0, 0, beta, cc, 1, NULL, 0, NULL, 0); - - if (alpha == ZERO) - continue; + if (args.nthreads == 1) { #endif - IDEBUG_START; - buffer_size = 2 * (j + k) + 128 / sizeof(FLOAT); -#ifdef WINDOWS_ABI - buffer_size += 160 / sizeof(FLOAT); -#endif - // for alignment - buffer_size = (buffer_size + 3) & ~3; - STACK_ALLOC(buffer_size, IFLOAT, buffer); + (gemmt[GEMMT_TABLE_INDEX])(&args, NULL, NULL, sa, sb, 0); #ifdef SMP + } else { - if (1L * j * k < 2304L * GEMM_MULTITHREAD_THRESHOLD) - nthreads = 1; - else - nthreads = num_cpu_avail(2); - - if (nthreads == 1) { -#endif + /* Split the triangle into column ranges of equal area; every + thread then runs the single-threaded driver on its own range + and writes a disjoint set of columns of C. */ + mode |= (uplo << BLAS_UPLO_SHIFT); -#if defined(COMPLEX) - if (!(transa & 1)) - (gemv[(int)transa]) (j, k, 0, alpha_r, alpha_i, - a, lda, bb, incb, cc, 1, - buffer); - else - (gemv[(int)transa]) (k, j, 0, alpha_r, alpha_i, - a, lda, bb, incb, cc, 1, - buffer); -#else - if (!(transa & 1)) - (gemv[(int)transa]) (j, k, 0, alpha, a, lda, bb, - incb, cc, 1, buffer); - else - (gemv[(int)transa]) (k, j, 0, alpha, a, lda, bb, - incb, cc, 1, buffer); + syrk_thread(mode, &args, NULL, NULL, gemmt[GEMMT_TABLE_INDEX], sa, sb, args.nthreads); + } #endif -#ifdef SMP - } else { - if (!(transa & 1)) - (gemv_thread[(int)transa]) (j, k, alpha, a, lda, - bb, incb, cc, 1, - buffer, nthreads); - else - (gemv_thread[(int)transa]) (k, j, alpha, a, lda, - bb, incb, cc, 1, - buffer, nthreads); - } -#endif + blas_memory_free(buffer); - STACK_FREE(buffer); - } - } + FUNCTION_PROFILE_END(COMPSIZE * COMPSIZE, args.m * args.k + args.k * args.n + args.m * args.n / 2, + args.m * args.n * args.k); IDEBUG_END; -/* transform B back if necessary */ -#if defined(COMPLEX) - if (transb > 1){ -#ifndef CBLAS - IMATCOPY_K_CNC(nrowb, ncolb, (FLOAT)(1.0), (FLOAT)(0.0), b, ldb); -#else - if (order == CblasColMajor) - IMATCOPY_K_CNC(nrowb, ncolb, (FLOAT)(1.0), (FLOAT)(0.0), b, ldb); - if (order == CblasRowMajor) - IMATCOPY_K_RNC(nrowb, ncolb, (FLOAT)(1.0), (FLOAT)(0.0), b, ldb); -#endif - } -#endif - return; } diff --git a/utest/test_extensions/test_cgemmt.c b/utest/test_extensions/test_cgemmt.c index 92590b1e8b..0f6fa76690 100644 --- a/utest/test_extensions/test_cgemmt.c +++ b/utest/test_extensions/test_cgemmt.c @@ -33,6 +33,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "utest/openblas_utest.h" #include "common.h" +#include +#include #define DATASIZE 100 @@ -2038,5 +2040,59 @@ CTEST(cgemmt, xerbla_c_api_rowmajor_ldc_invalid) M, K, lda, ldb, ldc, expected_info); ASSERT_EQUAL(TRUE, passed); } + +/** + * Test a size that is not a multiple of the register blocking and large + * enough to be split over several threads: the last block column of C then + * starts at an unaligned offset and its diagonal block has to be cut down + * by the kernel. Verified against GEMM, which may only differ by rounding, + * and the untouched triangle must be bit-identical. + */ +CTEST(cgemmt, upper_M_257_K_64_a_notrans_b_notrans) +{ + blasint M = 257, K = 64, i, j, e; + blasint lda = M, ldb = K, ldc = M; + blasint size_c = 2 * M * ldc; + char transa = 'N', transb = 'N', uplo = 'U'; + float alpha[] = {1.5, 0.5}, beta[] = {2.0, 1.0}; + float *a = (float *) malloc(2 * M * K * sizeof(float)); + float *b = (float *) malloc(2 * K * M * sizeof(float)); + float *c = (float *) malloc(size_c * sizeof(float)); + float *c_gemm = (float *) malloc(size_c * sizeof(float)); + float *c_orig = (float *) malloc(size_c * sizeof(float)); + double sumsq = 0.0, norm; + int untouched_ok = 1; + + ASSERT_TRUE(a != NULL && b != NULL && c != NULL && c_gemm != NULL && c_orig != NULL); + + srand_generate(a, 2 * M * K); + srand_generate(b, 2 * K * M); + srand_generate(c, size_c); + for (i = 0; i < size_c; i++) + c_gemm[i] = c_orig[i] = c[i]; + + BLASFUNC(cgemmt)(&uplo, &transa, &transb, &M, &K, alpha, a, &lda, b, &ldb, + beta, c, &ldc); + BLASFUNC(cgemm)(&transa, &transb, &M, &M, &K, alpha, a, &lda, b, &ldb, + beta, c_gemm, &ldc); + + for (j = 0; j < M; j++) /* column */ + for (i = 0; i < M; i++) /* row */ + for (e = 0; e < 2; e++) { + blasint idx = 2 * (i + j * ldc) + e; + if (i <= j) { + double d = (double)(c[idx] - c_gemm[idx]); + sumsq += d * d; + } else if (c[idx] != c_orig[idx]) { + untouched_ok = 0; + } + } + + norm = sqrt(sumsq) / size_c; + free(a); free(b); free(c); free(c_gemm); free(c_orig); + ASSERT_EQUAL(1, untouched_ok); + ASSERT_DBL_NEAR_TOL(0.0, norm, SINGLE_EPS); +} + #endif #endif diff --git a/utest/test_extensions/test_dgemmt.c b/utest/test_extensions/test_dgemmt.c index 9ce8859cca..57dba0b0e2 100644 --- a/utest/test_extensions/test_dgemmt.c +++ b/utest/test_extensions/test_dgemmt.c @@ -33,6 +33,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "utest/openblas_utest.h" #include "common.h" +#include +#include #define DATASIZE 100 @@ -1463,5 +1465,59 @@ CTEST(dgemmt, xerbla_c_api_rowmajor_ldc_invalid) M, K, lda, ldb, ldc, expected_info); ASSERT_EQUAL(TRUE, passed); } + +/** + * Test a size that is not a multiple of the register blocking and large + * enough to be split over several threads: the last block column of C then + * starts at an unaligned offset and its diagonal block has to be cut down + * by the kernel. Verified against GEMM, which may only differ by rounding, + * and the untouched triangle must be bit-identical. + */ +CTEST(dgemmt, upper_M_257_K_64_a_notrans_b_notrans) +{ + blasint M = 257, K = 64, i, j, e; + blasint lda = M, ldb = K, ldc = M; + blasint size_c = 1 * M * ldc; + char transa = 'N', transb = 'N', uplo = 'U'; + double alpha = 1.5, beta = 2.0; + double *a = (double *) malloc(1 * M * K * sizeof(double)); + double *b = (double *) malloc(1 * K * M * sizeof(double)); + double *c = (double *) malloc(size_c * sizeof(double)); + double *c_gemm = (double *) malloc(size_c * sizeof(double)); + double *c_orig = (double *) malloc(size_c * sizeof(double)); + double sumsq = 0.0, norm; + int untouched_ok = 1; + + ASSERT_TRUE(a != NULL && b != NULL && c != NULL && c_gemm != NULL && c_orig != NULL); + + drand_generate(a, 1 * M * K); + drand_generate(b, 1 * K * M); + drand_generate(c, size_c); + for (i = 0; i < size_c; i++) + c_gemm[i] = c_orig[i] = c[i]; + + BLASFUNC(dgemmt)(&uplo, &transa, &transb, &M, &K, &alpha, a, &lda, b, &ldb, + &beta, c, &ldc); + BLASFUNC(dgemm)(&transa, &transb, &M, &M, &K, &alpha, a, &lda, b, &ldb, + &beta, c_gemm, &ldc); + + for (j = 0; j < M; j++) /* column */ + for (i = 0; i < M; i++) /* row */ + for (e = 0; e < 1; e++) { + blasint idx = 1 * (i + j * ldc) + e; + if (i <= j) { + double d = (double)(c[idx] - c_gemm[idx]); + sumsq += d * d; + } else if (c[idx] != c_orig[idx]) { + untouched_ok = 0; + } + } + + norm = sqrt(sumsq) / size_c; + free(a); free(b); free(c); free(c_gemm); free(c_orig); + ASSERT_EQUAL(1, untouched_ok); + ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); +} + #endif #endif diff --git a/utest/test_extensions/test_sgemmt.c b/utest/test_extensions/test_sgemmt.c index f61fcf5fca..32c81aa875 100644 --- a/utest/test_extensions/test_sgemmt.c +++ b/utest/test_extensions/test_sgemmt.c @@ -33,6 +33,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "utest/openblas_utest.h" #include "common.h" +#include +#include #define DATASIZE 100 @@ -1463,5 +1465,59 @@ CTEST(sgemmt, xerbla_c_api_rowmajor_ldc_invalid) M, K, lda, ldb, ldc, expected_info); ASSERT_EQUAL(TRUE, passed); } + +/** + * Test a size that is not a multiple of the register blocking and large + * enough to be split over several threads: the last block column of C then + * starts at an unaligned offset and its diagonal block has to be cut down + * by the kernel. Verified against GEMM, which may only differ by rounding, + * and the untouched triangle must be bit-identical. + */ +CTEST(sgemmt, upper_M_257_K_64_a_notrans_b_notrans) +{ + blasint M = 257, K = 64, i, j, e; + blasint lda = M, ldb = K, ldc = M; + blasint size_c = 1 * M * ldc; + char transa = 'N', transb = 'N', uplo = 'U'; + float alpha = 1.5, beta = 2.0; + float *a = (float *) malloc(1 * M * K * sizeof(float)); + float *b = (float *) malloc(1 * K * M * sizeof(float)); + float *c = (float *) malloc(size_c * sizeof(float)); + float *c_gemm = (float *) malloc(size_c * sizeof(float)); + float *c_orig = (float *) malloc(size_c * sizeof(float)); + double sumsq = 0.0, norm; + int untouched_ok = 1; + + ASSERT_TRUE(a != NULL && b != NULL && c != NULL && c_gemm != NULL && c_orig != NULL); + + srand_generate(a, 1 * M * K); + srand_generate(b, 1 * K * M); + srand_generate(c, size_c); + for (i = 0; i < size_c; i++) + c_gemm[i] = c_orig[i] = c[i]; + + BLASFUNC(sgemmt)(&uplo, &transa, &transb, &M, &K, &alpha, a, &lda, b, &ldb, + &beta, c, &ldc); + BLASFUNC(sgemm)(&transa, &transb, &M, &M, &K, &alpha, a, &lda, b, &ldb, + &beta, c_gemm, &ldc); + + for (j = 0; j < M; j++) /* column */ + for (i = 0; i < M; i++) /* row */ + for (e = 0; e < 1; e++) { + blasint idx = 1 * (i + j * ldc) + e; + if (i <= j) { + double d = (double)(c[idx] - c_gemm[idx]); + sumsq += d * d; + } else if (c[idx] != c_orig[idx]) { + untouched_ok = 0; + } + } + + norm = sqrt(sumsq) / size_c; + free(a); free(b); free(c); free(c_gemm); free(c_orig); + ASSERT_EQUAL(1, untouched_ok); + ASSERT_DBL_NEAR_TOL(0.0, norm, SINGLE_EPS); +} + #endif #endif diff --git a/utest/test_extensions/test_zgemmt.c b/utest/test_extensions/test_zgemmt.c index dcd70e9e36..0f915ad2e8 100644 --- a/utest/test_extensions/test_zgemmt.c +++ b/utest/test_extensions/test_zgemmt.c @@ -33,6 +33,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "utest/openblas_utest.h" #include "common.h" +#include +#include #define DATASIZE 100 @@ -2036,5 +2038,59 @@ CTEST(zgemmt, xerbla_c_api_rowmajor_ldc_invalid) M, K, lda, ldb, ldc, expected_info); ASSERT_EQUAL(TRUE, passed); } + +/** + * Test a size that is not a multiple of the register blocking and large + * enough to be split over several threads: the last block column of C then + * starts at an unaligned offset and its diagonal block has to be cut down + * by the kernel. Verified against GEMM, which may only differ by rounding, + * and the untouched triangle must be bit-identical. + */ +CTEST(zgemmt, upper_M_257_K_64_a_notrans_b_notrans) +{ + blasint M = 257, K = 64, i, j, e; + blasint lda = M, ldb = K, ldc = M; + blasint size_c = 2 * M * ldc; + char transa = 'N', transb = 'N', uplo = 'U'; + double alpha[] = {1.5, 0.5}, beta[] = {2.0, 1.0}; + double *a = (double *) malloc(2 * M * K * sizeof(double)); + double *b = (double *) malloc(2 * K * M * sizeof(double)); + double *c = (double *) malloc(size_c * sizeof(double)); + double *c_gemm = (double *) malloc(size_c * sizeof(double)); + double *c_orig = (double *) malloc(size_c * sizeof(double)); + double sumsq = 0.0, norm; + int untouched_ok = 1; + + ASSERT_TRUE(a != NULL && b != NULL && c != NULL && c_gemm != NULL && c_orig != NULL); + + drand_generate(a, 2 * M * K); + drand_generate(b, 2 * K * M); + drand_generate(c, size_c); + for (i = 0; i < size_c; i++) + c_gemm[i] = c_orig[i] = c[i]; + + BLASFUNC(zgemmt)(&uplo, &transa, &transb, &M, &K, alpha, a, &lda, b, &ldb, + beta, c, &ldc); + BLASFUNC(zgemm)(&transa, &transb, &M, &M, &K, alpha, a, &lda, b, &ldb, + beta, c_gemm, &ldc); + + for (j = 0; j < M; j++) /* column */ + for (i = 0; i < M; i++) /* row */ + for (e = 0; e < 2; e++) { + blasint idx = 2 * (i + j * ldc) + e; + if (i <= j) { + double d = (double)(c[idx] - c_gemm[idx]); + sumsq += d * d; + } else if (c[idx] != c_orig[idx]) { + untouched_ok = 0; + } + } + + norm = sqrt(sumsq) / size_c; + free(a); free(b); free(c); free(c_gemm); free(c_orig); + ASSERT_EQUAL(1, untouched_ok); + ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); +} + #endif #endif