Skip to content

Replace the GEMV-loop ?GEMMT with a blocked level-3 driver #6060

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jgillis wants to merge 2 commits into
base: develop
Choose a base branch
from
Open

Replace the GEMV-loop ?GEMMT with a blocked level-3 driver #6060

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7e876d5
Implement GEMMT with a blocked level-3 driver instead of a GEMV loop
jgillis Sep 25, 2026
49f817a
Pass CR and RC as -DCR=CR / -DRC=RC when building GEMMT objects with …
jgillis Sep 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions common_c.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -514,6 +514,46 @@
#define CSYRK_KERNEL_U csyrk_kernel_U
#define CSYRK_KERNEL_L csyrk_kernel_L

#define CGEMMT_UNN cgemmt_UNN
#define CGEMMT_UNT cgemmt_UNT
#define CGEMMT_UNR cgemmt_UNR
#define CGEMMT_UNC cgemmt_UNC
#define CGEMMT_UTN cgemmt_UTN
#define CGEMMT_UTT cgemmt_UTT
#define CGEMMT_UTR cgemmt_UTR
#define CGEMMT_UTC cgemmt_UTC
#define CGEMMT_URN cgemmt_URN
#define CGEMMT_URT cgemmt_URT
#define CGEMMT_URR cgemmt_URR
#define CGEMMT_URC cgemmt_URC
#define CGEMMT_UCN cgemmt_UCN
#define CGEMMT_UCT cgemmt_UCT
#define CGEMMT_UCR cgemmt_UCR
#define CGEMMT_UCC cgemmt_UCC
#define CGEMMT_LNN cgemmt_LNN
#define CGEMMT_LNT cgemmt_LNT
#define CGEMMT_LNR cgemmt_LNR
#define CGEMMT_LNC cgemmt_LNC
#define CGEMMT_LTN cgemmt_LTN
#define CGEMMT_LTT cgemmt_LTT
#define CGEMMT_LTR cgemmt_LTR
#define CGEMMT_LTC cgemmt_LTC
#define CGEMMT_LRN cgemmt_LRN
#define CGEMMT_LRT cgemmt_LRT
#define CGEMMT_LRR cgemmt_LRR
#define CGEMMT_LRC cgemmt_LRC
#define CGEMMT_LCN cgemmt_LCN
#define CGEMMT_LCT cgemmt_LCT
#define CGEMMT_LCR cgemmt_LCR
#define CGEMMT_LCC cgemmt_LCC

#define CGEMMT_KERNEL_UCN cgemmt_kernel_UCN
#define CGEMMT_KERNEL_UNC cgemmt_kernel_UNC
#define CGEMMT_KERNEL_UCC cgemmt_kernel_UCC
#define CGEMMT_KERNEL_LCN cgemmt_kernel_LCN
#define CGEMMT_KERNEL_LNC cgemmt_kernel_LNC
#define CGEMMT_KERNEL_LCC cgemmt_kernel_LCC

#define CHERK_UN cherk_UN
#define CHERK_LN cherk_LN
#define CHERK_UC cherk_UC
Expand Down
9 changes: 9 additions & 0 deletions common_d.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,15 @@
#define DSYRK_KERNEL_U dsyrk_kernel_U
#define DSYRK_KERNEL_L dsyrk_kernel_L

#define DGEMMT_UNN dgemmt_UNN
#define DGEMMT_UNT dgemmt_UNT
#define DGEMMT_UTN dgemmt_UTN
#define DGEMMT_UTT dgemmt_UTT
#define DGEMMT_LNN dgemmt_LNN
#define DGEMMT_LNT dgemmt_LNT
#define DGEMMT_LTN dgemmt_LTN
#define DGEMMT_LTT dgemmt_LTT

#define DHERK_UN dsyrk_UN
#define DHERK_LN dsyrk_LN
#define DHERK_UC dsyrk_UT
Expand Down
145 changes: 145 additions & 0 deletions common_level3.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1689,6 +1689,151 @@ int ssyrk_thread_LN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLAS
int ssyrk_thread_LT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);

int dsyrk_thread_UN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);

int qgemmt_UNN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int qgemmt_UNT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int qgemmt_UTN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int qgemmt_UTT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int qgemmt_LNN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int qgemmt_LNT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int qgemmt_LTN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int qgemmt_LTT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);

int dgemmt_UNN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int dgemmt_UNT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int dgemmt_UTN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int dgemmt_UTT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int dgemmt_LNN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int dgemmt_LNT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int dgemmt_LTN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int dgemmt_LTT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);

int sgemmt_UNN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int sgemmt_UNT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int sgemmt_UTN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int sgemmt_UTT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int sgemmt_LNN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int sgemmt_LNT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int sgemmt_LTN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int sgemmt_LTT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);

int xgemmt_UNN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_UNT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_UNR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_UNC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_UTN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_UTT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_UTR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_UTC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_URN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_URT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_URR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_URC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_UCN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_UCT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_UCR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_UCC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_LNN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_LNT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_LNR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_LNC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_LTN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_LTT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_LTR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_LTC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_LRN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_LRT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_LRR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_LRC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_LCN(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_LCT(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_LCR(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_LCC(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
int xgemmt_kernel_UCN(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG);
int xgemmt_kernel_UNC(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG);
int xgemmt_kernel_UCC(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG);
int xgemmt_kernel_LCN(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG);
int xgemmt_kernel_LNC(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG);
int xgemmt_kernel_LCC(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG);

int zgemmt_UNN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_UNT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_UNR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_UNC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_UTN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_UTT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_UTR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_UTC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_URN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_URT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_URR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_URC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_UCN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_UCT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_UCR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_UCC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_LNN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_LNT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_LNR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_LNC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_LTN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_LTT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_LTR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_LTC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_LRN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_LRT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_LRR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_LRC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_LCN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_LCT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_LCR(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_LCC(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int zgemmt_kernel_UCN(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG);
int zgemmt_kernel_UNC(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG);
int zgemmt_kernel_UCC(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG);
int zgemmt_kernel_LCN(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG);
int zgemmt_kernel_LNC(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG);
int zgemmt_kernel_LCC(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG);

int cgemmt_UNN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_UNT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_UNR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_UNC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_UTN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_UTT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_UTR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_UTC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_URN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_URT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_URR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_URC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_UCN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_UCT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_UCR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_UCC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_LNN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_LNT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_LNR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_LNC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_LTN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_LTT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_LTR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_LTC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_LRN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_LRT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_LRR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_LRC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_LCN(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_LCT(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_LCR(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_LCC(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
int cgemmt_kernel_UCN(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG);
int cgemmt_kernel_UNC(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG);
int cgemmt_kernel_UCC(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG);
int cgemmt_kernel_LCN(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG);
int cgemmt_kernel_LNC(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG);
int cgemmt_kernel_LCC(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG);

int dsyrk_thread_UT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int dsyrk_thread_LN(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
int dsyrk_thread_LT(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
Expand Down
Loading
Loading