From d17ffe21fa20a4d3f526cb4997377dec5af623f3 Mon Sep 17 00:00:00 2001 From: Anna Mayne Date: Tue, 16 Dec 2025 13:26:38 +0000 Subject: [PATCH 1/8] Add interleaving to sgemm and dgemm. Disentangle trmm and symm from gemm. Co-authored-by: Chris Sidebottom --- common_c.h | 17 +- common_d.h | 9 +- common_level3.h | 21 + common_macro.h | 50 + common_param.h | 21 +- common_s.h | 9 +- common_z.h | 17 + driver/level3/symm_k.c | 31 + driver/level3/trmm_L.c | 25 +- driver/level3/trmm_R.c | 37 +- kernel/CMakeLists.txt | 30 +- kernel/Makefile.L3 | 631 +++++++- kernel/arm64/KERNEL.ARMV8SVE | 32 +- kernel/arm64/dgemm_kernel_inter_sve_v2x8.S | 1520 ++++++++++++++++++++ kernel/arm64/gemm_ncopy_sve_v2x8.c | 179 +++ kernel/arm64/gemm_tcopy_sve_v2x8.c | 174 +++ kernel/arm64/sgemm_kernel_inter_sve_v2x8.S | 1518 +++++++++++++++++++ kernel/setparam-ref.c | 10 +- 18 files changed, 4285 insertions(+), 46 deletions(-) create mode 100644 kernel/arm64/dgemm_kernel_inter_sve_v2x8.S create mode 100644 kernel/arm64/gemm_ncopy_sve_v2x8.c create mode 100644 kernel/arm64/gemm_tcopy_sve_v2x8.c create mode 100644 kernel/arm64/sgemm_kernel_inter_sve_v2x8.S diff --git a/common_c.h b/common_c.h index 88f8f5dcdf..cca27ea073 100644 --- a/common_c.h +++ b/common_c.h @@ -107,6 +107,8 @@ #define CTRMM_ILNNCOPY ctrmm_ilnncopy #define CTRMM_ILTUCOPY ctrmm_iltucopy #define CTRMM_ILTNCOPY ctrmm_iltncopy +#define CCOMM_NCOPY ccomm_ncopy +#define CCOMM_TCOPY ccomm_tcopy #define CTRSM_IUNUCOPY ctrsm_iunucopy #define CTRSM_IUNNCOPY ctrsm_iunncopy @@ -126,6 +128,11 @@ #define CGEMM_KERNEL_R cgemm_kernel_r #define CGEMM_KERNEL_B cgemm_kernel_b +#define CCOMM_KERNEL_N ccomm_kernel_n +#define CCOMM_KERNEL_L ccomm_kernel_l +#define CCOMM_KERNEL_R ccomm_kernel_r +#define CCOMM_KERNEL_B ccomm_kernel_b + #define CTRMM_KERNEL_LN ctrmm_kernel_LN #define CTRMM_KERNEL_LT ctrmm_kernel_LT #define CTRMM_KERNEL_LR ctrmm_kernel_LR @@ -321,18 +328,26 @@ #define CTRMM_IUTNCOPY gotoblas -> ctrmm_iutncopy #define CTRMM_ILNNCOPY gotoblas -> ctrmm_ilnncopy #define CTRMM_ILTNCOPY gotoblas -> ctrmm_iltncopy +#define CCOMM_NCOPY gotoblas -> ccomm_ncopy +#define CCOMM_TCOPY gotoblas -> ccomm_tcopy + #define CTRSM_IUNNCOPY gotoblas -> ctrsm_iunncopy #define CTRSM_IUTNCOPY gotoblas -> ctrsm_iutncopy #define CTRSM_ILNNCOPY gotoblas -> ctrsm_ilnncopy #define CTRSM_ILTNCOPY gotoblas -> ctrsm_iltncopy -#define CGEMM_BETA gotoblas -> cgemm_beta #define SME_CGEMM_KERNEL gotoblas -> sme_cgemm_kernel +#define CGEMM_BETA gotoblas -> cgemm_beta #define CGEMM_KERNEL_N gotoblas -> cgemm_kernel_n #define CGEMM_KERNEL_L gotoblas -> cgemm_kernel_l #define CGEMM_KERNEL_R gotoblas -> cgemm_kernel_r #define CGEMM_KERNEL_B gotoblas -> cgemm_kernel_b +#define CCOMM_KERNEL_N gotoblas -> ccomm_kernel_n +#define CCOMM_KERNEL_L gotoblas -> ccomm_kernel_l +#define CCOMM_KERNEL_R gotoblas -> ccomm_kernel_r +#define CCOMM_KERNEL_B gotoblas -> ccomm_kernel_b + #define CTRMM_KERNEL_LN gotoblas -> ctrmm_kernel_LN #define CTRMM_KERNEL_LT gotoblas -> ctrmm_kernel_LT #define CTRMM_KERNEL_LR gotoblas -> ctrmm_kernel_LR diff --git a/common_d.h b/common_d.h index a7a1e1cd14..342df59284 100644 --- a/common_d.h +++ b/common_d.h @@ -100,6 +100,8 @@ #define DTRMM_ILNNCOPY dtrmm_ilnncopy #define DTRMM_ILTUCOPY dtrmm_iltucopy #define DTRMM_ILTNCOPY dtrmm_iltncopy +#define DCOMM_NCOPY dcomm_ncopy +#define DCOMM_TCOPY dcomm_tcopy #define DTRSM_IUNUCOPY dtrsm_iunucopy #define DTRSM_IUNNCOPY dtrsm_iunncopy @@ -115,6 +117,7 @@ #define DGEMM_KERNEL dgemm_kernel #define SME_DGEMM_KERNEL sme_dgemm_kernel +#define DCOMM_KERNEL dcomm_kernel #define DTRMM_KERNEL_LN dtrmm_kernel_LN #define DTRMM_KERNEL_LT dtrmm_kernel_LT @@ -240,14 +243,18 @@ #define DTRMM_IUTNCOPY gotoblas -> dtrmm_iutncopy #define DTRMM_ILNNCOPY gotoblas -> dtrmm_ilnncopy #define DTRMM_ILTNCOPY gotoblas -> dtrmm_iltncopy +#define DCOMM_NCOPY gotoblas -> dcomm_ncopy +#define DCOMM_TCOPY gotoblas -> dcomm_tcopy + #define DTRSM_IUNNCOPY gotoblas -> dtrsm_iunncopy #define DTRSM_IUTNCOPY gotoblas -> dtrsm_iutncopy #define DTRSM_ILNNCOPY gotoblas -> dtrsm_ilnncopy #define DTRSM_ILTNCOPY gotoblas -> dtrsm_iltncopy -#define DGEMM_BETA gotoblas -> dgemm_beta +#define DGEMM_BETA gotoblas -> dgemm_beta #define DGEMM_KERNEL gotoblas -> dgemm_kernel #define SME_DGEMM_KERNEL gotoblas -> sme_dgemm_kernel +#define DCOMM_KERNEL gotoblas -> dcomm_kernel #define DTRMM_KERNEL_LN gotoblas -> dtrmm_kernel_LN #define DTRMM_KERNEL_LT gotoblas -> dtrmm_kernel_LT diff --git a/common_level3.h b/common_level3.h index 3e4edf1950..aeb516d2b4 100644 --- a/common_level3.h +++ b/common_level3.h @@ -1,5 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -334,6 +335,8 @@ int strmm_ilnucopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX int strmm_ilnncopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); int strmm_iltucopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); int strmm_iltncopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); +int scomm_ncopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, float *b); +int scomm_tcopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, float *b); int strmm_olnucopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); int strmm_olnncopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); int strmm_oltucopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); @@ -351,6 +354,8 @@ int dtrmm_ilnucopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG pos int dtrmm_ilnncopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); int dtrmm_iltucopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); int dtrmm_iltncopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); +int dcomm_tcopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, double *b); +int dcomm_ncopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, double *b); int dtrmm_olnucopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); int dtrmm_olnncopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); int dtrmm_oltucopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); @@ -385,6 +390,8 @@ int ctrmm_ilnucopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX int ctrmm_ilnncopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); int ctrmm_iltucopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); int ctrmm_iltncopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); +int ccomm_tcopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, float *b); +int ccomm_ncopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, float *b); int ctrmm_olnucopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); int ctrmm_olnncopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); int ctrmm_oltucopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); @@ -402,6 +409,8 @@ int ztrmm_ilnucopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG pos int ztrmm_ilnncopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); int ztrmm_iltucopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); int ztrmm_iltncopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); +int zcomm_tcopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, double *b); +int zcomm_ncopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, double *b); int ztrmm_olnucopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); int ztrmm_olnncopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); int ztrmm_oltucopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); @@ -629,6 +638,8 @@ int bgemm_kernel(BLASLONG, BLASLONG, BLASLONG, bfloat16, bfloat16 *, bfloat16 * int sbgemm_kernel(BLASLONG, BLASLONG, BLASLONG, float, bfloat16 *, bfloat16 *, float *, BLASLONG); int sgemm_kernel(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); int dgemm_kernel(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); +int scomm_kernel(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); +int dcomm_kernel(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); #ifdef QUAD_PRECISION int qgemm_kernel(BLASLONG, BLASLONG, BLASLONG, xidouble *, xidouble *, xidouble *, xdouble *, BLASLONG); @@ -778,6 +789,16 @@ int cgemm3m_kernel(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float int zgemm3m_kernel(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); int xgemm3m_kernel(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG); +int ccomm_kernel_n(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); +int ccomm_kernel_l(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); +int ccomm_kernel_r(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); +int ccomm_kernel_b(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + +int zcomm_kernel_n(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); +int zcomm_kernel_l(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); +int zcomm_kernel_r(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); +int zcomm_kernel_b(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int shgemm_nn(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16 *, BLASLONG); int shgemm_nt(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16 *, BLASLONG); int shgemm_tn(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16 *, BLASLONG); diff --git a/common_macro.h b/common_macro.h index 84c2473020..411c7d2aa7 100644 --- a/common_macro.h +++ b/common_macro.h @@ -407,6 +407,9 @@ #define TRMM_IUTCOPY DTRMM_IUTUCOPY #define TRMM_ILNCOPY DTRMM_ILNUCOPY #define TRMM_ILTCOPY DTRMM_ILTUCOPY +#define COMM_NCOPY DCOMM_NCOPY +#define COMM_TCOPY DCOMM_TCOPY + #define TRSM_IUNCOPY DTRSM_IUNUCOPY #define TRSM_IUTCOPY DTRSM_IUTUCOPY #define TRSM_ILNCOPY DTRSM_ILNUCOPY @@ -427,6 +430,9 @@ #define TRMM_IUTCOPY DTRMM_IUTNCOPY #define TRMM_ILNCOPY DTRMM_ILNNCOPY #define TRMM_ILTCOPY DTRMM_ILTNCOPY +#define COMM_NCOPY DCOMM_NCOPY +#define COMM_TCOPY DCOMM_TCOPY + #define TRSM_IUNCOPY DTRSM_IUNNCOPY #define TRSM_IUTCOPY DTRSM_IUTNCOPY #define TRSM_ILNCOPY DTRSM_ILNNCOPY @@ -441,6 +447,11 @@ #define GEMM_KERNEL_R DGEMM_KERNEL #define GEMM_KERNEL_B DGEMM_KERNEL +#define COMM_KERNEL_N DCOMM_KERNEL +#define COMM_KERNEL_L DCOMM_KERNEL +#define COMM_KERNEL_R DCOMM_KERNEL +#define COMM_KERNEL_B DCOMM_KERNEL + #define TRMM_KERNEL_LN DTRMM_KERNEL_LN #define TRMM_KERNEL_LT DTRMM_KERNEL_LT #define TRMM_KERNEL_LR DTRMM_KERNEL_LN @@ -867,6 +878,9 @@ #define TRMM_IUTCOPY STRMM_IUTUCOPY #define TRMM_ILNCOPY STRMM_ILNUCOPY #define TRMM_ILTCOPY STRMM_ILTUCOPY +#define COMM_NCOPY SCOMM_NCOPY +#define COMM_TCOPY SCOMM_TCOPY + #define TRSM_IUNCOPY STRSM_IUNUCOPY #define TRSM_IUTCOPY STRSM_IUTUCOPY #define TRSM_ILNCOPY STRSM_ILNUCOPY @@ -887,6 +901,9 @@ #define TRMM_IUTCOPY STRMM_IUTNCOPY #define TRMM_ILNCOPY STRMM_ILNNCOPY #define TRMM_ILTCOPY STRMM_ILTNCOPY +#define COMM_NCOPY SCOMM_NCOPY +#define COMM_TCOPY SCOMM_TCOPY + #define TRSM_IUNCOPY STRSM_IUNNCOPY #define TRSM_IUTCOPY STRSM_IUTNCOPY #define TRSM_ILNCOPY STRSM_ILNNCOPY @@ -1118,6 +1135,9 @@ #define TRMM_IUTCOPY STRMM_IUTUCOPY #define TRMM_ILNCOPY STRMM_ILNUCOPY #define TRMM_ILTCOPY STRMM_ILTUCOPY +#define COMM_NCOPY SCOMM_NCOPY +#define COMM_TCOPY SCOMM_TCOPY + #define TRSM_IUNCOPY STRSM_IUNUCOPY #define TRSM_IUTCOPY STRSM_IUTUCOPY #define TRSM_ILNCOPY STRSM_ILNUCOPY @@ -1138,6 +1158,9 @@ #define TRMM_IUTCOPY STRMM_IUTNCOPY #define TRMM_ILNCOPY STRMM_ILNNCOPY #define TRMM_ILTCOPY STRMM_ILTNCOPY +#define COMM_NCOPY SCOMM_NCOPY +#define COMM_TCOPY SCOMM_TCOPY + #define TRSM_IUNCOPY STRSM_IUNNCOPY #define TRSM_IUTCOPY STRSM_IUTNCOPY #define TRSM_ILNCOPY STRSM_ILNNCOPY @@ -1152,6 +1175,11 @@ #define GEMM_KERNEL_R SGEMM_KERNEL #define GEMM_KERNEL_B SGEMM_KERNEL +#define COMM_KERNEL_N SCOMM_KERNEL +#define COMM_KERNEL_L SCOMM_KERNEL +#define COMM_KERNEL_R SCOMM_KERNEL +#define COMM_KERNEL_B SCOMM_KERNEL + #define TRMM_KERNEL_LN STRMM_KERNEL_LN #define TRMM_KERNEL_LT STRMM_KERNEL_LT #define TRMM_KERNEL_LR STRMM_KERNEL_LN @@ -1859,6 +1887,9 @@ #define TRMM_IUTCOPY ZTRMM_IUTUCOPY #define TRMM_ILNCOPY ZTRMM_ILNUCOPY #define TRMM_ILTCOPY ZTRMM_ILTUCOPY +#define COMM_NCOPY ZCOMM_NCOPY +#define COMM_TCOPY ZCOMM_TCOPY + #define TRSM_IUNCOPY ZTRSM_IUNUCOPY #define TRSM_IUTCOPY ZTRSM_IUTUCOPY #define TRSM_ILNCOPY ZTRSM_ILNUCOPY @@ -1879,6 +1910,9 @@ #define TRMM_IUTCOPY ZTRMM_IUTNCOPY #define TRMM_ILNCOPY ZTRMM_ILNNCOPY #define TRMM_ILTCOPY ZTRMM_ILTNCOPY +#define COMM_NCOPY ZCOMM_NCOPY +#define COMM_TCOPY ZCOMM_TCOPY + #define TRSM_IUNCOPY ZTRSM_IUNNCOPY #define TRSM_IUTCOPY ZTRSM_IUTNCOPY #define TRSM_ILNCOPY ZTRSM_ILNNCOPY @@ -1921,6 +1955,11 @@ #define GEMM_KERNEL_R ZGEMM_KERNEL_R #define GEMM_KERNEL_B ZGEMM_KERNEL_B +#define COMM_KERNEL_N ZCOMM_KERNEL_N +#define COMM_KERNEL_L ZCOMM_KERNEL_L +#define COMM_KERNEL_R ZCOMM_KERNEL_R +#define COMM_KERNEL_B ZCOMM_KERNEL_B + #define GEMM3M_KERNEL ZGEMM3M_KERNEL #define TRMM_KERNEL_LN ZTRMM_KERNEL_LN @@ -2324,6 +2363,9 @@ #define TRMM_IUTCOPY CTRMM_IUTUCOPY #define TRMM_ILNCOPY CTRMM_ILNUCOPY #define TRMM_ILTCOPY CTRMM_ILTUCOPY +#define COMM_NCOPY CCOMM_NCOPY +#define COMM_TCOPY CCOMM_TCOPY + #define TRSM_IUNCOPY CTRSM_IUNUCOPY #define TRSM_IUTCOPY CTRSM_IUTUCOPY #define TRSM_ILNCOPY CTRSM_ILNUCOPY @@ -2344,6 +2386,9 @@ #define TRMM_IUTCOPY CTRMM_IUTNCOPY #define TRMM_ILNCOPY CTRMM_ILNNCOPY #define TRMM_ILTCOPY CTRMM_ILTNCOPY +#define COMM_NCOPY CCOMM_NCOPY +#define COMM_TCOPY CCOMM_TCOPY + #define TRSM_IUNCOPY CTRSM_IUNNCOPY #define TRSM_IUTCOPY CTRSM_IUTNCOPY #define TRSM_ILNCOPY CTRSM_ILNNCOPY @@ -2386,6 +2431,11 @@ #define GEMM_KERNEL_R CGEMM_KERNEL_R #define GEMM_KERNEL_B CGEMM_KERNEL_B +#define COMM_KERNEL_N CCOMM_KERNEL_N +#define COMM_KERNEL_L CCOMM_KERNEL_L +#define COMM_KERNEL_R CCOMM_KERNEL_R +#define COMM_KERNEL_B CCOMM_KERNEL_B + #define GEMM3M_KERNEL CGEMM3M_KERNEL #define TRMM_KERNEL_LN CTRMM_KERNEL_LN diff --git a/common_param.h b/common_param.h index 7d21652a5c..d0dc343c53 100644 --- a/common_param.h +++ b/common_param.h @@ -282,7 +282,7 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*sgemm_kernel )(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); int (*sgemm_beta )(BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); - + int (*scomm_kernel )(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); int (*sgemm_incopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*sgemm_itcopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); @@ -340,6 +340,8 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*strmm_ilnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*strmm_iltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*strmm_iltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); + int (*scomm_ncopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); + int (*scomm_tcopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*strmm_ounucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*strmm_ounncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*strmm_outucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); @@ -407,6 +409,7 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL #endif int (*dgemm_kernel )(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); int (*dgemm_beta )(BLASLONG, BLASLONG, BLASLONG, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG); + int (*dcomm_kernel )(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); int (*dgemm_incopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*dgemm_itcopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); @@ -464,6 +467,8 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*dtrmm_ilnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*dtrmm_iltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*dtrmm_iltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); + int (*dcomm_ncopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); + int (*dcomm_tcopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*dtrmm_ounucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*dtrmm_ounncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*dtrmm_outucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); @@ -629,6 +634,11 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*cgemm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); int (*cgemm_beta )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); + int (*ccomm_kernel_n )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*ccomm_kernel_l )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*ccomm_kernel_r )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*ccomm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*cgemm_incopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*cgemm_itcopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*cgemm_oncopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); @@ -721,6 +731,8 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*ctrmm_ilnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*ctrmm_iltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*ctrmm_iltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); + int (*ccomm_ncopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); + int (*ccomm_tcopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*ctrmm_ounucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*ctrmm_ounncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*ctrmm_outucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); @@ -843,6 +855,11 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*zgemm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); int (*zgemm_beta )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG); + int (*zcomm_kernel_n )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*zcomm_kernel_l )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*zcomm_kernel_r )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*zcomm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*zgemm_incopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*zgemm_itcopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*zgemm_oncopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); @@ -935,6 +952,8 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*ztrmm_ilnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*ztrmm_iltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*ztrmm_iltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); + int (*zcomm_ncopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); + int (*zcomm_tcopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*ztrmm_ounucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*ztrmm_ounncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*ztrmm_outucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); diff --git a/common_s.h b/common_s.h index 6df9ba0f1e..e0575130c2 100644 --- a/common_s.h +++ b/common_s.h @@ -122,6 +122,8 @@ #define STRMM_ILNNCOPY strmm_ilnncopy #define STRMM_ILTUCOPY strmm_iltucopy #define STRMM_ILTNCOPY strmm_iltncopy +#define SCOMM_NCOPY scomm_ncopy +#define SCOMM_TCOPY scomm_tcopy #define STRSM_IUNUCOPY strsm_iunucopy #define STRSM_IUNNCOPY strsm_iunncopy @@ -136,6 +138,7 @@ #define SGEMM_BETA sgemm_beta #define SGEMM_KERNEL sgemm_kernel +#define SCOMM_KERNEL scomm_kernel #define STRMM_KERNEL_LN strmm_kernel_LN #define STRMM_KERNEL_LT strmm_kernel_LT @@ -288,13 +291,17 @@ #define STRMM_IUTNCOPY gotoblas -> strmm_iutncopy #define STRMM_ILNNCOPY gotoblas -> strmm_ilnncopy #define STRMM_ILTNCOPY gotoblas -> strmm_iltncopy +#define SCOMM_NCOPY gotoblas -> scomm_ncopy +#define SCOMM_TCOPY gotoblas -> scomm_tcopy + #define STRSM_IUNNCOPY gotoblas -> strsm_iunncopy #define STRSM_IUTNCOPY gotoblas -> strsm_iutncopy #define STRSM_ILNNCOPY gotoblas -> strsm_ilnncopy #define STRSM_ILTNCOPY gotoblas -> strsm_iltncopy -#define SGEMM_BETA gotoblas -> sgemm_beta +#define SGEMM_BETA gotoblas -> sgemm_beta #define SGEMM_KERNEL gotoblas -> sgemm_kernel +#define SCOMM_KERNEL gotoblas -> scomm_kernel #define STRMM_KERNEL_LN gotoblas -> strmm_kernel_LN #define STRMM_KERNEL_LT gotoblas -> strmm_kernel_LT diff --git a/common_z.h b/common_z.h index 5a9fb88b77..475644ef07 100644 --- a/common_z.h +++ b/common_z.h @@ -89,6 +89,8 @@ #define ZTRMM_ILNNCOPY ztrmm_olnncopy #define ZTRMM_ILTUCOPY ztrmm_oltucopy #define ZTRMM_ILTNCOPY ztrmm_oltncopy +#define ZCOMM_NCOPY zcomm_ncopy +#define ZCOMM_TCOPY zcomm_tcopy #define ZTRSM_IUNUCOPY ztrsm_ounucopy #define ZTRSM_IUNNCOPY ztrsm_ounncopy @@ -107,6 +109,8 @@ #define ZTRMM_ILNNCOPY ztrmm_ilnncopy #define ZTRMM_ILTUCOPY ztrmm_iltucopy #define ZTRMM_ILTNCOPY ztrmm_iltncopy +#define ZCOMM_NCOPY zcomm_ncopy +#define ZCOMM_TCOPY zcomm_tcopy #define ZTRSM_IUNUCOPY ztrsm_iunucopy #define ZTRSM_IUNNCOPY ztrsm_iunncopy @@ -126,6 +130,11 @@ #define ZGEMM_KERNEL_R zgemm_kernel_r #define ZGEMM_KERNEL_B zgemm_kernel_b +#define ZCOMM_KERNEL_N zcomm_kernel_n +#define ZCOMM_KERNEL_L zcomm_kernel_l +#define ZCOMM_KERNEL_R zcomm_kernel_r +#define ZCOMM_KERNEL_B zcomm_kernel_b + #define ZTRMM_KERNEL_LN ztrmm_kernel_LN #define ZTRMM_KERNEL_LT ztrmm_kernel_LT #define ZTRMM_KERNEL_LR ztrmm_kernel_LR @@ -321,6 +330,9 @@ #define ZTRMM_IUTNCOPY gotoblas -> ztrmm_iutncopy #define ZTRMM_ILNNCOPY gotoblas -> ztrmm_ilnncopy #define ZTRMM_ILTNCOPY gotoblas -> ztrmm_iltncopy +#define ZCOMM_NCOPY gotoblas -> zcomm_ncopy +#define ZCOMM_TCOPY gotoblas -> zcomm_tcopy + #define ZTRSM_IUNNCOPY gotoblas -> ztrsm_iunncopy #define ZTRSM_IUTNCOPY gotoblas -> ztrsm_iutncopy #define ZTRSM_ILNNCOPY gotoblas -> ztrsm_ilnncopy @@ -333,6 +345,11 @@ #define ZGEMM_KERNEL_R gotoblas -> zgemm_kernel_r #define ZGEMM_KERNEL_B gotoblas -> zgemm_kernel_b +#define ZCOMM_KERNEL_N gotoblas -> zcomm_kernel_n +#define ZCOMM_KERNEL_L gotoblas -> zcomm_kernel_l +#define ZCOMM_KERNEL_R gotoblas -> zcomm_kernel_r +#define ZCOMM_KERNEL_B gotoblas -> zcomm_kernel_b + #define ZTRMM_KERNEL_LN gotoblas -> ztrmm_kernel_LN #define ZTRMM_KERNEL_LT gotoblas -> ztrmm_kernel_LT #define ZTRMM_KERNEL_LR gotoblas -> ztrmm_kernel_LR diff --git a/driver/level3/symm_k.c b/driver/level3/symm_k.c index 567896a436..db2bc172f4 100644 --- a/driver/level3/symm_k.c +++ b/driver/level3/symm_k.c @@ -1,5 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -57,6 +58,35 @@ #endif #endif + +#ifndef ICOPY_OPERATION +#if defined(NN) || defined(NT) || defined(NC) || defined(NR) || \ + defined(RN) || defined(RT) || defined(RC) || defined(RR) +#define ICOPY_OPERATION(M, N, A, LDA, X, Y, BUFFER) COMM_TCOPY(M, N, (IFLOAT *)(A) + ((Y) + (X) * (LDA)) * COMPSIZE, LDA, BUFFER); +#else +#define ICOPY_OPERATION(M, N, A, LDA, X, Y, BUFFER) COMM_NCOPY(M, N, (IFLOAT *)(A) + ((X) + (Y) * (LDA)) * COMPSIZE, LDA, BUFFER); +#endif +#endif + + + + +//#ifndef KERNEL_FUNC +#if defined(NN) || defined(NT) || defined(TN) || defined(TT) +#define KERNEL_FUNC COMM_KERNEL_N +#endif +#if defined(CN) || defined(CT) || defined(RN) || defined(RT) +#define KERNEL_FUNC COMM_KERNEL_L +#endif +#if defined(NC) || defined(TC) || defined(NR) || defined(TR) +#define KERNEL_FUNC COMM_KERNEL_R +#endif +#if defined(CC) || defined(CR) || defined(RC) || defined(RR) +#define KERNEL_FUNC COMM_KERNEL_B +#endif +//#endif + + #ifndef RSIDE #define K args -> m #ifndef LOWER @@ -73,6 +103,7 @@ #endif #endif + #ifdef THREADED_LEVEL3 #include "level3_thread.c" #else diff --git a/driver/level3/trmm_L.c b/driver/level3/trmm_L.c index e25ea7afe8..7927532531 100644 --- a/driver/level3/trmm_L.c +++ b/driver/level3/trmm_L.c @@ -1,5 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -43,11 +44,11 @@ const static FLOAT dp1 = 1.; #ifdef CONJ -#define GEMM_KERNEL GEMM_KERNEL_L +#define COMM_KERNEL COMM_KERNEL_L #define TRMM_KERNEL_N TRMM_KERNEL_LR #define TRMM_KERNEL_T TRMM_KERNEL_LC #else -#define GEMM_KERNEL GEMM_KERNEL_N +#define COMM_KERNEL COMM_KERNEL_N #define TRMM_KERNEL_N TRMM_KERNEL_LN #define TRMM_KERNEL_T TRMM_KERNEL_LT #endif @@ -206,9 +207,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO START_RPCC(); #ifndef TRANSA - GEMM_ITCOPY(min_l, min_i, a + (ls * lda) * COMPSIZE, lda, sa); + COMM_TCOPY(min_l, min_i, a + (ls * lda) * COMPSIZE, lda, sa); #else - GEMM_INCOPY(min_l, min_i, a + (ls ) * COMPSIZE, lda, sa); + COMM_NCOPY(min_l, min_i, a + (ls ) * COMPSIZE, lda, sa); #endif STOP_RPCC(innercost); @@ -231,7 +232,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO START_RPCC(); - GEMM_KERNEL(min_i, min_jj, min_l, dp1, + COMM_KERNEL(min_i, min_jj, min_l, dp1, #ifdef COMPLEX ZERO, #endif @@ -251,16 +252,16 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO START_RPCC(); #ifndef TRANSA - GEMM_ITCOPY(min_l, min_i, a + (is + ls * lda) * COMPSIZE, lda, sa); + COMM_TCOPY(min_l, min_i, a + (is + ls * lda) * COMPSIZE, lda, sa); #else - GEMM_INCOPY(min_l, min_i, a + (ls + is * lda) * COMPSIZE, lda, sa); + COMM_NCOPY(min_l, min_i, a + (ls + is * lda) * COMPSIZE, lda, sa); #endif STOP_RPCC(innercost); START_RPCC(); - GEMM_KERNEL(min_i, min_j, min_l, dp1, + COMM_KERNEL(min_i, min_j, min_l, dp1, #ifdef COMPLEX ZERO, #endif @@ -466,16 +467,16 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO START_RPCC(); #ifndef TRANSA - GEMM_ITCOPY(min_l, min_i, a + (is + (ls - min_l) * lda) * COMPSIZE, lda, sa); + COMM_TCOPY(min_l, min_i, a + (is + (ls - min_l) * lda) * COMPSIZE, lda, sa); #else - GEMM_INCOPY(min_l, min_i, a + ((ls - min_l) + is * lda) * COMPSIZE, lda, sa); + COMM_NCOPY(min_l, min_i, a + ((ls - min_l) + is * lda) * COMPSIZE, lda, sa); #endif STOP_RPCC(innercost); START_RPCC(); - GEMM_KERNEL(min_i, min_j, min_l, dp1, + COMM_KERNEL(min_i, min_j, min_l, dp1, #ifdef COMPLEX ZERO, #endif @@ -492,7 +493,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO #ifdef TIMING total = (double)outercost + (double)innercost + (double)gemmcost + (double)trmmcost; - printf( "Copy A : %5.2f Copy B: %5.2f GEMM Kernel : %5.2f TRMM Kerlnel : %5.2f kernel Effi. : %5.2f Total Effi. : %5.2f\n", + printf( "Copy A : %5.2f Copy B: %5.2f GEMM Kernel : %5.2f TRMM Kernel : %5.2f kernel Effi. : %5.2f Total Effi. : %5.2f\n", innercost / total * 100., outercost / total * 100., gemmcost / total * 100., trmmcost / total * 100., (double)n * (double)n * (double)n / (double)(trmmcost + gemmcost) * 100. * (double)COMPSIZE / 2., diff --git a/driver/level3/trmm_R.c b/driver/level3/trmm_R.c index ab9cdfae8e..9c9975339b 100644 --- a/driver/level3/trmm_R.c +++ b/driver/level3/trmm_R.c @@ -1,5 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -43,11 +44,11 @@ const static FLOAT dp1 = 1.; #ifdef CONJ -#define GEMM_KERNEL GEMM_KERNEL_R +#define COMM_KERNEL COMM_KERNEL_R #define TRMM_KERNEL_N TRMM_KERNEL_RR #define TRMM_KERNEL_T TRMM_KERNEL_RC #else -#define GEMM_KERNEL GEMM_KERNEL_N +#define COMM_KERNEL COMM_KERNEL_N #define TRMM_KERNEL_N TRMM_KERNEL_RN #define TRMM_KERNEL_T TRMM_KERNEL_RT #endif @@ -118,7 +119,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO min_i = m; if (min_i > GEMM_P) min_i = GEMM_P; - GEMM_ITCOPY(min_l, min_i, b + (ls * ldb) * COMPSIZE, ldb, sa); + COMM_TCOPY(min_l, min_i, b + (ls * ldb) * COMPSIZE, ldb, sa); for(jjs = 0; jjs < ls - js; jjs += min_jj){ min_jj = ls - js - jjs; @@ -136,7 +137,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO GEMM_OTCOPY(min_l, min_jj, a + ((js + jjs) + ls * lda) * COMPSIZE, lda, sb + min_l * jjs * COMPSIZE); #endif - GEMM_KERNEL(min_i, min_jj, min_l, dp1, + COMM_KERNEL(min_i, min_jj, min_l, dp1, #ifdef COMPLEX ZERO, #endif @@ -173,9 +174,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO min_i = m - is; if (min_i > GEMM_P) min_i = GEMM_P; - GEMM_ITCOPY(min_l, min_i, b + (is + ls * ldb) * COMPSIZE, ldb, sa); + COMM_TCOPY(min_l, min_i, b + (is + ls * ldb) * COMPSIZE, ldb, sa); - GEMM_KERNEL(min_i, ls - js, min_l, dp1, + COMM_KERNEL(min_i, ls - js, min_l, dp1, #ifdef COMPLEX ZERO, #endif @@ -199,7 +200,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO min_i = m; if (min_i > GEMM_P) min_i = GEMM_P; - GEMM_ITCOPY(min_l, min_i, b + (ls * ldb) * COMPSIZE, ldb, sa); + COMM_TCOPY(min_l, min_i, b + (ls * ldb) * COMPSIZE, ldb, sa); for(jjs = js; jjs < js + min_j; jjs += min_jj){ min_jj = min_j + js - jjs; @@ -217,7 +218,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO GEMM_OTCOPY(min_l, min_jj, a + (jjs + ls * lda) * COMPSIZE, lda, sb + min_l * (jjs - js) * COMPSIZE); #endif - GEMM_KERNEL(min_i, min_jj, min_l, dp1, + COMM_KERNEL(min_i, min_jj, min_l, dp1, #ifdef COMPLEX ZERO, #endif @@ -229,9 +230,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO min_i = m - is; if (min_i > GEMM_P) min_i = GEMM_P; - GEMM_ITCOPY(min_l, min_i, b + (is + ls * ldb) * COMPSIZE, ldb, sa); + COMM_TCOPY(min_l, min_i, b + (is + ls * ldb) * COMPSIZE, ldb, sa); - GEMM_KERNEL(min_i, min_j, min_l, dp1, + COMM_KERNEL(min_i, min_j, min_l, dp1, #ifdef COMPLEX ZERO, #endif @@ -254,7 +255,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO min_i = m; if (min_i > GEMM_P) min_i = GEMM_P; - GEMM_ITCOPY(min_l, min_i, b + (ls * ldb) * COMPSIZE, ldb, sa); + COMM_TCOPY(min_l, min_i, b + (ls * ldb) * COMPSIZE, ldb, sa); for(jjs = 0; jjs < min_l; jjs += min_jj){ min_jj = min_l - jjs; @@ -299,7 +300,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO sb + min_l * (min_l + jjs) * COMPSIZE); #endif - GEMM_KERNEL(min_i, min_jj, min_l, dp1, + COMM_KERNEL(min_i, min_jj, min_l, dp1, #ifdef COMPLEX ZERO, #endif @@ -312,7 +313,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO min_i = m - is; if (min_i > GEMM_P) min_i = GEMM_P; - GEMM_ITCOPY(min_l, min_i, b + (is + ls * ldb) * COMPSIZE, ldb, sa); + COMM_TCOPY(min_l, min_i, b + (is + ls * ldb) * COMPSIZE, ldb, sa); TRMM_KERNEL_N(min_i, min_l, min_l, dp1, #ifdef COMPLEX @@ -323,7 +324,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO b + (is + ls * ldb) * COMPSIZE, ldb, 0); if (js - ls - min_l > 0) { - GEMM_KERNEL(min_i, js - ls - min_l, min_l, dp1, + COMM_KERNEL(min_i, js - ls - min_l, min_l, dp1, #ifdef COMPLEX ZERO, #endif @@ -340,7 +341,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO min_i = m; if (min_i > GEMM_P) min_i = GEMM_P; - GEMM_ITCOPY(min_l, min_i, b + (ls * ldb) * COMPSIZE, ldb, sa); + COMM_TCOPY(min_l, min_i, b + (ls * ldb) * COMPSIZE, ldb, sa); for(jjs = js; jjs < js + min_j; jjs += min_jj){ min_jj = min_j + js - jjs; @@ -358,7 +359,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO GEMM_OTCOPY(min_l, min_jj, a + ((jjs - min_j) + ls * lda) * COMPSIZE, lda, sb + min_l * (jjs - js) * COMPSIZE); #endif - GEMM_KERNEL(min_i, min_jj, min_l, dp1, + COMM_KERNEL(min_i, min_jj, min_l, dp1, #ifdef COMPLEX ZERO, #endif @@ -370,9 +371,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO min_i = m - is; if (min_i > GEMM_P) min_i = GEMM_P; - GEMM_ITCOPY(min_l, min_i, b + (is + ls * ldb) * COMPSIZE, ldb, sa); + COMM_TCOPY(min_l, min_i, b + (is + ls * ldb) * COMPSIZE, ldb, sa); - GEMM_KERNEL(min_i, min_j, min_l, dp1, + COMM_KERNEL(min_i, min_j, min_l, dp1, #ifdef COMPLEX ZERO, #endif diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 1a1443a6a3..a66918c9ba 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -339,8 +339,12 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS) foreach (float_type SINGLE DOUBLE) string(SUBSTRING ${float_type} 0 1 float_char) GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "" "gemm_kernel" false "" "" false ${float_type}) + if (${float_char}COMMKERNEL) + GenerateNamedObjects("${KERNELDIR}/${${float_char}COMMKERNEL}" "" "comm_kernel" false "" "" false ${float_type}) + endif () endforeach() - if (BUILD_COMPLEX16 AND NOT BUILD_DOUBLE) + + if (BUILD_COMPLEX16 AND NOT BUILD_DOUBLE) GenerateNamedObjects("${KERNELDIR}/${DGEMMKERNEL}" "" "gemm_kernel" false "" "" false "DOUBLE") if (DGEMMINCOPY) GenerateNamedObjects("${KERNELDIR}/${DGEMMINCOPY}" "DOUBLE" "${DGEMMINCOPYOBJ}" false "" "" true "DOUBLE") @@ -507,10 +511,27 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS) if (USE_TRMM) set(TRMM_KERNEL "${${float_char}TRMMKERNEL}") + elseif (${float_char}COMMKERNEL) + set(TRMM_KERNEL "${${float_char}COMMKERNEL}") else () set(TRMM_KERNEL "${${float_char}GEMMKERNEL}") endif () + if (${float_char}COMMNCOPY) + GenerateNamedObjects("${KERNELDIR}/${${float_char}COMMNCOPY}" "${float_type}" "${${float_char}COMMNCOPYOBJ}" false "" "" true ${float_type}) + elseif (${float_char}GEMMINCOPY) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMINCOPY}" "${float_type}" "${${float_char}COMMNCOPYOBJ}" false "" "" true ${float_type}) + else () + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMONCOPY}" "${float_type}" "${${float_char}COMMNCOPYOBJ}" false "" "" true ${float_type}) + endif () + + if (${float_char}COMMTCOPY) + GenerateNamedObjects("${KERNELDIR}/${${float_char}COMMTCOPY}" "${float_type}" "${${float_char}COMMTCOPYOBJ}" false "" "" true ${float_type}) + elseif (${float_char}GEMMITCOPY) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMITCOPY}" "${float_type}" "${${float_char}COMMTCOPYOBJ}" false "" "" true ${float_type}) + else () + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMOTCOPY}" "${float_type}" "${${float_char}COMMTCOPYOBJ}" false "" "" true ${float_type}) + endif () if (${float_type} STREQUAL "COMPLEX" OR ${float_type} STREQUAL "ZCOMPLEX") @@ -521,6 +542,13 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS) GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "NC" "gemm_kernel_r" false "" "" false ${float_type}) GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "CC" "gemm_kernel_b" false "" "" false ${float_type}) + if (${float_char}COMMKERNEL) + GenerateNamedObjects("${KERNELDIR}/${${float_char}COMMKERNEL}" "NN" "comm_kernel_n" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}COMMKERNEL}" "CN" "comm_kernel_l" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}COMMKERNEL}" "NC" "comm_kernel_r" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}COMMKERNEL}" "CC" "comm_kernel_b" false "" "" false ${float_type}) + endif () + GenerateNamedObjects("${KERNELDIR}/${TRMM_KERNEL}" "TRMMKERNEL;LEFT;NN" "trmm_kernel_LN" false "" "" false ${float_type}) GenerateNamedObjects("${KERNELDIR}/${TRMM_KERNEL}" "TRMMKERNEL;LEFT;TRANSA;NN" "trmm_kernel_LT" false "" "" false ${float_type}) GenerateNamedObjects("${KERNELDIR}/${TRMM_KERNEL}" "TRMMKERNEL;LEFT;CONJ;CN" "trmm_kernel_LR" false "" "" false ${float_type}) diff --git a/kernel/Makefile.L3 b/kernel/Makefile.L3 index f151f5bb5f..dc4c749feb 100644 --- a/kernel/Makefile.L3 +++ b/kernel/Makefile.L3 @@ -137,6 +137,9 @@ ifeq ($(CORE), Z14) USE_TRMM = 1 endif + + + ifdef USE_DIRECT_SGEMM ifndef SGEMMDIRECTKERNEL ifeq ($(ARCH), x86_64) @@ -246,6 +249,7 @@ endif ifneq "$(or $(BUILD_SINGLE),$(BUILD_DOUBLE),$(BUILD_COMPLEX))" "" SKERNELOBJS += \ sgemm_kernel$(TSUFFIX).$(SUFFIX) \ + scomm_kernel$(TSUFFIX).$(SUFFIX) \ sgemm_beta$(TSUFFIX).$(SUFFIX) \ $(SGEMMINCOPYOBJ) $(SGEMMITCOPYOBJ) \ $(SGEMMONCOPYOBJ) $(SGEMMOTCOPYOBJ) @@ -312,6 +316,7 @@ ifneq "$(or $(BUILD_DOUBLE),$(BUILD_COMPLEX16))" "" DKERNELOBJS += \ dgemm_beta$(TSUFFIX).$(SUFFIX) \ dgemm_kernel$(TSUFFIX).$(SUFFIX) \ + dcomm_kernel$(TSUFFIX).$(SUFFIX) \ $(DGEMMINCOPYOBJ) $(DGEMMITCOPYOBJ) \ $(DGEMMONCOPYOBJ) $(DGEMMOTCOPYOBJ) endif @@ -325,6 +330,8 @@ ifneq "$(or $(BUILD_COMPLEX),$(BUILD_COMPLEX16))" "" CKERNELOBJS += \ cgemm_kernel_n$(TSUFFIX).$(SUFFIX) cgemm_kernel_r$(TSUFFIX).$(SUFFIX) \ cgemm_kernel_l$(TSUFFIX).$(SUFFIX) cgemm_kernel_b$(TSUFFIX).$(SUFFIX) \ + ccomm_kernel_n$(TSUFFIX).$(SUFFIX) ccomm_kernel_r$(TSUFFIX).$(SUFFIX) \ + ccomm_kernel_l$(TSUFFIX).$(SUFFIX) ccomm_kernel_b$(TSUFFIX).$(SUFFIX) \ $(CGEMMINCOPYOBJ) $(CGEMMITCOPYOBJ) \ $(CGEMMONCOPYOBJ) $(CGEMMOTCOPYOBJ) endif @@ -333,6 +340,8 @@ ifeq ($(BUILD_COMPLEX16),1) ZKERNELOBJS += \ zgemm_kernel_n$(TSUFFIX).$(SUFFIX) zgemm_kernel_r$(TSUFFIX).$(SUFFIX) \ zgemm_kernel_l$(TSUFFIX).$(SUFFIX) zgemm_kernel_b$(TSUFFIX).$(SUFFIX) \ + zcomm_kernel_n$(TSUFFIX).$(SUFFIX) zcomm_kernel_r$(TSUFFIX).$(SUFFIX) \ + zcomm_kernel_l$(TSUFFIX).$(SUFFIX) zcomm_kernel_b$(TSUFFIX).$(SUFFIX) \ $(ZGEMMINCOPYOBJ) $(ZGEMMITCOPYOBJ) \ $(ZGEMMONCOPYOBJ) $(ZGEMMOTCOPYOBJ) endif @@ -444,6 +453,7 @@ SBLASOBJS += \ strmm_ilnucopy$(TSUFFIX).$(SUFFIX) strmm_ilnncopy$(TSUFFIX).$(SUFFIX) \ strmm_iutucopy$(TSUFFIX).$(SUFFIX) strmm_iutncopy$(TSUFFIX).$(SUFFIX) \ strmm_iltucopy$(TSUFFIX).$(SUFFIX) strmm_iltncopy$(TSUFFIX).$(SUFFIX) \ + scomm_ncopy$(TSUFFIX).$(SUFFIX) scomm_tcopy$(TSUFFIX).$(SUFFIX) \ strmm_ounucopy$(TSUFFIX).$(SUFFIX) strmm_ounncopy$(TSUFFIX).$(SUFFIX) \ strmm_olnucopy$(TSUFFIX).$(SUFFIX) strmm_olnncopy$(TSUFFIX).$(SUFFIX) \ strmm_outucopy$(TSUFFIX).$(SUFFIX) strmm_outncopy$(TSUFFIX).$(SUFFIX) \ @@ -466,6 +476,7 @@ DBLASOBJS += \ dtrmm_ilnucopy$(TSUFFIX).$(SUFFIX) dtrmm_ilnncopy$(TSUFFIX).$(SUFFIX) \ dtrmm_iutucopy$(TSUFFIX).$(SUFFIX) dtrmm_iutncopy$(TSUFFIX).$(SUFFIX) \ dtrmm_iltucopy$(TSUFFIX).$(SUFFIX) dtrmm_iltncopy$(TSUFFIX).$(SUFFIX) \ + dcomm_ncopy$(TSUFFIX).$(SUFFIX) dcomm_tcopy$(TSUFFIX).$(SUFFIX) \ dtrmm_ounucopy$(TSUFFIX).$(SUFFIX) dtrmm_ounncopy$(TSUFFIX).$(SUFFIX) \ dtrmm_olnucopy$(TSUFFIX).$(SUFFIX) dtrmm_olnncopy$(TSUFFIX).$(SUFFIX) \ dtrmm_outucopy$(TSUFFIX).$(SUFFIX) dtrmm_outncopy$(TSUFFIX).$(SUFFIX) \ @@ -508,6 +519,7 @@ CBLASOBJS += \ ctrmm_ilnucopy$(TSUFFIX).$(SUFFIX) ctrmm_ilnncopy$(TSUFFIX).$(SUFFIX) \ ctrmm_iutucopy$(TSUFFIX).$(SUFFIX) ctrmm_iutncopy$(TSUFFIX).$(SUFFIX) \ ctrmm_iltucopy$(TSUFFIX).$(SUFFIX) ctrmm_iltncopy$(TSUFFIX).$(SUFFIX) \ + ccomm_ncopy$(TSUFFIX).$(SUFFIX) ccomm_tcopy$(TSUFFIX).$(SUFFIX) \ ctrmm_ounucopy$(TSUFFIX).$(SUFFIX) ctrmm_ounncopy$(TSUFFIX).$(SUFFIX) \ ctrmm_olnucopy$(TSUFFIX).$(SUFFIX) ctrmm_olnncopy$(TSUFFIX).$(SUFFIX) \ ctrmm_outucopy$(TSUFFIX).$(SUFFIX) ctrmm_outncopy$(TSUFFIX).$(SUFFIX) \ @@ -535,6 +547,7 @@ ZBLASOBJS += \ ztrmm_ilnucopy$(TSUFFIX).$(SUFFIX) ztrmm_ilnncopy$(TSUFFIX).$(SUFFIX) \ ztrmm_iutucopy$(TSUFFIX).$(SUFFIX) ztrmm_iutncopy$(TSUFFIX).$(SUFFIX) \ ztrmm_iltucopy$(TSUFFIX).$(SUFFIX) ztrmm_iltncopy$(TSUFFIX).$(SUFFIX) \ + zcomm_ncopy$(TSUFFIX).$(SUFFIX) zcomm_tcopy$(TSUFFIX).$(SUFFIX) \ ztrmm_ounucopy$(TSUFFIX).$(SUFFIX) ztrmm_ounncopy$(TSUFFIX).$(SUFFIX) \ ztrmm_olnucopy$(TSUFFIX).$(SUFFIX) ztrmm_olnncopy$(TSUFFIX).$(SUFFIX) \ ztrmm_outucopy$(TSUFFIX).$(SUFFIX) ztrmm_outncopy$(TSUFFIX).$(SUFFIX) \ @@ -920,7 +933,39 @@ ifeq ($(OS), AIX) else $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ endif - + +endif + +ifdef SCOMMNCOPY +$(KDIR)scomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SCOMMNCOPY) +else ifdef SGEMMINCOPY +$(KDIR)scomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMINCOPY) +else +$(KDIR)scomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMONCOPY) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -UDOUBLE -UCOMPLEX $< -o - > scommncopy.s + $(M4_AIX) scommncopy.s > scommncopy_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX scommncopy_nomacros.s -o $@ + rm scommncopy.s scommncopy_nomacros.s +else + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ +endif + +ifdef SCOMMTCOPY +$(KDIR)scomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SCOMMTCOPY) +else ifdef SGEMMITCOPY +$(KDIR)scomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMITCOPY) +else +$(KDIR)scomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMOTCOPY) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -UDOUBLE -UCOMPLEX $< -o - > scommtcopy.s + $(M4_AIX) scommtcopy.s > scommtcopy_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX scommtcopy_nomacros.s -o $@ + rm scommtcopy.s scommtcopy_nomacros.s +else + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ endif $(KDIR)$(DGEMMONCOPYOBJ) : $(KERNELDIR)/$(DGEMMONCOPY) @@ -1049,6 +1094,21 @@ else $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ endif + +ifdef SCOMMKERNEL +$(KDIR)scomm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SCOMMKERNEL) $(SGEMMDEPEND) +else +$(KDIR)scomm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) $(SGEMMDEPEND) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -UDOUBLE -UCOMPLEX $< -o - > scomm_kernel$(TSUFFIX).s + $(M4_AIX) scomm_kernel$(TSUFFIX).s > scomm_kernel$(TSUFFIX)_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX scomm_kernel$(TSUFFIX)_nomacros.s -o $@ + rm scomm_kernel$(TSUFFIX).s scomm_kernel$(TSUFFIX)_nomacros.s +else + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ +endif + ifdef USE_DIRECT_SGEMM ifeq ($(ARCH), x86_64) $(KDIR)sgemm_direct_performant$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMDIRECTPERFORMANT) @@ -1128,6 +1188,20 @@ else $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ endif +ifdef DCOMMKERNEL +$(KDIR)dcomm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DCOMMKERNEL) $(DGEMMDEPEND) +else +$(KDIR)dcomm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) $(DGEMMDEPEND) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -DDOUBLE -UCOMPLEX $< -o - > dcomm_kernel$(TSUFFIX).s + $(M4_AIX) dcomm_kernel$(TSUFFIX).s > dcomm_kernel$(TSUFFIX)_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX dcomm_kernel$(TSUFFIX)_nomacros.s -o $@ + rm dcomm_kernel$(TSUFFIX).s dcomm_kernel$(TSUFFIX)_nomacros.s +else + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ +endif + $(KDIR)qgemm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(QGEMMKERNEL) $(QGEMMDEPEND) $(CC) $(CFLAGS) -c -DXDOUBLE -UCOMPLEX $< -o $@ @@ -1171,6 +1245,62 @@ else $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCC $< -o $@ endif +ifdef CCOMMKERNEL +$(KDIR)ccomm_kernel_n$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) $(CGEMMDEPEND) +else +$(KDIR)ccomm_kernel_n$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -UDOUBLE -DCOMPLEX -DNN $< -o - > ccomm_kernel_n.s + $(M4_AIX) ccomm_kernel_n.s > ccomm_kernel_n_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNN ccomm_kernel_n_nomacros.s -o $@ + rm ccomm_kernel_n.s ccomm_kernel_n_nomacros.s +else + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNN $< -o $@ +endif + +ifdef CCOMMKERNEL +$(KDIR)ccomm_kernel_l$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) $(CGEMMDEPEND) +else +$(KDIR)ccomm_kernel_l$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -UDOUBLE -DCOMPLEX -DCN $< -o - > ccomm_kernel_l.s + $(M4_AIX) ccomm_kernel_l.s > ccomm_kernel_l_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCN ccomm_kernel_l_nomacros.s -o $@ + rm ccomm_kernel_l.s ccomm_kernel_l_nomacros.s +else + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCN $< -o $@ +endif + +ifdef CCOMMKERNEL +$(KDIR)ccomm_kernel_r$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) $(CGEMMDEPEND) +else +$(KDIR)ccomm_kernel_r$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -UDOUBLE -DCOMPLEX -DNC $< -o - > ccomm_kernel_r.s + $(M4_AIX) ccomm_kernel_r.s > ccomm_kernel_r_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNC ccomm_kernel_r_nomacros.s -o $@ + rm ccomm_kernel_r.s ccomm_kernel_r_nomacros.s +else + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNC $< -o $@ +endif + +ifdef CCOMMKERNEL +$(KDIR)ccomm_kernel_b$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) $(CGEMMDEPEND) +else +$(KDIR)ccomm_kernel_b$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -UDOUBLE -DCOMPLEX -DCC $< -o - > ccomm_kernel_b.s + $(M4_AIX) ccomm_kernel_b.s > ccomm_kernel_b_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCC ccomm_kernel_b_nomacros.s -o $@ + rm ccomm_kernel_b.s ccomm_kernel_b_nomacros.s +else + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCC $< -o $@ +endif + $(KDIR)zgemm_kernel_n$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) ifeq ($(OS), AIX) $(CC) $(CFLAGS) -S -DDOUBLE -DCOMPLEX -DNN $< -o - > zgemm_kernel_n.s @@ -1219,6 +1349,70 @@ else $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCC $< -o $@ endif +ifdef ZCOMMKERNEL +$(KDIR)zcomm_kernel_n$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) $(ZGEMMDEPEND) +else +$(KDIR)zcomm_kernel_n$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -DDOUBLE -DCOMPLEX -DNN $< -o - > zcomm_kernel_n.s + $(M4_AIX) zcomm_kernel_n.s > zcomm_kernel_n_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNN zcomm_kernel_n_nomacros.s -o $@ + rm zcomm_kernel_n.s zcomm_kernel_n_nomacros.s +else ifeq ($(CORE),SANDYBRIDGE) + $(CC) $(filter-out -mavx,$(CFLAGS)) -c -DDOUBLE -DCOMPLEX -DNN $< -o $@ +else + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNN $< -o $@ +endif + +ifdef ZCOMMKERNEL +$(KDIR)zcomm_kernel_l$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) $(ZGEMMDEPEND) +else +$(KDIR)zcomm_kernel_l$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -DDOUBLE -DCOMPLEX -DCN $< -o - > zcomm_kernel_l.s + $(M4_AIX) zcomm_kernel_l.s > zcomm_kernel_l_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCN zcomm_kernel_l_nomacros.s -o $@ + rm zcomm_kernel_l.s zcomm_kernel_l_nomacros.s +else ifeq ($(CORE),SANDYBRIDGE) + $(CC) $(filter-out -mavx,$(CFLAGS)) -c -DDOUBLE -DCOMPLEX -DCN $< -o $@ +else + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCN $< -o $@ +endif + +ifdef ZCOMMKERNEL +$(KDIR)zcomm_kernel_r$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) $(ZGEMMDEPEND) +else +$(KDIR)zcomm_kernel_r$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -DDOUBLE -DCOMPLEX -DNC $< -o - > zcomm_kernel_r.s + $(M4_AIX) zcomm_kernel_r.s > zcomm_kernel_r_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNC zcomm_kernel_r_nomacros.s -o $@ + rm zcomm_kernel_r.s zcomm_kernel_r_nomacros.s +else ifeq ($(CORE),SANDYBRIDGE) + $(CC) $(filter-out -mavx,$(CFLAGS)) -c -DDOUBLE -DCOMPLEX -DNC $< -o $@ +else + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNC $< -o $@ +endif + +ifdef ZCOMMKERNEL +$(KDIR)zcomm_kernel_b$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) $(ZGEMMDEPEND) +else +$(KDIR)zcomm_kernel_b$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -DDOUBLE -DCOMPLEX -DCC $< -o - > zcomm_kernel_b.s + $(M4_AIX) zcomm_kernel_b.s > zcomm_kernel_b_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCC zcomm_kernel_b_nomacros.s -o $@ + rm zcomm_kernel_b.s zcomm_kernel_b_nomacros.s +else ifeq ($(CORE),SANDYBRIDGE) + $(CC) $(filter-out -mavx,$(CFLAGS)) -c -DDOUBLE -DCOMPLEX -DCC $< -o $@ +else + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCC $< -o $@ +endif + $(KDIR)xgemm_kernel_n$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(XGEMMKERNEL) $(XGEMMDEPEND) $(CC) $(CFLAGS) -c -DXDOUBLE -DCOMPLEX -DNN $< -o $@ @@ -1532,17 +1726,34 @@ else $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -DTRANSA -DCONJ -DNC $< -o $@ endif +else + +ifdef SCOMMKERNEL +$(KDIR)strmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SCOMMKERNEL) else $(KDIR)strmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) +endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -DLEFT -UTRANSA $< -o $@ +ifdef SCOMMKERNEL +$(KDIR)strmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SCOMMKERNEL) +else $(KDIR)strmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) +endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -DLEFT -DTRANSA $< -o $@ +ifdef SCOMMKERNEL +$(KDIR)strmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SCOMMKERNEL) +else $(KDIR)strmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) +endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -UTRANSA $< -o $@ +ifdef SCOMMKERNEL +$(KDIR)strmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SCOMMKERNEL) +else $(KDIR)strmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) +endif ifeq ($(OS), AIX) $(CC) $(CFLAGS) -S -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o - > strmm_kernel_rt.s $(M4_AIX) strmm_kernel_rt.s > strmm_kernel_rt_nomacros.s @@ -1552,16 +1763,32 @@ else $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o $@ endif +ifdef DCOMMKERNEL +$(KDIR)dtrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DCOMMKERNEL) +else $(KDIR)dtrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) +endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -UTRANSA $< -o $@ +ifdef DCOMMKERNEL +$(KDIR)dtrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DCOMMKERNEL) +else $(KDIR)dtrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) +endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -DTRANSA $< -o $@ +ifdef DCOMMKERNEL +$(KDIR)dtrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DCOMMKERNEL) +else $(KDIR)dtrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) +endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -ULEFT -UTRANSA $< -o $@ +ifdef DCOMMKERNEL +$(KDIR)dtrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DCOMMKERNEL) +else $(KDIR)dtrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) +endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o $@ $(KDIR)qtrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(QGEMMKERNEL) @@ -1576,74 +1803,144 @@ $(KDIR)qtrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(QGEMMKERNEL) $(KDIR)qtrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(QGEMMKERNEL) $(CC) $(CFLAGS) -c -DTRMMKERNEL -DXDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o $@ +ifdef CCOMMKERNEL +$(KDIR)ctrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +else $(KDIR)ctrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) +endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -UTRANSA -UCONJ -DNN $< -o $@ +ifdef CCOMMKERNEL +$(KDIR)ctrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +else $(KDIR)ctrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) +endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -DTRANSA -UCONJ -DNN $< -o $@ +ifdef CCOMMKERNEL +$(KDIR)ctrmm_kernel_LR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +else $(KDIR)ctrmm_kernel_LR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) +endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -UTRANSA -DCONJ -DCN $< -o $@ +ifdef CCOMMKERNEL +$(KDIR)ctrmm_kernel_LC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +else $(KDIR)ctrmm_kernel_LC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) +endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -DTRANSA -DCONJ -DCN $< -o $@ +ifdef CCOMMKERNEL +$(KDIR)ctrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +else $(KDIR)ctrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) +endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -UTRANSA -UCONJ -DNN $< -o $@ +ifdef CCOMMKERNEL +$(KDIR)ctrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +else $(KDIR)ctrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) +endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -DTRANSA -UCONJ -DNN $< -o $@ +ifdef CCOMMKERNEL +$(KDIR)ctrmm_kernel_RR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +else $(KDIR)ctrmm_kernel_RR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) +endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -UTRANSA -DCONJ -DNC $< -o $@ +ifdef CCOMMKERNEL +$(KDIR)ctrmm_kernel_RC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +else $(KDIR)ctrmm_kernel_RC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) +endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -DTRANSA -DCONJ -DNC $< -o $@ +ifdef ZCOMMKERNEL +$(KDIR)ztrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +else $(KDIR)ztrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) +endif ifeq ($(CORE),SANDYBRIDGE) $(CC) $(filter-out -mavx,$(CFLAGS)) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -UTRANSA -UCONJ -DNN $< -o $@ else $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -UTRANSA -UCONJ -DNN $< -o $@ endif +ifdef ZCOMMKERNEL +$(KDIR)ztrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +else $(KDIR)ztrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) +endif ifeq ($(CORE),SANDYBRIDGE) $(CC) $(filter-out -mavx,$(CFLAGS)) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -DTRANSA -UCONJ -DNN $< -o $@ else $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -DTRANSA -UCONJ -DNN $< -o $@ endif + +ifdef ZCOMMKERNEL +$(KDIR)ztrmm_kernel_LR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +else $(KDIR)ztrmm_kernel_LR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) +endif ifeq ($(CORE),SANDYBRIDGE) $(CC) $(filter-out -mavx,$(CFLAGS)) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -UTRANSA -DCONJ -DCN $< -o $@ else $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -UTRANSA -DCONJ -DCN $< -o $@ endif + +ifdef ZCOMMKERNEL +$(KDIR)ztrmm_kernel_LC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +else $(KDIR)ztrmm_kernel_LC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) +endif ifeq ($(CORE),SANDYBRIDGE) $(CC) $(filter-out -mavx,$(CFLAGS)) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -DTRANSA -DCONJ -DCN $< -o $@ else $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -DTRANSA -DCONJ -DCN $< -o $@ endif + +ifdef ZCOMMKERNEL +$(KDIR)ztrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +else $(KDIR)ztrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) +endif ifeq ($(CORE),SANDYBRIDGE) $(CC) $(filter-out -mavx,$(CFLAGS)) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -UTRANSA -UCONJ -DNN $< -o $@ else $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -UTRANSA -UCONJ -DNN $< -o $@ endif + +ifdef ZCOMMKERNEL +$(KDIR)ztrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +else $(KDIR)ztrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) +endif ifeq ($(CORE),SANDYBRIDGE) $(CC) $(filter-out -mavx,$(CFLAGS)) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -DTRANSA -UCONJ -DNN $< -o $@ else $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -DTRANSA -UCONJ -DNN $< -o $@ endif + +ifdef ZCOMMKERNEL +$(KDIR)ztrmm_kernel_RR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +else $(KDIR)ztrmm_kernel_RR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) +endif ifeq ($(CORE),SANDYBRIDGE) $(CC) $(filter-out -mavx,$(CFLAGS)) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -UTRANSA -DCONJ -DNC $< -o $@ else $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -UTRANSA -DCONJ -DNC $< -o $@ endif + +ifdef ZCOMMKERNEL +$(KDIR)ztrmm_kernel_RC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +else $(KDIR)ztrmm_kernel_RC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) +endif ifeq ($(CORE),SANDYBRIDGE) $(CC) $(filter-out -mavx,$(CFLAGS)) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -DTRANSA -DCONJ -DNC $< -o $@ else @@ -1652,8 +1949,6 @@ endif endif - - $(KDIR)xtrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(XGEMMKERNEL) $(CC) $(CFLAGS) -c -DTRMMKERNEL -DXDOUBLE -DCOMPLEX -DLEFT -UTRANSA -UCONJ -DNN $< -o $@ @@ -1939,6 +2234,39 @@ $(KDIR)dtrmm_iltncopy$(TSUFFIX).$(SUFFIX) : generic/trmm_ltcopy_$(DGEMM_UNROLL_M $(CC) -c $(CFLAGS) $(NO_UNINITIALIZED_WARN) -DDOUBLE -UCOMPLEX -UOUTER -DLOWER -UUNIT $< -o $@ endif +ifdef DCOMMNCOPY +$(KDIR)dcomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DCOMMNCOPY) +else ifdef DGEMMINCOPY +$(KDIR)dcomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMINCOPY) +else +$(KDIR)dcomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMONCOPY) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -DDOUBLE -UCOMPLEX $< -o - > dcomm_ncopy.s + $(M4_AIX) dcomm_ncopy.s > dcomm_ncopy_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX dcomm_ncopy_nomacros.s -o $@ + rm dcomm_ncopy.s dcomm_ncopy_nomacros.s +else + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ +endif + +ifdef DCOMMTCOPY +$(KDIR)dcomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DCOMMTCOPY) +else ifdef DGEMMITCOPY +$(KDIR)dcomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMITCOPY) +else +$(KDIR)dcomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMOTCOPY) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -DDOUBLE -UCOMPLEX $< -o - > dcomm_tcopy.s + $(M4_AIX) dcomm_tcopy.s > dcomm_tcopy_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX dcomm_tcopy_nomacros.s -o $@ + rm dcomm_tcopy.s dcomm_tcopy_nomacros.s +else + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ +endif + + $(KDIR)dtrmm_ounucopy$(TSUFFIX).$(SUFFIX) : generic/trmm_uncopy_$(DGEMM_UNROLL_N).c $(CC) -c $(CFLAGS) $(NO_UNINITIALIZED_WARN) -DDOUBLE -UCOMPLEX -DOUTER -ULOWER -DUNIT $< -o $@ @@ -2067,6 +2395,31 @@ $(KDIR)ctrmm_iltncopy$(TSUFFIX).$(SUFFIX) : generic/ztrmm_ltcopy_$(CGEMM_UNROLL_ $(CC) -c $(CFLAGS) $(NO_UNINITIALIZED_WARN) -UDOUBLE -DCOMPLEX -UOUTER -ULOWER -UUNIT $< -o $@ endif +ifdef CCOMMNCOPY +$(KDIR)ccomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMNCOPY) +else ifdef CGEMMINCOPY +$(KDIR)ccomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMINCOPY) +else +$(KDIR)ccomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMONCOPY) +endif + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ + +ifdef CCOMMTCOPY +$(KDIR)ccomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMTCOPY) +else ifdef CGEMMITCOPY +$(KDIR)ccomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMITCOPY) +else +$(KDIR)ccomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMOTCOPY) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -UDOUBLE -UCOMPLEX -S $< -o - > ccomm_tcopy.s + $(M4_AIX) ccomm_tcopy.s > ccomm_tcopy_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX ccomm_tcopy_nomacros.s -o $@ + rm ccomm_tcopy.s ccomm_tcopy_nomacros.s +else + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ +endif + $(KDIR)ctrmm_ounucopy$(TSUFFIX).$(SUFFIX) : generic/ztrmm_uncopy_$(CGEMM_UNROLL_N).c $(CC) -c $(CFLAGS) $(NO_UNINITIALIZED_WARN) -UDOUBLE -DCOMPLEX -DOUTER -ULOWER -DUNIT $< -o $@ @@ -2147,6 +2500,32 @@ $(KDIR)ztrmm_iltncopy$(TSUFFIX).$(SUFFIX) : generic/ztrmm_ltcopy_$(ZGEMM_UNROLL_ $(CC) -c $(CFLAGS) $(NO_UNINITIALIZED_WARN) -DDOUBLE -DCOMPLEX -UOUTER -DLOWER -UUNIT $< -o $@ endif +ifdef ZCOMMNCOPY +$(KDIR)zcomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMNCOPY) +else ifdef ZGEMMINCOPY +$(KDIR)zcomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMINCOPY) +else +$(KDIR)zcomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMONCOPY) +endif + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ + +ifdef ZCOMMTCOPY +$(KDIR)zcomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMTCOPY) +else ifdef ZGEMMITCOPY +$(KDIR)zcomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMITCOPY) +else +$(KDIR)zcomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMOTCOPY) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -DDOUBLE -UCOMPLEX $< -o - > zcomm_tcopy.s + $(M4_AIX) zcomm_tcopy.s > zcomm_tcopy_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX zcomm_tcopy_nomacros.s -o $@ + rm zcomm_tcopy.s zcomm_tcopy_nomacros.s +else + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ +endif + + $(KDIR)ztrmm_ounucopy$(TSUFFIX).$(SUFFIX) : generic/ztrmm_uncopy_$(ZGEMM_UNROLL_N).c $(CC) -c $(CFLAGS) $(NO_UNINITIALIZED_WARN) -DDOUBLE -DCOMPLEX -DOUTER -ULOWER -DUNIT $< -o $@ @@ -3222,6 +3601,25 @@ $(SGEMMITCOPYOBJ_P) : $(KERNELDIR)/$(SGEMMITCOPY) endif +ifdef SCOMMNCOPY +$(KDIR)scomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SCOMMNCOPY) +else ifdef SGEMMINCOPY +$(KDIR)scomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMINCOPY) +else +$(KDIR)scomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMONCOPY) +endif + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ + +ifdef SCOMMTCOPY +$(KDIR)scomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SCOMMTCOPY) +else ifdef SGEMMITCOPY +$(KDIR)scomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMITCOPY) +else +$(KDIR)scomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMOTCOPY) +endif + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ + + $(DGEMMONCOPYOBJ_P) : $(KERNELDIR)/$(DGEMMONCOPY) $(CC) $(PFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ @@ -3238,6 +3636,24 @@ $(DGEMMITCOPYOBJ_P) : $(KERNELDIR)/$(DGEMMITCOPY) endif +ifdef DCOMMNCOPY +$(KDIR)dcomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DCOMMNCOPY) +else ifdef DGEMMINCOPY +$(KDIR)dcomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMINCOPY) +else +$(KDIR)dcomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMONCOPY) +endif + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ + +ifdef DCOMMTCOPY +$(KDIR)dcomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DCOMMTCOPY) +else ifdef DGEMMITCOPY +$(KDIR)dcomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMITCOPY) +else +$(KDIR)dcomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMOTCOPY) +endif + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ + ifdef EXPRECISION $(QGEMMONCOPYOBJ_P) : $(KERNELDIR)/$(QGEMMONCOPY) @@ -3274,6 +3690,24 @@ $(CGEMMITCOPYOBJ_P) : $(KERNELDIR)/$(CGEMMITCOPY) endif +ifdef CCOMMNCOPY +$(KDIR)ccomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMNCOPY) +else ifdef CGEMMINCOPY +$(KDIR)ccomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMINCOPY) +else +$(KDIR)ccomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMONCOPY) +endif + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ + +ifdef CCOMMTCOPY +$(KDIR)ccomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMTCOPY) +else ifdef CGEMMITCOPY +$(KDIR)ccomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMITCOPY) +else +$(KDIR)ccomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMOTCOPY) +endif + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ + $(ZGEMMONCOPYOBJ_P) : $(KERNELDIR)/$(ZGEMMONCOPY) $(CC) $(PFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ @@ -3290,6 +3724,24 @@ $(ZGEMMITCOPYOBJ_P) : $(KERNELDIR)/$(ZGEMMITCOPY) endif +ifdef ZCOMMNCOPY +$(KDIR)zcomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMNCOPY) +else ifdef ZGEMMINCOPY +$(KDIR)zcomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMINCOPY) +else +$(KDIR)zcomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMONCOPY) +endif + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ + +ifdef ZCOMMTCOPY +$(KDIR)zcomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMTCOPY) +else ifdef ZGEMMITCOPY +$(KDIR)zcomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMITCOPY) +else +$(KDIR)zcomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMOTCOPY) +endif + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ + ifdef EXPRECISION $(XGEMMONCOPYOBJ_P) : $(KERNELDIR)/$(XGEMMONCOPY) @@ -3326,9 +3778,23 @@ endif $(KDIR)sgemm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) $(SGEMMDEPEND) $(CC) $(PFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ +ifdef SCOMMKERNEL +$(KDIR)scomm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SCOMMKERNEL) $(SGEMMDEPEND) +else +$(KDIR)scomm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) $(SGEMMDEPEND) +endif + $(CC) $(PFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ + $(KDIR)dgemm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) $(DGEMMDEPEND) $(CC) $(PFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ +ifdef DCOMMKERNEL +$(KDIR)dcomm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DCOMMKERNEL) $(DGEMMDEPEND) +else +$(KDIR)dcomm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) $(DGEMMDEPEND) +endif + $(CC) $(PFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ + $(KDIR)qgemm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(QGEMMKERNEL) $(QGEMMDEPEND) $(CC) $(PFLAGS) -c -DXDOUBLE -UCOMPLEX $< -o $@ @@ -3351,6 +3817,41 @@ endif $(KDIR)cgemm_kernel_b$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) $(CC) $(PFLAGS) -c -UDOUBLE -DCOMPLEX -DCC $< -o $@ +ifdef CCOMMKERNEL +$(KDIR)ccomm_kernel_n$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) $(CGEMMDEPEND) +else +$(KDIR)ccomm_kernel_n$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +endif + $(CC) $(PFLAGS) -c -UDOUBLE -DCOMPLEX -DNN $< -o $@ + +ifdef CCOMMKERNEL +$(KDIR)ccomm_kernel_l$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) $(CGEMMDEPEND) +else +$(KDIR)ccomm_kernel_l$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +endif + $(CC) $(PFLAGS) -c -UDOUBLE -DCOMPLEX -DCN $< -o $@ + +ifdef CCOMMKERNEL +$(KDIR)ccomm_kernel_r$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) $(CGEMMDEPEND) +else +$(KDIR)ccomm_kernel_r$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +endif +ifeq ($(OS), AIX) + $(CC) $(PFLAGS) -S -UDOUBLE -DCOMPLEX -DNC $< -o - > ccomm_kernel_r.s + $(M4_AIX) ccomm_kernel_r.s > ccomm_kernel_r_nomacros.s + $(CC) $(PFLAGS) -c -UDOUBLE -DCOMPLEX -DNC ccomm_kernel_r_nomacros.s -o $@ + rm ccomm_kernel_r.s ccomm_kernel_r_nomacros.s +else + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNC $< -o $@ +endif + +ifdef CCOMMKERNEL +$(KDIR)ccomm_kernel_b$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) $(CGEMMDEPEND) +else +$(KDIR)ccomm_kernel_b$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +endif + $(CC) $(PFLAGS) -c -UDOUBLE -DCOMPLEX -DCC $< -o $@ + $(KDIR)zgemm_kernel_n$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) $(CC) $(PFLAGS) -c -DDOUBLE -DCOMPLEX -DNN $< -o $@ @@ -3363,6 +3864,34 @@ $(KDIR)zgemm_kernel_r$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMM $(KDIR)zgemm_kernel_b$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) $(CC) $(PFLAGS) -c -DDOUBLE -DCOMPLEX -DCC $< -o $@ +ifdef ZCOMMKERNEL +$(KDIR)zcomm_kernel_n$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) $(ZGEMMDEPEND) +else +$(KDIR)zcomm_kernel_n$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +endif + $(CC) $(PFLAGS) -c -DDOUBLE -DCOMPLEX -DNN $< -o $@ + +ifdef ZCOMMKERNEL +$(KDIR)zcomm_kernel_l$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) $(ZGEMMDEPEND) +else +$(KDIR)zcomm_kernel_l$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +endif + $(CC) $(PFLAGS) -c -DDOUBLE -DCOMPLEX -DCN $< -o $@ + +ifdef ZCOMMKERNEL +$(KDIR)zcomm_kernel_r$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) $(ZGEMMDEPEND) +else +$(KDIR)zcomm_kernel_r$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +endif + $(CC) $(PFLAGS) -c -DDOUBLE -DCOMPLEX -DNC $< -o $@ + +ifdef ZCOMMKERNEL +$(KDIR)zcomm_kernel_b$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) $(ZGEMMDEPEND) +else +$(KDIR)zcomm_kernel_b$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +endif + $(CC) $(PFLAGS) -c -DDOUBLE -DCOMPLEX -DCC $< -o $@ + $(KDIR)xgemm_kernel_n$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(XGEMMKERNEL) $(XGEMMDEPEND) $(CC) $(PFLAGS) -c -DXDOUBLE -DCOMPLEX -DNN $< -o $@ @@ -3375,16 +3904,32 @@ $(KDIR)xgemm_kernel_r$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(XGEMMKERNEL) $(XGEMM $(KDIR)xgemm_kernel_b$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(XGEMMKERNEL) $(XGEMMDEPEND) $(CC) $(PFLAGS) -c -DXDOUBLE -DCOMPLEX -DCC $< -o $@ +ifdef SCOMMKERNEL +$(KDIR)strmm_kernel_LN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SCOMMKERNEL) +else $(KDIR)strmm_kernel_LN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -DLEFT -UTRANSA $< -o $@ +ifdef SCOMMKERNEL +$(KDIR)strmm_kernel_LT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SCOMMKERNEL) +else $(KDIR)strmm_kernel_LT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -DLEFT -DTRANSA $< -o $@ +ifdef SCOMMKERNEL +$(KDIR)strmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SCOMMKERNEL) +else $(KDIR)strmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -UTRANSA $< -o $@ +ifdef SCOMMKERNEL +$(KDIR)strmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SCOMMKERNEL) +else $(KDIR)strmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) +endif ifeq ($(OS), AIX) $(CC) $(CFLAGS) -S -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o - > strmm_kernel_rt.s $(M4_AIX) strmmkernel_rn.s > strmm_kernel_rt_nomacros.s @@ -3394,16 +3939,32 @@ else $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o $@ endif +ifdef DCOMMKERNEL +$(KDIR)dtrmm_kernel_LN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DCOMMKERNEL) +else $(KDIR)dtrmm_kernel_LN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -UTRANSA $< -o $@ +ifdef DCOMMKERNEL +$(KDIR)dtrmm_kernel_LT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DCOMMKERNEL) +else $(KDIR)dtrmm_kernel_LT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -DTRANSA $< -o $@ +ifdef DCOMMKERNEL +$(KDIR)dtrmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DCOMMKERNEL) +else $(KDIR)dtrmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -ULEFT -UTRANSA $< -o $@ +ifdef DCOMMKERNEL +$(KDIR)dtrmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DCOMMKERNEL) +else $(KDIR)dtrmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o $@ $(KDIR)qtrmm_kernel_LN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(QGEMMKERNEL) @@ -3418,52 +3979,116 @@ $(KDIR)qtrmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(QGEMMKERNEL) $(KDIR)qtrmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(QGEMMKERNEL) $(CC) $(PFLAGS) -c -DTRMMKERNEL -DXDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o $@ +ifdef CCOMMKERNEL +$(KDIR)ctrmm_kernel_LN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +else $(KDIR)ctrmm_kernel_LN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -UTRANSA -UCONJ -DNN $< -o $@ +ifdef CCOMMKERNEL +$(KDIR)ctrmm_kernel_LT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +else $(KDIR)ctrmm_kernel_LT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -DTRANSA -UCONJ -DNN $< -o $@ +ifdef CCOMMKERNEL +$(KDIR)ctrmm_kernel_LR$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +else $(KDIR)ctrmm_kernel_LR$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -UTRANSA -DCONJ -DCN $< -o $@ +ifdef CCOMMKERNEL +$(KDIR)ctrmm_kernel_LC$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +else $(KDIR)ctrmm_kernel_LC$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -DTRANSA -DCONJ -DCN $< -o $@ +ifdef CCOMMKERNEL +$(KDIR)ctrmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +else $(KDIR)ctrmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -UTRANSA -UCONJ -DNN $< -o $@ +ifdef CCOMMKERNEL +$(KDIR)ctrmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +else $(KDIR)ctrmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -DTRANSA -UCONJ -DNN $< -o $@ +ifdef CCOMMKERNEL +$(KDIR)ctrmm_kernel_RR$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +else $(KDIR)ctrmm_kernel_RR$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -UTRANSA -DCONJ -DNC $< -o $@ +ifdef CCOMMKERNEL +$(KDIR)ctrmm_kernel_RC$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +else $(KDIR)ctrmm_kernel_RC$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -DTRANSA -DCONJ -DNC $< -o $@ +ifdef ZCOMMKERNEL +$(KDIR)ztrmm_kernel_LN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +else $(KDIR)ztrmm_kernel_LN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -UTRANSA -UCONJ -DNN $< -o $@ +ifdef ZCOMMKERNEL +$(KDIR)ztrmm_kernel_LT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +else $(KDIR)ztrmm_kernel_LT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -DTRANSA -UCONJ -DNN $< -o $@ +ifdef ZCOMMKERNEL +$(KDIR)ztrmm_kernel_LR$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +else $(KDIR)ztrmm_kernel_LR$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -UTRANSA -DCONJ -DCN $< -o $@ +ifdef ZCOMMKERNEL +$(KDIR)ztrmm_kernel_LC$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +else $(KDIR)ztrmm_kernel_LC$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -DTRANSA -DCONJ -DCN $< -o $@ +ifdef ZCOMMKERNEL +$(KDIR)ztrmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +else $(KDIR)ztrmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -UTRANSA -UCONJ -DNN $< -o $@ +ifdef ZCOMMKERNEL +$(KDIR)ztrmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +else $(KDIR)ztrmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -DTRANSA -UCONJ -DNN $< -o $@ +ifdef ZCOMMKERNEL +$(KDIR)ztrmm_kernel_RR$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +else $(KDIR)ztrmm_kernel_RR$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -UTRANSA -DCONJ -DNC $< -o $@ +ifdef ZCOMMKERNEL +$(KDIR)ztrmm_kernel_RC$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +else $(KDIR)ztrmm_kernel_RC$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) +endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -DTRANSA -DCONJ -DNC $< -o $@ $(KDIR)xtrmm_kernel_LN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(XGEMMKERNEL) diff --git a/kernel/arm64/KERNEL.ARMV8SVE b/kernel/arm64/KERNEL.ARMV8SVE index a8371a03cd..1c10d8e257 100644 --- a/kernel/arm64/KERNEL.ARMV8SVE +++ b/kernel/arm64/KERNEL.ARMV8SVE @@ -123,11 +123,12 @@ DSDOTKERNEL = dot.S DGEMM_BETA = dgemm_beta.S SGEMM_BETA = sgemm_beta.S -SGEMMKERNEL = sgemm_kernel_sve_v2x$(SGEMM_UNROLL_N).S +SGEMMKERNEL = sgemm_kernel_inter_sve_v2x$(SGEMM_UNROLL_N).S +SCOMMKERNEL = sgemm_kernel_sve_v2x$(SGEMM_UNROLL_N).S STRMMKERNEL = strmm_kernel_sve_v1x$(SGEMM_UNROLL_N).S -SGEMMINCOPY = gemm_ncopy_sve_v1x$(SGEMM_UNROLL_N).c -SGEMMITCOPY = gemm_tcopy_sve_v1x$(SGEMM_UNROLL_N).c +SGEMMINCOPY = gemm_ncopy_sve_v2x$(SGEMM_UNROLL_N).c +SGEMMITCOPY = gemm_tcopy_sve_v2x$(SGEMM_UNROLL_N).c SGEMMONCOPY = sgemm_ncopy_$(SGEMM_UNROLL_N).S SGEMMOTCOPY = sgemm_tcopy_$(SGEMM_UNROLL_N).S @@ -135,6 +136,8 @@ SGEMMINCOPYOBJ = sgemm_incopy$(TSUFFIX).$(SUFFIX) SGEMMITCOPYOBJ = sgemm_itcopy$(TSUFFIX).$(SUFFIX) SGEMMONCOPYOBJ = sgemm_oncopy$(TSUFFIX).$(SUFFIX) SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX) +SCOMMNCOPYOBJ = scomm_ncopy$(TSUFFIX).$(SUFFIX) +SCOMMTCOPYOBJ = scomm_tcopy$(TSUFFIX).$(SUFFIX) SGEMM_SMALL_M_PERMIT = gemm_small_kernel_permit_sve.c SGEMM_SMALL_K_NT = sgemm_small_kernel_nt_sve.c @@ -150,15 +153,18 @@ STRMMUNCOPY_M = trmm_uncopy_sve_v1.c STRMMLNCOPY_M = trmm_lncopy_sve_v1.c STRMMUTCOPY_M = trmm_utcopy_sve_v1.c STRMMLTCOPY_M = trmm_ltcopy_sve_v1.c +SCOMMNCOPY = gemm_ncopy_sve_v1x$(SGEMM_UNROLL_N).c +SCOMMTCOPY = gemm_tcopy_sve_v1x$(SGEMM_UNROLL_N).c SSYMMUCOPY_M = symm_ucopy_sve.c SSYMMLCOPY_M = symm_lcopy_sve.c -DGEMMKERNEL = dgemm_kernel_sve_v2x$(DGEMM_UNROLL_N).S +DGEMMKERNEL = dgemm_kernel_inter_sve_v2x$(DGEMM_UNROLL_N).S +DCOMMKERNEL = dgemm_kernel_sve_v2x$(DGEMM_UNROLL_N).S DTRMMKERNEL = dtrmm_kernel_sve_v1x$(DGEMM_UNROLL_N).S -DGEMMINCOPY = gemm_ncopy_sve_v1x$(DGEMM_UNROLL_N).c -DGEMMITCOPY = gemm_tcopy_sve_v1x$(DGEMM_UNROLL_N).c +DGEMMINCOPY = gemm_ncopy_sve_v2x$(DGEMM_UNROLL_N).c +DGEMMITCOPY = gemm_tcopy_sve_v2x$(DGEMM_UNROLL_N).c DGEMMONCOPY = dgemm_ncopy_$(DGEMM_UNROLL_N).S DGEMMOTCOPY = dgemm_tcopy_$(DGEMM_UNROLL_N).S @@ -166,6 +172,8 @@ DGEMMINCOPYOBJ = dgemm_incopy$(TSUFFIX).$(SUFFIX) DGEMMITCOPYOBJ = dgemm_itcopy$(TSUFFIX).$(SUFFIX) DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX) DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX) +DCOMMNCOPYOBJ = dcomm_ncopy$(TSUFFIX).$(SUFFIX) +DCOMMTCOPYOBJ = dcomm_tcopy$(TSUFFIX).$(SUFFIX) DGEMM_SMALL_M_PERMIT = gemm_small_kernel_permit_sve.c DGEMM_SMALL_K_NT = dgemm_small_kernel_nt_sve.c @@ -181,11 +189,14 @@ DTRMMUNCOPY_M = trmm_uncopy_sve_v1.c DTRMMLNCOPY_M = trmm_lncopy_sve_v1.c DTRMMUTCOPY_M = trmm_utcopy_sve_v1.c DTRMMLTCOPY_M = trmm_ltcopy_sve_v1.c +DCOMMNCOPY = gemm_ncopy_sve_v1x$(DGEMM_UNROLL_N).c +DCOMMTCOPY = gemm_tcopy_sve_v1x$(DGEMM_UNROLL_N).c DSYMMUCOPY_M = symm_ucopy_sve.c DSYMMLCOPY_M = symm_lcopy_sve.c CGEMMKERNEL = cgemm_kernel_sve_v1x$(ZGEMM_UNROLL_N).S +CCOMMKERNEL = cgemm_kernel_sve_v1x$(ZGEMM_UNROLL_N).S CTRMMKERNEL = ctrmm_kernel_sve_v1x$(ZGEMM_UNROLL_N).S CGEMMINCOPY = gemm_ncopy_complex_sve_v1x$(ZGEMM_UNROLL_N).c @@ -197,11 +208,15 @@ CGEMMINCOPYOBJ = cgemm_incopy$(TSUFFIX).$(SUFFIX) CGEMMITCOPYOBJ = cgemm_itcopy$(TSUFFIX).$(SUFFIX) CGEMMONCOPYOBJ = cgemm_oncopy$(TSUFFIX).$(SUFFIX) CGEMMOTCOPYOBJ = cgemm_otcopy$(TSUFFIX).$(SUFFIX) +CCOMMNCOPYOBJ = ccomm_ncopy$(TSUFFIX).$(SUFFIX) +CCOMMTCOPYOBJ = ccomm_tcopy$(TSUFFIX).$(SUFFIX) CTRMMUNCOPY_M = ztrmm_uncopy_sve_v1.c CTRMMLNCOPY_M = ztrmm_lncopy_sve_v1.c CTRMMUTCOPY_M = ztrmm_utcopy_sve_v1.c CTRMMLTCOPY_M = ztrmm_ltcopy_sve_v1.c +CCOMMNCOPY = gemm_ncopy_complex_sve_v1x$(ZGEMM_UNROLL_N).c +CCOMMTCOPY = gemm_tcopy_complex_sve_v1x$(ZGEMM_UNROLL_N).c CHEMMLTCOPY_M = zhemm_ltcopy_sve.c CHEMMUTCOPY_M = zhemm_utcopy_sve.c @@ -210,6 +225,7 @@ CSYMMUCOPY_M = zsymm_ucopy_sve.c CSYMMLCOPY_M = zsymm_lcopy_sve.c ZGEMMKERNEL = zgemm_kernel_sve_v1x$(ZGEMM_UNROLL_N).S +ZCOMMKERNEL = zgemm_kernel_sve_v1x$(ZGEMM_UNROLL_N).S ZTRMMKERNEL = ztrmm_kernel_sve_v1x$(ZGEMM_UNROLL_N).S ZGEMMINCOPY = gemm_ncopy_complex_sve_v1x$(ZGEMM_UNROLL_N).c @@ -221,11 +237,15 @@ ZGEMMINCOPYOBJ = zgemm_incopy$(TSUFFIX).$(SUFFIX) ZGEMMITCOPYOBJ = zgemm_itcopy$(TSUFFIX).$(SUFFIX) ZGEMMONCOPYOBJ = zgemm_oncopy$(TSUFFIX).$(SUFFIX) ZGEMMOTCOPYOBJ = zgemm_otcopy$(TSUFFIX).$(SUFFIX) +ZCOMMNCOPYOBJ = zcomm_ncopy$(TSUFFIX).$(SUFFIX) +ZCOMMTCOPYOBJ = zcomm_tcopy$(TSUFFIX).$(SUFFIX) ZTRMMUNCOPY_M = ztrmm_uncopy_sve_v1.c ZTRMMLNCOPY_M = ztrmm_lncopy_sve_v1.c ZTRMMUTCOPY_M = ztrmm_utcopy_sve_v1.c ZTRMMLTCOPY_M = ztrmm_ltcopy_sve_v1.c +ZCOMMNCOPY = gemm_ncopy_complex_sve_v1x$(ZGEMM_UNROLL_N).c +ZCOMMTCOPY = gemm_tcopy_complex_sve_v1x$(ZGEMM_UNROLL_N).c ZHEMMLTCOPY_M = zhemm_ltcopy_sve.c ZHEMMUTCOPY_M = zhemm_utcopy_sve.c diff --git a/kernel/arm64/dgemm_kernel_inter_sve_v2x8.S b/kernel/arm64/dgemm_kernel_inter_sve_v2x8.S new file mode 100644 index 0000000000..57532c3511 --- /dev/null +++ b/kernel/arm64/dgemm_kernel_inter_sve_v2x8.S @@ -0,0 +1,1520 @@ +/******************************************************************************* +Copyright (c) 2025, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +/* This is an SVE dgemm kernel with size 2*SVE_LEN x 8. +However, the data layout is the same as for the kernel 1*SVE_LEN x 8. +This means that we sweep two panels of packed A when iterating in a loop over K. +With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */ + +#define ASSEMBLER +#include "common.h" + +/* X0 X1 X2 s0 X3 x4 x5 x6 */ +/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc )*/ + +#define origM x0 +#define origN x1 +#define origK x2 +#define origPA x3 +#define origPB x4 +#define pC x5 +#define LDC x6 +#define temp x7 +#define counterL x8 +#define counterI x9 +#define counterJ x10 +#define pB x11 +#define pCRow0 x12 +#define pCRow1 x13 +#define pCRow2 x14 + +#define lanes x15 +#define pA1 x16 +#define pA2 x17 +#define alpha x18 +#define vec_len x19 +#define vec_lenx2 x20 + +#define alpha0 d10 +#define alphaZ z7.d + +#define A_PRE_SIZE 1536 +#define B_PRE_SIZE 512 +#define C_PRE_SIZE 128 + +// 00 origM +// 01 origN +// 02 origK +// 03 origPA +// 04 origPB +// 05 pC +// 06 origLDC -> LDC +// 07 temp +// 08 counterL +// 09 counterI +// 10 counterJ +// 11 pB +// 12 pCRow0 +// 13 pCRow1 +// 14 pCRow2 +// 15 lanes +// 16 pA1 +// 17 pA1 +// 18 must save alpha +// 19 must save vec_len +// 20 must save +// 21 must save +// 22 must save +// 23 must save +// 24 must save +// 25 must save +// 26 must save +// 27 must save +// 28 must save +// 29 frame +// 30 link +// 31 sp + +//v00 ALPHA -> pA10_0 +//v01 pA10_1 +//v02 pA20_0 +//v03 pA20_1 +//v04 +//v05 +//v06 +//v07 ALPHA0 +//v08 must save pB0_0 +//v09 must save pB0_1 +//v10 must save pB0_2 +//v11 must save pB0_3 +//v12 must save pB0_4 +//v13 must save pB0_5 +//v14 must save pB0_6 +//v15 must save pB0_7 +//v16 must save C0 +//v17 must save C1 +//v18 must save C2 +//v19 must save C3 +//v20 must save C4 +//v21 must save C5 +//v22 must save C6 +//v23 must save C7 +//v24 must save C8 +//v25 must save C9 +//v26 must save C10 +//v27 must save C11 +//v28 must save C12 +//v29 must save C13 +//v30 must save C14 +//v31 must save C15 + +/******************************************************************************* +* Macro definitions +*******************************************************************************/ + +.macro INITv2x8 + dup z16.d, #0 + dup z17.d, #0 + dup z18.d, #0 + dup z19.d, #0 + dup z20.d, #0 + dup z21.d, #0 + dup z22.d, #0 + dup z23.d, #0 + dup z24.d, #0 + dup z25.d, #0 + dup z26.d, #0 + dup z27.d, #0 + dup z28.d, #0 + dup z29.d, #0 + dup z30.d, #0 + dup z31.d, #0 +.endm + +.macro KERNELv2x8_I + ld1d z0.d, p0/z, [pA1] // start + ld1d z1.d, p0/z, [pA1, #1, MUL VL] // pA1 + vec + ld1d z2.d, p0/z, [pA1, #2, MUL VL] // pA1 + vec * 2 + ld1d z3.d, p0/z, [pA1, #3, MUL VL] // pA1 + vec * 3 + add pA1, pA1, vec_len, lsl #5 // pA1 = pA1 + vec_len * 8 * 2 * 2 + + + ld1rd z8.d, p0/z, [pB] + ld1rd z9.d, p0/z, [pB, 8] + ld1rd z10.d, p0/z, [pB, 16] + ld1rd z11.d, p0/z, [pB, 24] + ld1rd z12.d, p0/z, [pB, 32] + ld1rd z13.d, p0/z, [pB, 40] + ld1rd z14.d, p0/z, [pB, 48] + ld1rd z15.d, p0/z, [pB, 56] + + add pB, pB, 64 + + fmla z16.d, p0/m, z0.d, z8.d + fmla z17.d, p0/m, z1.d, z8.d + ld1rd z8.d, p0/z, [pB] + fmla z18.d, p0/m, z0.d, z9.d + fmla z19.d, p0/m, z1.d, z9.d + ld1rd z9.d, p0/z, [pB, 8] + fmla z20.d, p0/m, z0.d, z10.d + fmla z21.d, p0/m, z1.d, z10.d + ld1rd z10.d, p0/z, [pB, 16] + fmla z22.d, p0/m, z0.d, z11.d + fmla z23.d, p0/m, z1.d, z11.d + ld1rd z11.d, p0/z, [pB, 24] + fmla z24.d, p0/m, z0.d, z12.d + fmla z25.d, p0/m, z1.d, z12.d + ld1rd z12.d, p0/z, [pB, 32] + fmla z26.d, p0/m, z0.d, z13.d + fmla z27.d, p0/m, z1.d, z13.d + ld1rd z13.d, p0/z, [pB, 40] + fmla z28.d, p0/m, z0.d, z14.d + fmla z29.d, p0/m, z1.d, z14.d + ld1rd z14.d, p0/z, [pB, 48] + fmla z30.d, p0/m, z0.d, z15.d + fmla z31.d, p0/m, z1.d, z15.d + ld1rd z15.d, p0/z, [pB, 56] + + add pB, pB, 64 +.endm + +.macro KERNELv2x8_M1 + ld1d z2.d, p0/z, [pA1] + ld1d z3.d, p0/z, [pA1, vec_len, lsl #3] + add pA1, pA1, vec_len, lsl #4 // pA1 = pA1 + vec_len * 8 * 2 + + fmla z16.d, p0/m, z0.d, z8.d + fmla z17.d, p0/m, z1.d, z8.d + ld1rd z8.d, p0/z, [pB] + fmla z18.d, p0/m, z0.d, z9.d + fmla z19.d, p0/m, z1.d, z9.d + ld1rd z9.d, p0/z, [pB, 8] + fmla z20.d, p0/m, z0.d, z10.d + fmla z21.d, p0/m, z1.d, z10.d + ld1rd z10.d, p0/z, [pB, 16] + fmla z22.d, p0/m, z0.d, z11.d + fmla z23.d, p0/m, z1.d, z11.d + ld1rd z11.d, p0/z, [pB, 24] + fmla z24.d, p0/m, z0.d, z12.d + fmla z25.d, p0/m, z1.d, z12.d + ld1rd z12.d, p0/z, [pB, 32] + fmla z26.d, p0/m, z0.d, z13.d + fmla z27.d, p0/m, z1.d, z13.d + ld1rd z13.d, p0/z, [pB, 40] + fmla z28.d, p0/m, z0.d, z14.d + fmla z29.d, p0/m, z1.d, z14.d + ld1rd z14.d, p0/z, [pB, 48] + fmla z30.d, p0/m, z0.d, z15.d + fmla z31.d, p0/m, z1.d, z15.d + ld1rd z15.d, p0/z, [pB, 56] + + add pB, pB, 64 +.endm + +.macro KERNELv2x8_M2 + ld1d z0.d, p0/z, [pA1] + ld1d z1.d, p0/z, [pA1, vec_len, lsl #3] + add pA1, pA1, vec_len, lsl #4 // pA1 = pA1 + vec_len * 8 * 2 + + fmla z16.d, p0/m, z2.d, z8.d + fmla z17.d, p0/m, z3.d, z8.d + ld1rd z8.d, p0/z, [pB] + fmla z18.d, p0/m, z2.d, z9.d + fmla z19.d, p0/m, z3.d, z9.d + ld1rd z9.d, p0/z, [pB, 8] + fmla z20.d, p0/m, z2.d, z10.d + fmla z21.d, p0/m, z3.d, z10.d + ld1rd z10.d, p0/z, [pB, 16] + fmla z22.d, p0/m, z2.d, z11.d + fmla z23.d, p0/m, z3.d, z11.d + ld1rd z11.d, p0/z, [pB, 24] + fmla z24.d, p0/m, z2.d, z12.d + fmla z25.d, p0/m, z3.d, z12.d + ld1rd z12.d, p0/z, [pB, 32] + fmla z26.d, p0/m, z2.d, z13.d + fmla z27.d, p0/m, z3.d, z13.d + ld1rd z13.d, p0/z, [pB, 40] + fmla z28.d, p0/m, z2.d, z14.d + fmla z29.d, p0/m, z3.d, z14.d + ld1rd z14.d, p0/z, [pB, 48] + fmla z30.d, p0/m, z2.d, z15.d + fmla z31.d, p0/m, z3.d, z15.d + ld1rd z15.d, p0/z, [pB, 56] + + add pB, pB, 64 +.endm + +.macro KERNELv2x8_E + fmla z16.d, p0/m, z2.d, z8.d + fmla z17.d, p0/m, z3.d, z8.d + fmla z18.d, p0/m, z2.d, z9.d + fmla z19.d, p0/m, z3.d, z9.d + fmla z20.d, p0/m, z2.d, z10.d + fmla z21.d, p0/m, z3.d, z10.d + fmla z22.d, p0/m, z2.d, z11.d + fmla z23.d, p0/m, z3.d, z11.d + fmla z24.d, p0/m, z2.d, z12.d + fmla z25.d, p0/m, z3.d, z12.d + fmla z26.d, p0/m, z2.d, z13.d + fmla z27.d, p0/m, z3.d, z13.d + fmla z28.d, p0/m, z2.d, z14.d + fmla z29.d, p0/m, z3.d, z14.d + fmla z30.d, p0/m, z2.d, z15.d + fmla z31.d, p0/m, z3.d, z15.d +.endm + +.macro KERNELv2x8_SUB + ld1d z0.d, p0/z, [pA1] + ld1d z1.d, p0/z, [pA1, vec_len, lsl #3] + add pA1, pA1, vec_len, lsl #4 // pA1 = pA1 + vec_len * 8 * 2 + + ld1rd z8.d, p0/z, [pB] + ld1rd z9.d, p0/z, [pB, 8] + ld1rd z10.d, p0/z, [pB, 16] + ld1rd z11.d, p0/z, [pB, 24] + ld1rd z12.d, p0/z, [pB, 32] + ld1rd z13.d, p0/z, [pB, 40] + ld1rd z14.d, p0/z, [pB, 48] + ld1rd z15.d, p0/z, [pB, 56] + + add pB, pB, 64 + + fmla z16.d, p0/m, z0.d, z8.d + fmla z17.d, p0/m, z1.d, z8.d + fmla z18.d, p0/m, z0.d, z9.d + fmla z19.d, p0/m, z1.d, z9.d + fmla z20.d, p0/m, z0.d, z10.d + fmla z21.d, p0/m, z1.d, z10.d + fmla z22.d, p0/m, z0.d, z11.d + fmla z23.d, p0/m, z1.d, z11.d + fmla z24.d, p0/m, z0.d, z12.d + fmla z25.d, p0/m, z1.d, z12.d + fmla z26.d, p0/m, z0.d, z13.d + fmla z27.d, p0/m, z1.d, z13.d + fmla z28.d, p0/m, z0.d, z14.d + fmla z29.d, p0/m, z1.d, z14.d + fmla z30.d, p0/m, z0.d, z15.d + fmla z31.d, p0/m, z1.d, z15.d +.endm + +.macro SAVEv2x8 + add pCRow1, pCRow0, LDC + ld1d z8.d, p0/z, [pCRow0] + ld1d z9.d, p0/z, [pCRow0, #1, mul vl] + fmla z8.d, p0/m, z16.d, alphaZ + fmla z9.d, p0/m, z17.d, alphaZ + st1d z8.d, p0, [pCRow0] + st1d z9.d, p0, [pCRow0, #1, mul vl] + + add pCRow2, pCRow1, LDC + ld1d z10.d, p0/z, [pCRow1] + ld1d z11.d, p0/z, [pCRow1, #1, mul vl] + fmla z10.d, p0/m, z18.d, alphaZ + fmla z11.d, p0/m, z19.d, alphaZ + st1d z10.d, p0, [pCRow1] + st1d z11.d, p0, [pCRow1, #1, mul vl] + + add pCRow1, pCRow2, LDC + ld1d z12.d, p0/z, [pCRow2] + ld1d z13.d, p0/z, [pCRow2, #1, mul vl] + fmla z12.d, p0/m, z20.d, alphaZ + fmla z13.d, p0/m, z21.d, alphaZ + st1d z12.d, p0, [pCRow2] + st1d z13.d, p0, [pCRow2, #1, mul vl] + + add pCRow2, pCRow1, LDC + ld1d z14.d, p0/z, [pCRow1] + ld1d z15.d, p0/z, [pCRow1, #1, mul vl] + fmla z14.d, p0/m, z22.d, alphaZ + fmla z15.d, p0/m, z23.d, alphaZ + st1d z14.d, p0, [pCRow1] + st1d z15.d, p0, [pCRow1, #1, mul vl] + + add pCRow1, pCRow2, LDC + ld1d z8.d, p0/z, [pCRow2] + ld1d z9.d, p0/z, [pCRow2, #1, mul vl] + fmla z8.d, p0/m, z24.d, alphaZ + fmla z9.d, p0/m, z25.d, alphaZ + st1d z8.d, p0, [pCRow2] + st1d z9.d, p0, [pCRow2, #1, mul vl] + + add pCRow2, pCRow1, LDC + ld1d z10.d, p0/z, [pCRow1] + ld1d z11.d, p0/z, [pCRow1, #1, mul vl] + fmla z10.d, p0/m, z26.d, alphaZ + fmla z11.d, p0/m, z27.d, alphaZ + st1d z10.d, p0, [pCRow1] + st1d z11.d, p0, [pCRow1, #1, mul vl] + + add pCRow1, pCRow2, LDC + ld1d z12.d, p0/z, [pCRow2] + ld1d z13.d, p0/z, [pCRow2, #1, mul vl] + fmla z12.d, p0/m, z28.d, alphaZ + fmla z13.d, p0/m, z29.d, alphaZ + st1d z12.d, p0, [pCRow2] + st1d z13.d, p0, [pCRow2, #1, mul vl] + + ld1d z14.d, p0/z, [pCRow1] + ld1d z15.d, p0/z, [pCRow1, #1, mul vl] + fmla z14.d, p0/m, z30.d, alphaZ + fmla z15.d, p0/m, z31.d, alphaZ + st1d z14.d, p0, [pCRow1] + st1d z15.d, p0, [pCRow1, #1, mul vl] + + add pCRow0, pCRow0, vec_len, lsl #4 // pC = pC + vec_len * 8 * 2 + +.endm + +.macro INITv2x4 + dup z16.d, #0 + dup z17.d, #0 + dup z18.d, #0 + dup z19.d, #0 + dup z20.d, #0 + dup z21.d, #0 + dup z22.d, #0 + dup z23.d, #0 +.endm + +.macro KERNELv2x4_SUB + ld1d z0.d, p0/z, [pA1] + ld1d z1.d, p0/z, [pA1, vec_len, lsl #3] + add pA1, pA1, vec_len, lsl #4 // pA1 = pA1 + vec_len * 8 * 2 + + ld1rd z8.d, p0/z, [pB] + ld1rd z9.d, p0/z, [pB, 8] + ld1rd z10.d, p0/z, [pB, 16] + ld1rd z11.d, p0/z, [pB, 24] + + add pB, pB, 32 + + fmla z16.d, p0/m, z0.d, z8.d + fmla z17.d, p0/m, z1.d, z8.d + fmla z18.d, p0/m, z0.d, z9.d + fmla z19.d, p0/m, z1.d, z9.d + fmla z20.d, p0/m, z0.d, z10.d + fmla z21.d, p0/m, z1.d, z10.d + fmla z22.d, p0/m, z0.d, z11.d + fmla z23.d, p0/m, z1.d, z11.d +.endm + +.macro SAVEv2x4 + add pCRow1, pCRow0, LDC + ld1d z8.d, p0/z, [pCRow0] + ld1d z9.d, p0/z, [pCRow0, #1, mul vl] + fmla z8.d, p0/m, z16.d, alphaZ + fmla z9.d, p0/m, z17.d, alphaZ + st1d z8.d, p0, [pCRow0] + st1d z9.d, p0, [pCRow0, #1, mul vl] + + add pCRow2, pCRow1, LDC + ld1d z10.d, p0/z, [pCRow1] + ld1d z11.d, p0/z, [pCRow1, #1, mul vl] + fmla z10.d, p0/m, z18.d, alphaZ + fmla z11.d, p0/m, z19.d, alphaZ + st1d z10.d, p0, [pCRow1] + st1d z11.d, p0, [pCRow1, #1, mul vl] + + add pCRow1, pCRow2, LDC + ld1d z12.d, p0/z, [pCRow2] + ld1d z13.d, p0/z, [pCRow2, #1, mul vl] + fmla z12.d, p0/m, z20.d, alphaZ + fmla z13.d, p0/m, z21.d, alphaZ + st1d z12.d, p0, [pCRow2] + st1d z13.d, p0, [pCRow2, #1, mul vl] + + ld1d z14.d, p0/z, [pCRow1] + ld1d z15.d, p0/z, [pCRow1, #1, mul vl] + fmla z14.d, p0/m, z22.d, alphaZ + fmla z15.d, p0/m, z23.d, alphaZ + st1d z14.d, p0, [pCRow1] + st1d z15.d, p0, [pCRow1, #1, mul vl] + + add pCRow0, pCRow0, vec_len, lsl #4 // pC = pC + vec_len * 8 * 2 + +.endm + +.macro INITv2x2 + dup z16.d, #0 + dup z17.d, #0 + dup z18.d, #0 + dup z19.d, #0 +.endm + +.macro KERNELv2x2_SUB + ld1d z0.d, p0/z, [pA1] + ld1d z1.d, p0/z, [pA1, vec_len, lsl #3] + add pA1, pA1, vec_len, lsl #4 // pA1 = pA1 + vec_len * 8 * 2 + + ld1rd z8.d, p0/z, [pB] + ld1rd z9.d, p0/z, [pB, 8] + + add pB, pB, 16 + + fmla z16.d, p0/m, z0.d, z8.d + fmla z17.d, p0/m, z1.d, z8.d + fmla z18.d, p0/m, z0.d, z9.d + fmla z19.d, p0/m, z1.d, z9.d +.endm + +.macro SAVEv2x2 + add pCRow1, pCRow0, LDC + ld1d z8.d, p0/z, [pCRow0] + ld1d z9.d, p0/z, [pCRow0, #1, mul vl] + fmla z8.d, p0/m, z16.d, alphaZ + fmla z9.d, p0/m, z17.d, alphaZ + st1d z8.d, p0, [pCRow0] + st1d z9.d, p0, [pCRow0, #1, mul vl] + + ld1d z10.d, p0/z, [pCRow1] + ld1d z11.d, p0/z, [pCRow1, #1, mul vl] + fmla z10.d, p0/m, z18.d, alphaZ + fmla z11.d, p0/m, z19.d, alphaZ + st1d z10.d, p0, [pCRow1] + st1d z11.d, p0, [pCRow1, #1, mul vl] + + add pCRow0, pCRow0, vec_len, lsl #4 // pC = pC + vec_len * 8 * 2 +.endm + +.macro INITv2x1 + dup z16.d, #0 + dup z17.d, #0 +.endm + +.macro KERNELv2x1_SUB + ld1d z0.d, p0/z, [pA1] + ld1d z1.d, p0/z, [pA1, vec_len, lsl #3] + add pA1, pA1, vec_len, lsl #4 // pA1 = pA1 + vec_len * 8 * 2 + + ld1rd z8.d, p0/z, [pB] + + add pB, pB, 8 + + fmla z16.d, p0/m, z0.d, z8.d + fmla z17.d, p0/m, z1.d, z8.d +.endm + +.macro SAVEv2x1 + add pCRow1, pCRow0, LDC + ld1d z8.d, p0/z, [pCRow0] + ld1d z9.d, p0/z, [pCRow0, #1, mul vl] + fmla z8.d, p0/m, z16.d, alphaZ + fmla z9.d, p0/m, z17.d, alphaZ + st1d z8.d, p0, [pCRow0] + st1d z9.d, p0, [pCRow0, #1, mul vl] + + add pCRow0, pCRow0, vec_len, lsl #4 // pC = pC + vec_len * 8 * 2 + +.endm + +.macro INITv1x8 + dup z16.d, #0 + dup z17.d, #0 + dup z18.d, #0 + dup z19.d, #0 + dup z20.d, #0 + dup z21.d, #0 + dup z22.d, #0 + dup z23.d, #0 +.endm + +.macro KERNELv1x8_I + ld1d z0.d, p1/z, [pA1] + ld1d z1.d, p1/z, [pA1, lanes, lsl #3] // next one + add pA1, pA1, lanes, lsl #4 // pA1 = pA1 + lanes * 2 * 8 + + ld1rd z8.d, p0/z, [pB] + ld1rd z9.d, p0/z, [pB, 8] + ld1rd z10.d, p0/z, [pB, 16] + ld1rd z11.d, p0/z, [pB, 24] + ld1rd z12.d, p0/z, [pB, 32] + ld1rd z13.d, p0/z, [pB, 40] + ld1rd z14.d, p0/z, [pB, 48] + ld1rd z15.d, p0/z, [pB, 56] + + add pB, pB, 64 + + fmla z16.d, p1/m, z0.d, z8.d + ld1rd z8.d, p0/z, [pB] + fmla z17.d, p1/m, z0.d, z9.d + ld1rd z9.d, p0/z, [pB, 8] + fmla z18.d, p1/m, z0.d, z10.d + ld1rd z10.d, p0/z, [pB, 16] + fmla z19.d, p1/m, z0.d, z11.d + ld1rd z11.d, p0/z, [pB, 24] + fmla z20.d, p1/m, z0.d, z12.d + ld1rd z12.d, p0/z, [pB, 32] + fmla z21.d, p1/m, z0.d, z13.d + ld1rd z13.d, p0/z, [pB, 40] + fmla z22.d, p1/m, z0.d, z14.d + ld1rd z14.d, p0/z, [pB, 48] + fmla z23.d, p1/m, z0.d, z15.d + ld1rd z15.d, p0/z, [pB, 56] + + add pB, pB, 64 +.endm + +.macro KERNELv1x8_M1 + ld1d z1.d, p1/z, [pA1] + add pA1, pA1, lanes, lsl #3 // pA1 = pA1 + lanes * 8 + + fmla z16.d, p1/m, z0.d, z8.d + ld1rd z8.d, p0/z, [pB] + fmla z17.d, p1/m, z0.d, z9.d + ld1rd z9.d, p0/z, [pB, 8] + fmla z18.d, p1/m, z0.d, z10.d + ld1rd z10.d, p0/z, [pB, 16] + fmla z19.d, p1/m, z0.d, z11.d + ld1rd z11.d, p0/z, [pB, 24] + fmla z20.d, p1/m, z0.d, z12.d + ld1rd z12.d, p0/z, [pB, 32] + fmla z21.d, p1/m, z0.d, z13.d + ld1rd z13.d, p0/z, [pB, 40] + fmla z22.d, p1/m, z0.d, z14.d + ld1rd z14.d, p0/z, [pB, 48] + fmla z23.d, p1/m, z0.d, z15.d + ld1rd z15.d, p0/z, [pB, 56] + + add pB, pB, 64 +.endm + +.macro KERNELv1x8_M2 + ld1d z0.d, p1/z, [pA1] + add pA1, pA1, lanes, lsl #3 // pA1 = pA1 + lanes * 8 + + fmla z16.d, p1/m, z1.d, z8.d + ld1rd z8.d, p0/z, [pB] + fmla z17.d, p1/m, z1.d, z9.d + ld1rd z9.d, p0/z, [pB, 8] + fmla z18.d, p1/m, z1.d, z10.d + ld1rd z10.d, p0/z, [pB, 16] + fmla z19.d, p1/m, z1.d, z11.d + ld1rd z11.d, p0/z, [pB, 24] + fmla z20.d, p1/m, z1.d, z12.d + ld1rd z12.d, p0/z, [pB, 32] + fmla z21.d, p1/m, z1.d, z13.d + ld1rd z13.d, p0/z, [pB, 40] + fmla z22.d, p1/m, z1.d, z14.d + ld1rd z14.d, p0/z, [pB, 48] + fmla z23.d, p1/m, z1.d, z15.d + ld1rd z15.d, p0/z, [pB, 56] + + add pB, pB, 64 +.endm + +.macro KERNELv1x8_E + fmla z16.d, p1/m, z1.d, z8.d + fmla z17.d, p1/m, z1.d, z9.d + fmla z18.d, p1/m, z1.d, z10.d + fmla z19.d, p1/m, z1.d, z11.d + fmla z20.d, p1/m, z1.d, z12.d + fmla z21.d, p1/m, z1.d, z13.d + fmla z22.d, p1/m, z1.d, z14.d + fmla z23.d, p1/m, z1.d, z15.d +.endm + +.macro KERNELv1x8_SUB + ld1d z0.d, p1/z, [pA1] + add pA1, pA1, lanes, lsl #3 // pA1 = pA1 + lanes * 8 + + ld1rd z8.d, p0/z, [pB] + ld1rd z9.d, p0/z, [pB, 8] + ld1rd z10.d, p0/z, [pB, 16] + ld1rd z11.d, p0/z, [pB, 24] + ld1rd z12.d, p0/z, [pB, 32] + ld1rd z13.d, p0/z, [pB, 40] + ld1rd z14.d, p0/z, [pB, 48] + ld1rd z15.d, p0/z, [pB, 56] + + add pB, pB, 64 + + fmla z16.d, p1/m, z0.d, z8.d + fmla z17.d, p1/m, z0.d, z9.d + fmla z18.d, p1/m, z0.d, z10.d + fmla z19.d, p1/m, z0.d, z11.d + fmla z20.d, p1/m, z0.d, z12.d + fmla z21.d, p1/m, z0.d, z13.d + fmla z22.d, p1/m, z0.d, z14.d + fmla z23.d, p1/m, z0.d, z15.d + + +.endm + +.macro SAVEv1x8 + add pCRow1, pCRow0, LDC + ld1d z24.d, p1/z, [pCRow0] + fmla z24.d, p1/m, z16.d, alphaZ + st1d z24.d, p1, [pCRow0] + + add pCRow2, pCRow1, LDC + ld1d z25.d, p1/z, [pCRow1] + fmla z25.d, p1/m, z17.d, alphaZ + st1d z25.d, p1, [pCRow1] + + add pCRow1, pCRow2, LDC + ld1d z26.d, p1/z, [pCRow2] + fmla z26.d, p1/m, z18.d, alphaZ + st1d z26.d, p1, [pCRow2] + + add pCRow2, pCRow1, LDC + ld1d z27.d, p1/z, [pCRow1] + fmla z27.d, p1/m, z19.d, alphaZ + st1d z27.d, p1, [pCRow1] + + add pCRow1, pCRow2, LDC + ld1d z28.d, p1/z, [pCRow2] + fmla z28.d, p1/m, z20.d, alphaZ + st1d z28.d, p1, [pCRow2] + + add pCRow2, pCRow1, LDC + ld1d z29.d, p1/z, [pCRow1] + fmla z29.d, p1/m, z21.d, alphaZ + st1d z29.d, p1, [pCRow1] + + add pCRow1, pCRow2, LDC + ld1d z30.d, p1/z, [pCRow2] + fmla z30.d, p1/m, z22.d, alphaZ + st1d z30.d, p1, [pCRow2] + + ld1d z31.d, p1/z, [pCRow1] + fmla z31.d, p1/m, z23.d, alphaZ + st1d z31.d, p1, [pCRow1] + + add pCRow0, pCRow0, lanes, lsl #3 // pC = pC + lanes * 8 + +.endm + +/******************************************************************************/ + +.macro INITv1x4 + dup z16.d, #0 + dup z17.d, #0 + dup z18.d, #0 + dup z19.d, #0 +.endm + +.macro KERNELv1x4_SUB + ld1d z0.d, p1/z, [pA1] + add pA1, pA1, lanes, lsl #3 // pA1 = pA1 + lanes * 8 + + ld1rd z8.d, p0/z, [pB] + ld1rd z9.d, p0/z, [pB, 8] + ld1rd z10.d, p0/z, [pB, 16] + ld1rd z11.d, p0/z, [pB, 24] + + add pB, pB, 32 + + fmla z16.d, p1/m, z0.d, z8.d + fmla z17.d, p1/m, z0.d, z9.d + fmla z18.d, p1/m, z0.d, z10.d + fmla z19.d, p1/m, z0.d, z11.d + +.endm + +.macro SAVEv1x4 + add pCRow1, pCRow0, LDC + ld1d z24.d, p1/z, [pCRow0] + fmla z24.d, p1/m, z16.d, alphaZ + st1d z24.d, p1, [pCRow0] + + add pCRow2, pCRow1, LDC + ld1d z25.d, p1/z, [pCRow1] + fmla z25.d, p1/m, z17.d, alphaZ + st1d z25.d, p1, [pCRow1] + + add pCRow1, pCRow2, LDC + ld1d z26.d, p1/z, [pCRow2] + fmla z26.d, p1/m, z18.d, alphaZ + st1d z26.d, p1, [pCRow2] + + ld1d z27.d, p1/z, [pCRow1] + fmla z27.d, p1/m, z19.d, alphaZ + st1d z27.d, p1, [pCRow1] + + add pCRow0, pCRow0, lanes, lsl #3 // pC = pC + lanes * 8 + +.endm + +/******************************************************************************/ + +.macro INITv1x2 + dup z16.d, #0 + dup z17.d, #0 +.endm + +.macro KERNELv1x2_SUB + ld1d z0.d, p1/z, [pA1] + add pA1, pA1, lanes, lsl #3 // pA1 = pA1 + lanes * 8 + + ld1rd z8.d, p0/z, [pB] + ld1rd z9.d, p0/z, [pB, 8] + + add pB, pB, 16 + + fmla z16.d, p1/m, z0.d, z8.d + fmla z17.d, p1/m, z0.d, z9.d + +.endm + +.macro SAVEv1x2 + add pCRow1, pCRow0, LDC + ld1d z24.d, p1/z, [pCRow0] + fmla z24.d, p1/m, z16.d, alphaZ + st1d z24.d, p1, [pCRow0] + + ld1d z25.d, p1/z, [pCRow1] + fmla z25.d, p1/m, z17.d, alphaZ + st1d z25.d, p1, [pCRow1] + + add pCRow0, pCRow0, lanes, lsl #3 // pC = pC + lanes * 8 + +.endm + +/******************************************************************************/ + +.macro INITv1x1 + dup z16.d, #0 +.endm + +.macro KERNELv1x1_SUB + ld1d z0.d, p1/z, [pA1] + add pA1, pA1, lanes, lsl #3 // pA1 = pA1 + lanes * 8 + + ld1rd z8.d, p0/z, [pB] + + add pB, pB, 8 + + fmla z16.d, p1/m, z0.d, z8.d + +.endm + +.macro SAVEv1x1 + ld1d z24.d, p1/z, [pCRow0] + fmla z24.d, p1/m, z16.d, alphaZ + st1d z24.d, p1, [pCRow0] + + add pCRow0, pCRow0, lanes, lsl #3 // pC = pC + lanes * 8 + +.endm + + +/******************************************************************************* +* End of macro definitions +*******************************************************************************/ + + PROLOGUE + + .align 5 + add sp, sp, #-(11 * 16) + stp d8, d9, [sp, #(0 * 16)] + stp d10, d11, [sp, #(1 * 16)] + stp d12, d13, [sp, #(2 * 16)] + stp d14, d15, [sp, #(3 * 16)] + stp d16, d17, [sp, #(4 * 16)] + stp x18, x19, [sp, #(5 * 16)] + stp x20, x21, [sp, #(6 * 16)] + stp x22, x23, [sp, #(7 * 16)] + stp x24, x25, [sp, #(8 * 16)] + stp x26, x27, [sp, #(9 * 16)] + str x28, [sp, #(10 * 16)] + + + fmov alpha, d0 + dup alphaZ, alpha + cntd vec_len + lsl vec_lenx2, vec_len, #1 + + lsl LDC, LDC, #3 // ldc = ldc * 8 + ptrue p0.d // create true predicate + + mov pB, origPB +// Loop over N + mov counterJ, origN + asr counterJ, counterJ, #3 // J = J / 8 + cmp counterJ, #0 + ble .Ldgemm_kernel_L4_BEGIN + +/******************************************************************************/ +/* Repeat this as long as there are 8 left in N */ + + .align 5 +.Ldgemm_kernel_L8_BEGIN: + mov pCRow0, pC + + add pC, pC, LDC, lsl #3 // add 8 x LDC + + mov pA1, origPA // pA1 = start of A array + +.Ldgemm_kernel_L8_Mv2_BEGIN: + + mov counterI, #0 + cmp origM, vec_lenx2 // Check if M < 2*SVE_LEN + blt .Ldgemm_kernel_L8_Mv1_BEGIN + + mov counterI, origM + +/* Until we have at least 2*SVE_LEN iters left in M, we do them with V2*8 kernel */ + + .align 5 +.Ldgemm_kernel_L8_Mv2_20: + + mov pB, origPB + INITv2x8 // fill with zeros + + asr counterL , origK, #3 // L = K / 8 + cmp counterL , #2 // is there at least 4 to do? + blt .Ldgemm_kernel_L8_Mv2_32 + + KERNELv2x8_I + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_M2 + + subs counterL, counterL, #2 // subtract 2 + ble .Ldgemm_kernel_L8_Mv2_22a + + .align 5 +.Ldgemm_kernel_L8_Mv2_22: + + KERNELv2x8_M1 + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_M2 + + subs counterL, counterL, #1 + bgt .Ldgemm_kernel_L8_Mv2_22 + + .align 5 +.Ldgemm_kernel_L8_Mv2_22a: + + KERNELv2x8_M1 + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_E + + b .Ldgemm_kernel_L8_Mv2_44 + + .align 5 +.Ldgemm_kernel_L8_Mv2_32: + + tst counterL, #1 + ble .Ldgemm_kernel_L8_Mv2_40 + + KERNELv2x8_I + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_E + + + b .Ldgemm_kernel_L8_Mv2_44 + +.Ldgemm_kernel_L8_Mv2_40: + + INITv2x8 + +.Ldgemm_kernel_L8_Mv2_44: + + ands counterL , origK, #7 + ble .Ldgemm_kernel_L8_Mv2_100 + + .align 5 +.Ldgemm_kernel_L8_Mv2_46: + + KERNELv2x8_SUB + + subs counterL, counterL, #1 + bne .Ldgemm_kernel_L8_Mv2_46 + +.Ldgemm_kernel_L8_Mv2_100: + + SAVEv2x8 + +.Ldgemm_kernel_L8_Mv2_END: + sub counterI, counterI, vec_lenx2 + cmp counterI, vec_lenx2 + bge .Ldgemm_kernel_L8_Mv2_20 + sub counterI, origM, counterI + + cmp counterI, origM + beq .Ldgemm_kernel_L8_END + +////////////////////////////////////////// +// We have less than 2*SVE_LEN left. We do this with V1x8 kernel. +.Ldgemm_kernel_L8_Mv1_BEGIN: + + whilelt p1.d, counterI, origM //SVE instruction + cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension + + .align 5 +.Ldgemm_kernel_L8_Mv1_20: + + mov pB, origPB + INITv1x8 // fill with zeros + + asr counterL , origK, #3 // L = K / 8 + cmp counterL , #2 // is there at least 4 to do? + blt .Ldgemm_kernel_L8_Mv1_32 + + KERNELv1x8_I + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_M2 + + subs counterL, counterL, #2 // subtract 2 + ble .Ldgemm_kernel_L8_Mv1_22a + + .align 5 +.Ldgemm_kernel_L8_Mv1_22: + + KERNELv1x8_M1 + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_M2 + + subs counterL, counterL, #1 + bgt .Ldgemm_kernel_L8_Mv1_22 + + .align 5 +.Ldgemm_kernel_L8_Mv1_22a: + + KERNELv1x8_M1 + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_E + + b .Ldgemm_kernel_L8_Mv1_44 + + .align 5 +.Ldgemm_kernel_L8_Mv1_32: + + tst counterL, #1 + ble .Ldgemm_kernel_L8_Mv1_40 + + KERNELv1x8_I + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_E + + + b .Ldgemm_kernel_L8_Mv1_44 + +.Ldgemm_kernel_L8_Mv1_40: + + INITv1x8 + +.Ldgemm_kernel_L8_Mv1_44: + + ands counterL , origK, #7 + ble .Ldgemm_kernel_L8_Mv1_100 + + .align 5 +.Ldgemm_kernel_L8_Mv1_46: + + KERNELv1x8_SUB + + subs counterL, counterL, #1 + bne .Ldgemm_kernel_L8_Mv1_46 + +.Ldgemm_kernel_L8_Mv1_100: + + SAVEv1x8 + +.Ldgemm_kernel_L8_Mv1_END: + + incd counterI + whilelt p1.d, counterI, origM //SVE instruction + cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension + b.any .Ldgemm_kernel_L8_Mv1_20 + +.Ldgemm_kernel_L8_END: + + lsl temp, origK, #6 + add origPB, origPB, temp // B = B + K * 8 * 8 + + subs counterJ, counterJ , #1 // j-- + bgt .Ldgemm_kernel_L8_BEGIN + +/******************************************************************************/ +/* Repeat the same thing if 4 left in N */ + + .align 5 +.Ldgemm_kernel_L4_BEGIN: + + mov counterJ , origN + tst counterJ , #4 + ble .Ldgemm_kernel_L2_BEGIN + + + mov pCRow0, pC + + add pC, pC, LDC, lsl #2 // add 4 x LDC + + mov pA1, origPA // pA1 = start of A array + +.Ldgemm_kernel_L4_Mv2_BEGIN: + + mov counterI, #0 + cmp origM, vec_lenx2 + blt .Ldgemm_kernel_L4_Mv1_BEGIN + + mov counterI, origM + + .align 5 +.Ldgemm_kernel_L4_Mv2_20: + + mov pB, origPB + INITv2x4 // fill with zeros + + asr counterL , origK, #3 // L = K / 8 + cmp counterL , #0 // is there at least 4 to do? + ble .Ldgemm_kernel_L4_Mv2_44 + + .align 5 +.Ldgemm_kernel_L4_Mv2_22: + + KERNELv2x4_SUB + KERNELv2x4_SUB + KERNELv2x4_SUB + KERNELv2x4_SUB + KERNELv2x4_SUB + KERNELv2x4_SUB + KERNELv2x4_SUB + KERNELv2x4_SUB + + subs counterL, counterL, #1 + bgt .Ldgemm_kernel_L4_Mv2_22 + +.Ldgemm_kernel_L4_Mv2_44: + + ands counterL , origK, #7 + ble .Ldgemm_kernel_L4_Mv2_100 + + .align 5 +.Ldgemm_kernel_L4_Mv2_46: + + KERNELv2x4_SUB + + subs counterL, counterL, #1 + bne .Ldgemm_kernel_L4_Mv2_46 + +.Ldgemm_kernel_L4_Mv2_100: + + SAVEv2x4 + +.Ldgemm_kernel_L4_Mv2_END: + sub counterI, counterI, vec_lenx2 + cmp counterI, vec_lenx2 + bge .Ldgemm_kernel_L4_Mv2_20 + sub counterI, origM, counterI + + cmp counterI, origM + beq .Ldgemm_kernel_L4_END + +////////////////////////////////// +// We have less than 2*SVE_LEN left. We do this with V1x4 kernel. +.Ldgemm_kernel_L4_Mv1_BEGIN: + + whilelt p1.d, counterI, origM //SVE instruction + cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension + + .align 5 +.Ldgemm_kernel_L4_Mv1_20: + + mov pB, origPB + INITv1x4 // fill with zeros + + asr counterL , origK, #3 // L = K / 8 + cmp counterL , #0 // is there at least 4 to do? + ble .Ldgemm_kernel_L4_Mv1_44 + + .align 5 +.Ldgemm_kernel_L4_Mv1_22: + + KERNELv1x4_SUB + KERNELv1x4_SUB + KERNELv1x4_SUB + KERNELv1x4_SUB + KERNELv1x4_SUB + KERNELv1x4_SUB + KERNELv1x4_SUB + KERNELv1x4_SUB + + subs counterL, counterL, #1 + bgt .Ldgemm_kernel_L4_Mv1_22 + +.Ldgemm_kernel_L4_Mv1_44: + + ands counterL , origK, #7 + ble .Ldgemm_kernel_L4_Mv1_100 + + .align 5 +.Ldgemm_kernel_L4_Mv1_46: + + KERNELv1x4_SUB + + subs counterL, counterL, #1 + bne .Ldgemm_kernel_L4_Mv1_46 + +.Ldgemm_kernel_L4_Mv1_100: + + SAVEv1x4 + +.Ldgemm_kernel_L4_Mv1_END: + + incd counterI + whilelt p1.d, counterI, origM //SVE instruction + cntp lanes, p0, p1.d + b.any .Ldgemm_kernel_L4_Mv1_20 + + +.Ldgemm_kernel_L4_END: + lsl temp, origK, #5 + add origPB, origPB, temp // B = B + K * 4 * 8 + +/******************************************************************************/ +/* Repeat the same thing if 2 left in N */ + + .align 5 +.Ldgemm_kernel_L2_BEGIN: + + mov counterJ , origN + tst counterJ , #2 + ble .Ldgemm_kernel_L1_BEGIN + + mov pCRow0, pC + + add pC, pC, LDC, lsl #1 // add 2 x LDC + + mov pA1, origPA // pA1 = start of A array + +.Ldgemm_kernel_L2_Mv2_BEGIN: + + mov counterI, #0 + cmp origM, vec_lenx2 + blt .Ldgemm_kernel_L2_Mv1_BEGIN + + mov counterI, origM + + .align 5 +.Ldgemm_kernel_L2_Mv2_20: + + mov pB, origPB + INITv2x2 // fill with zeros + + asr counterL , origK, #3 // L = K / 8 + cmp counterL , #0 // is there at least 4 to do? + ble .Ldgemm_kernel_L2_Mv2_44 + + .align 5 +.Ldgemm_kernel_L2_Mv2_22: + + KERNELv2x2_SUB + KERNELv2x2_SUB + KERNELv2x2_SUB + KERNELv2x2_SUB + KERNELv2x2_SUB + KERNELv2x2_SUB + KERNELv2x2_SUB + KERNELv2x2_SUB + + subs counterL, counterL, #1 + bgt .Ldgemm_kernel_L2_Mv2_22 + +.Ldgemm_kernel_L2_Mv2_44: + + ands counterL , origK, #7 + ble .Ldgemm_kernel_L2_Mv2_100 + + .align 5 +.Ldgemm_kernel_L2_Mv2_46: + + KERNELv2x2_SUB + + subs counterL, counterL, #1 + bne .Ldgemm_kernel_L2_Mv2_46 + +.Ldgemm_kernel_L2_Mv2_100: + + SAVEv2x2 + +.Ldgemm_kernel_L2_Mv2_END: + sub counterI, counterI, vec_lenx2 + cmp counterI, vec_lenx2 + bge .Ldgemm_kernel_L2_Mv2_20 + sub counterI, origM, counterI + + cmp counterI, origM + beq .Ldgemm_kernel_L2_END + + +////////////////////////////////// +// We have less than 2*SVE_LEN left. We do this with V1x2 kernel. +.Ldgemm_kernel_L2_Mv1_BEGIN: + + whilelt p1.d, counterI, origM //SVE instruction + cntp lanes, p0, p1.d + + .align 5 +.Ldgemm_kernel_L2_Mv1_20: + + mov pB, origPB + INITv1x2 // fill with zeros + + asr counterL , origK, #3 // L = K / 8 + cmp counterL , #0 // is there at least 4 to do? + ble .Ldgemm_kernel_L2_Mv1_44 + + .align 5 +.Ldgemm_kernel_L2_Mv1_22: + + KERNELv1x2_SUB + KERNELv1x2_SUB + KERNELv1x2_SUB + KERNELv1x2_SUB + KERNELv1x2_SUB + KERNELv1x2_SUB + KERNELv1x2_SUB + KERNELv1x2_SUB + + subs counterL, counterL, #1 + bgt .Ldgemm_kernel_L2_Mv1_22 + +.Ldgemm_kernel_L2_Mv1_44: + + ands counterL , origK, #7 + ble .Ldgemm_kernel_L2_Mv1_100 + + .align 5 +.Ldgemm_kernel_L2_Mv1_46: + + KERNELv1x2_SUB + + subs counterL, counterL, #1 + bne .Ldgemm_kernel_L2_Mv1_46 + +.Ldgemm_kernel_L2_Mv1_100: + + SAVEv1x2 + +.Ldgemm_kernel_L2_Mv1_END: + + incd counterI + whilelt p1.d, counterI, origM //SVE instruction + cntp lanes, p0, p1.d + b.any .Ldgemm_kernel_L2_Mv1_20 + + +.Ldgemm_kernel_L2_END: + add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8 + +/******************************************************************************/ +/* Repeat the same thing if 1 left in N */ + + .align 5 +.Ldgemm_kernel_L1_BEGIN: + + mov counterJ , origN + tst counterJ , #1 + ble .Ldgemm_kernel_L999 // done + + mov pCRow0, pC + + add pC, pC, LDC // add 1 x LDC + + mov pA1, origPA // pA1 = start of A array + +.Ldgemm_kernel_L1_Mv2_BEGIN: + + mov counterI, #0 + cmp origM, vec_lenx2 + blt .Ldgemm_kernel_L1_Mv1_BEGIN + + mov counterI, origM + + .align 5 +.Ldgemm_kernel_L1_Mv2_20: + + mov pB, origPB + INITv2x1 // fill with zeros + + asr counterL , origK, #3 // L = K / 8 + cmp counterL , #0 // is there at least 8 to do? + ble .Ldgemm_kernel_L1_Mv2_44 + + .align 5 +.Ldgemm_kernel_L1_Mv2_22: + + KERNELv2x1_SUB + KERNELv2x1_SUB + KERNELv2x1_SUB + KERNELv2x1_SUB + KERNELv2x1_SUB + KERNELv2x1_SUB + KERNELv2x1_SUB + KERNELv2x1_SUB + + subs counterL, counterL, #1 + bgt .Ldgemm_kernel_L1_Mv2_22 + +.Ldgemm_kernel_L1_Mv2_44: + + ands counterL , origK, #7 + ble .Ldgemm_kernel_L1_Mv2_100 + + .align 5 +.Ldgemm_kernel_L1_Mv2_46: + + KERNELv2x1_SUB + + subs counterL, counterL, #1 + bgt .Ldgemm_kernel_L1_Mv2_46 + +.Ldgemm_kernel_L1_Mv2_100: + + SAVEv2x1 + +.Ldgemm_kernel_L1_Mv2_END: + sub counterI, counterI, vec_lenx2 + cmp counterI, vec_lenx2 + bge .Ldgemm_kernel_L1_Mv2_20 + sub counterI, origM, counterI + + cmp counterI, origM + beq .Ldgemm_kernel_L1_END + + +////////////////////////////////// +// We have less than 2*SVE_LEN left. We do this with V1x1 kernel. +.Ldgemm_kernel_L1_Mv1_BEGIN: + + whilelt p1.d, counterI, origM //SVE instruction + cntp lanes, p0, p1.d + + .align 5 +.Ldgemm_kernel_L1_Mv1_20: + + mov pB, origPB + INITv1x1 // fill with zeros + + asr counterL , origK, #3 // L = K / 8 + cmp counterL , #0 // is there at least 8 to do? + ble .Ldgemm_kernel_L1_Mv1_44 + + .align 5 +.Ldgemm_kernel_L1_Mv1_22: + + KERNELv1x1_SUB + KERNELv1x1_SUB + KERNELv1x1_SUB + KERNELv1x1_SUB + KERNELv1x1_SUB + KERNELv1x1_SUB + KERNELv1x1_SUB + KERNELv1x1_SUB + + subs counterL, counterL, #1 + bgt .Ldgemm_kernel_L1_Mv1_22 + +.Ldgemm_kernel_L1_Mv1_44: + + ands counterL , origK, #7 + ble .Ldgemm_kernel_L1_Mv1_100 + + .align 5 +.Ldgemm_kernel_L1_Mv1_46: + + KERNELv1x1_SUB + + subs counterL, counterL, #1 + bgt .Ldgemm_kernel_L1_Mv1_46 + +.Ldgemm_kernel_L1_Mv1_100: + + SAVEv1x1 + +.Ldgemm_kernel_L1_Mv1_END: + + incd counterI + whilelt p1.d, counterI, origM //SVE instruction + cntp lanes, p0, p1.d + b.any .Ldgemm_kernel_L1_Mv1_20 + + +.Ldgemm_kernel_L1_END: + +/******************************************************************************/ + +.Ldgemm_kernel_L999: + mov x0, #0 // set return value + ldp d8, d9, [sp, #(0 * 16)] + ldp d10, d11, [sp, #(1 * 16)] + ldp d12, d13, [sp, #(2 * 16)] + ldp d14, d15, [sp, #(3 * 16)] + ldp d16, d17, [sp, #(4 * 16)] + ldp x18, x19, [sp, #(5 * 16)] + ldp x20, x21, [sp, #(6 * 16)] + ldp x22, x23, [sp, #(7 * 16)] + ldp x24, x25, [sp, #(8 * 16)] + ldp x26, x27, [sp, #(9 * 16)] + ldr x28, [sp, #(10 * 16)] + add sp, sp, #(11*16) + ret + + EPILOGUE + diff --git a/kernel/arm64/gemm_ncopy_sve_v2x8.c b/kernel/arm64/gemm_ncopy_sve_v2x8.c new file mode 100644 index 0000000000..c810b929d5 --- /dev/null +++ b/kernel/arm64/gemm_ncopy_sve_v2x8.c @@ -0,0 +1,179 @@ +/*************************************************************************** +Copyright (c) 2023, 2025, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A00 PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include +#include +#include + +#include "common.h" + +#ifdef DOUBLE +#define COUNT "cntd" +#define SV_TYPE svfloat64_t +#define SV_INDEX svuint64_t +#define SV_INDEXER svindex_u64 +#define SV_TRUE svptrue_b64 +#define SV_WHILE svwhilelt_b64 +#define SV_PREFETCH svprfd_gather_index +#else +#define COUNT "cntw" +#define SV_TYPE svfloat32_t +#define SV_INDEX svuint32_t +#define SV_INDEXER svindex_u32 +#define SV_TRUE svptrue_b32 +#define SV_WHILE svwhilelt_b32 +#define SV_PREFETCH svprfw_gather_index +#endif + +#define INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active) \ + a_vec = svld1_gather_index(pg, a_offset_inner, lda_vec); \ + svst1(pg, b_offset, a_vec); \ + b_offset += active; \ + a_vec = svld1_gather_index(pg, a_offset_inner + sve_size * lda, lda_vec); \ + svst1(pg, b_offset, a_vec); \ + b_offset += active; \ + a_offset_inner++; + +#define INNER_COPY(pg, a_offset_inner, b_offset, lda, active) \ + a_vec = svld1_gather_index(pg, a_offset_inner, lda_vec); \ + svst1(pg, b_offset, a_vec); \ + a_offset_inner++; \ + b_offset += active; + +int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b) { + uint64_t sve_size; + asm(COUNT" %[SIZE_]" : [SIZE_] "=r" (sve_size) : : ); + + IFLOAT *a_offset, *a_offset_inner, *b_offset; + a_offset = a; + b_offset = b; + + SV_INDEX lda_vec = SV_INDEXER(0LL, lda); + SV_TYPE a_vec; + svbool_t pg_true = SV_TRUE(); + + uint64_t double_sve_size = sve_size * 2; + BLASLONG double_vectors_n = n & -double_sve_size; + for (BLASLONG j = 0; j < double_vectors_n; j += double_sve_size) { + a_offset_inner = a_offset; + + svbool_t pg = pg_true; + uint64_t active = sve_size; + uint64_t i_cnt = m >> 3; + while (i_cnt--) { + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + } + + if (m & 4) { + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + } + + if (m & 2) { + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + } + + if (m & 1) { + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + } + + // Next two rows + a_offset += double_sve_size * lda; + } + + BLASLONG single_vectors_n = (n - double_vectors_n) & -sve_size; + if (single_vectors_n) { + a_offset_inner = a_offset; + + svbool_t pg = pg_true; + uint64_t active = sve_size; + uint64_t i_cnt = m >> 3; + while (i_cnt--) { + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + } + + if (m & 4) { + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + } + + if (m & 2) { + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + } + + if (m & 1) { + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + } + + a_offset += sve_size * lda; + } + + BLASLONG remaining_n = n - double_vectors_n - single_vectors_n; + if (remaining_n) { + a_offset_inner = a_offset; + svbool_t pg = SV_WHILE((uint64_t)0L, (uint64_t)remaining_n); + uint64_t active = remaining_n; + uint64_t i_cnt = m >> 2; + while (i_cnt--) { + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + } + + if (m & 2) { + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + } + + if (m & 1) { + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + } + } + + return 0; +} diff --git a/kernel/arm64/gemm_tcopy_sve_v2x8.c b/kernel/arm64/gemm_tcopy_sve_v2x8.c new file mode 100644 index 0000000000..e4899fe576 --- /dev/null +++ b/kernel/arm64/gemm_tcopy_sve_v2x8.c @@ -0,0 +1,174 @@ +/*************************************************************************** +Copyright (c) 2023, 2025, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A00 PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include +#include +#include + +#include "common.h" + +#ifdef DOUBLE +#define COUNT "cntd" +#define SV_TYPE svfloat64_t +#define SV_TRUE svptrue_b64 +#define SV_WHILE svwhilelt_b64 +#else +#define COUNT "cntw" +#define SV_TYPE svfloat32_t +#define SV_TRUE svptrue_b32 +#define SV_WHILE svwhilelt_b32 +#endif + +#define INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active) \ + a_vec = svld1(pg, a_offset_inner); \ + svst1(pg, b_offset, a_vec); \ + b_offset += active; \ + a_vec = svld1(pg, a_offset_inner + sve_size); \ + svst1(pg, b_offset, a_vec); \ + a_offset_inner += lda; \ + b_offset += active; + +#define INNER_COPY(pg, a_offset_inner, b_offset, lda, active) \ + a_vec = svld1(pg, a_offset_inner); \ + svst1(pg, b_offset, a_vec); \ + a_offset_inner += lda; \ + b_offset += active; + +int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){ + uint64_t sve_size = svcntw(); + asm(COUNT" %[SIZE_]" : [SIZE_] "=r" (sve_size) : : ); + + IFLOAT *a_offset, *a_offset_inner, *b_offset; + a_offset = a; + b_offset = b; + + SV_TYPE a_vec; + svbool_t pg_true = SV_TRUE(); + + uint64_t double_sve_size = sve_size * 2; + BLASLONG double_vectors_n = n & -double_sve_size; + for (BLASLONG j = 0; j < double_vectors_n; j += double_sve_size) { + a_offset_inner = a_offset; + + svbool_t pg = pg_true; + uint64_t active = sve_size; + uint64_t i_cnt = m >> 3; + while (i_cnt--) { + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + } + + if (m & 4) { + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + } + + if (m & 2) { + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + } + + if (m & 1) { + INNER_INTERLEAVE_COPY(pg, a_offset_inner, b_offset, lda, active); + } + + // Next two rows + a_offset += double_sve_size; + } + + BLASLONG single_vectors_n = (n - double_vectors_n) & -sve_size; + if (single_vectors_n) { + + a_offset_inner = a_offset; + + svbool_t pg = pg_true; + uint64_t active = sve_size; + uint64_t i_cnt = m >> 3; + while (i_cnt--) { + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + } + + if (m & 4) { + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + } + + if (m & 2) { + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + } + + if (m & 1) { + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + } + + a_offset += sve_size; + } + + BLASLONG remaining_n = n - double_vectors_n - single_vectors_n; + if (remaining_n) { + a_offset_inner = a_offset; + svbool_t pg = SV_WHILE((uint64_t)0L, (uint64_t)remaining_n); + uint64_t active = remaining_n; + uint64_t i_cnt = m >> 2; + while (i_cnt--) { + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + } + + if (m & 2) { + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + } + + if (m & 1) { + INNER_COPY(pg, a_offset_inner, b_offset, lda, active); + } + } + + return 0; +} + diff --git a/kernel/arm64/sgemm_kernel_inter_sve_v2x8.S b/kernel/arm64/sgemm_kernel_inter_sve_v2x8.S new file mode 100644 index 0000000000..49dee97ef7 --- /dev/null +++ b/kernel/arm64/sgemm_kernel_inter_sve_v2x8.S @@ -0,0 +1,1518 @@ +/******************************************************************************* +Copyright (c) 2025, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +/* This is an SVE sgemm kernel with size 2*SVE_LEN x 8. +However, the data layout is the same as for the kernel 1*SVE_LEN x 8. +This means that we sweep two panels of packed A when iterating in a loop over K. +With this approach, we can reuse sgemm_n|tcopy_sve_v1.c packing functions. */ + +#define ASSEMBLER +#include "common.h" + +/* X0 X1 X2 s0 X3 x4 x5 x6 */ +/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc )*/ + +#define origM x0 +#define origN x1 +#define origK x2 +#define origPA x3 +#define origPB x4 +#define pC x5 +#define LDC x6 +#define temp x7 +#define counterL x8 +#define counterI x9 +#define counterJ x10 +#define pB x11 +#define pCRow0 x12 +#define pCRow1 x13 +#define pCRow2 x14 + +#define lanes x15 +#define pA1 x16 +#define pA2 x17 +#define alpha w18 +#define vec_len x19 +#define vec_lenx2 x20 + +#define alpha0 s10 +#define alphaZ z7.s + +#define A_PRE_SIZE 1536 +#define B_PRE_SIZE 512 +#define C_PRE_SIZE 128 + +// 00 origM +// 01 origN +// 02 origK +// 03 origPA +// 04 origPB +// 05 pC +// 06 origLDC -> LDC +// 07 temp +// 08 counterL +// 09 counterI +// 10 counterJ +// 11 pB +// 12 pCRow0 +// 13 pCRow1 +// 14 pCRow2 +// 15 lanes +// 16 pA1 +// 17 pA1 +// 18 must save alpha +// 19 must save vec_len +// 20 must save +// 21 must save +// 22 must save +// 23 must save +// 24 must save +// 25 must save +// 26 must save +// 27 must save +// 28 must save +// 29 frame +// 30 link +// 31 sp + +//v00 ALPHA -> pA10_0 +//v01 pA10_1 +//v02 pA20_0 +//v03 pA20_1 +//v04 +//v05 +//v06 +//v07 ALPHA0 +//v08 must save pB0_0 +//v09 must save pB0_1 +//v10 must save pB0_2 +//v11 must save pB0_3 +//v12 must save pB0_4 +//v13 must save pB0_5 +//v14 must save pB0_6 +//v15 must save pB0_7 +//v16 must save C0 +//v17 must save C1 +//v18 must save C2 +//v19 must save C3 +//v20 must save C4 +//v21 must save C5 +//v22 must save C6 +//v23 must save C7 +//v24 must save C8 +//v25 must save C9 +//v26 must save C10 +//v27 must save C11 +//v28 must save C12 +//v29 must save C13 +//v30 must save C14 +//v31 must save C15 + +/******************************************************************************* +* Macro definitions +*******************************************************************************/ + +.macro INITv2x8 + dup z16.s, #0 + dup z17.s, #0 + dup z18.s, #0 + dup z19.s, #0 + dup z20.s, #0 + dup z21.s, #0 + dup z22.s, #0 + dup z23.s, #0 + dup z24.s, #0 + dup z25.s, #0 + dup z26.s, #0 + dup z27.s, #0 + dup z28.s, #0 + dup z29.s, #0 + dup z30.s, #0 + dup z31.s, #0 +.endm + +.macro KERNELv2x8_I + ld1w z0.s, p0/z, [pA1] // start + ld1w z1.s, p0/z, [pA1, #1, MUL VL] // pA1 + vec + ld1w z2.s, p0/z, [pA1, #2, MUL VL] // pA1 + vec * 2 + ld1w z3.s, p0/z, [pA1, #3, MUL VL] // pA1 + vec * 3 + add pA1, pA1, vec_len, lsl #4 // pA1 = pA1 + vec_len * 4 * 2 * 2 + + ld1rw z8.s, p0/z, [pB] + ld1rw z9.s, p0/z, [pB, 4] + ld1rw z10.s, p0/z, [pB, 8] + ld1rw z11.s, p0/z, [pB, 12] + ld1rw z12.s, p0/z, [pB, 16] + ld1rw z13.s, p0/z, [pB, 20] + ld1rw z14.s, p0/z, [pB, 24] + ld1rw z15.s, p0/z, [pB, 28] + + add pB, pB, 32 + + fmla z16.s, p0/m, z0.s, z8.s + fmla z17.s, p0/m, z1.s, z8.s + ld1rw z8.s, p0/z, [pB] + fmla z18.s, p0/m, z0.s, z9.s + fmla z19.s, p0/m, z1.s, z9.s + ld1rw z9.s, p0/z, [pB, 4] + fmla z20.s, p0/m, z0.s, z10.s + fmla z21.s, p0/m, z1.s, z10.s + ld1rw z10.s, p0/z, [pB, 8] + fmla z22.s, p0/m, z0.s, z11.s + fmla z23.s, p0/m, z1.s, z11.s + ld1rw z11.s, p0/z, [pB, 12] + fmla z24.s, p0/m, z0.s, z12.s + fmla z25.s, p0/m, z1.s, z12.s + ld1rw z12.s, p0/z, [pB, 16] + fmla z26.s, p0/m, z0.s, z13.s + fmla z27.s, p0/m, z1.s, z13.s + ld1rw z13.s, p0/z, [pB, 20] + fmla z28.s, p0/m, z0.s, z14.s + fmla z29.s, p0/m, z1.s, z14.s + ld1rw z14.s, p0/z, [pB, 24] + fmla z30.s, p0/m, z0.s, z15.s + fmla z31.s, p0/m, z1.s, z15.s + ld1rw z15.s, p0/z, [pB, 28] + + add pB, pB, 32 +.endm + +.macro KERNELv2x8_M1 + ld1w z2.s, p0/z, [pA1] + ld1w z3.s, p0/z, [pA1, vec_len, lsl #2] + add pA1, pA1, vec_len, lsl #3 // pA1 = pA1 + vec_len * 4 * 2 + + fmla z16.s, p0/m, z0.s, z8.s + fmla z17.s, p0/m, z1.s, z8.s + ld1rw z8.s, p0/z, [pB] + fmla z18.s, p0/m, z0.s, z9.s + fmla z19.s, p0/m, z1.s, z9.s + ld1rw z9.s, p0/z, [pB, 4] + fmla z20.s, p0/m, z0.s, z10.s + fmla z21.s, p0/m, z1.s, z10.s + ld1rw z10.s, p0/z, [pB, 8] + fmla z22.s, p0/m, z0.s, z11.s + fmla z23.s, p0/m, z1.s, z11.s + ld1rw z11.s, p0/z, [pB, 12] + fmla z24.s, p0/m, z0.s, z12.s + fmla z25.s, p0/m, z1.s, z12.s + ld1rw z12.s, p0/z, [pB, 16] + fmla z26.s, p0/m, z0.s, z13.s + fmla z27.s, p0/m, z1.s, z13.s + ld1rw z13.s, p0/z, [pB, 20] + fmla z28.s, p0/m, z0.s, z14.s + fmla z29.s, p0/m, z1.s, z14.s + ld1rw z14.s, p0/z, [pB, 24] + fmla z30.s, p0/m, z0.s, z15.s + fmla z31.s, p0/m, z1.s, z15.s + ld1rw z15.s, p0/z, [pB, 28] + + add pB, pB, 32 +.endm + +.macro KERNELv2x8_M2 + ld1w z0.s, p0/z, [pA1] + ld1w z1.s, p0/z, [pA1, vec_len, lsl #2] + add pA1, pA1, vec_len, lsl #3 // pA1 = pA1 + vec_len * 4 * 2 + + fmla z16.s, p0/m, z2.s, z8.s + fmla z17.s, p0/m, z3.s, z8.s + ld1rw z8.s, p0/z, [pB] + fmla z18.s, p0/m, z2.s, z9.s + fmla z19.s, p0/m, z3.s, z9.s + ld1rw z9.s, p0/z, [pB, 4] + fmla z20.s, p0/m, z2.s, z10.s + fmla z21.s, p0/m, z3.s, z10.s + ld1rw z10.s, p0/z, [pB, 8] + fmla z22.s, p0/m, z2.s, z11.s + fmla z23.s, p0/m, z3.s, z11.s + ld1rw z11.s, p0/z, [pB, 12] + fmla z24.s, p0/m, z2.s, z12.s + fmla z25.s, p0/m, z3.s, z12.s + ld1rw z12.s, p0/z, [pB, 16] + fmla z26.s, p0/m, z2.s, z13.s + fmla z27.s, p0/m, z3.s, z13.s + ld1rw z13.s, p0/z, [pB, 20] + fmla z28.s, p0/m, z2.s, z14.s + fmla z29.s, p0/m, z3.s, z14.s + ld1rw z14.s, p0/z, [pB, 24] + fmla z30.s, p0/m, z2.s, z15.s + fmla z31.s, p0/m, z3.s, z15.s + ld1rw z15.s, p0/z, [pB, 28] + + add pB, pB, 32 +.endm + +.macro KERNELv2x8_E + fmla z16.s, p0/m, z2.s, z8.s + fmla z17.s, p0/m, z3.s, z8.s + fmla z18.s, p0/m, z2.s, z9.s + fmla z19.s, p0/m, z3.s, z9.s + fmla z20.s, p0/m, z2.s, z10.s + fmla z21.s, p0/m, z3.s, z10.s + fmla z22.s, p0/m, z2.s, z11.s + fmla z23.s, p0/m, z3.s, z11.s + fmla z24.s, p0/m, z2.s, z12.s + fmla z25.s, p0/m, z3.s, z12.s + fmla z26.s, p0/m, z2.s, z13.s + fmla z27.s, p0/m, z3.s, z13.s + fmla z28.s, p0/m, z2.s, z14.s + fmla z29.s, p0/m, z3.s, z14.s + fmla z30.s, p0/m, z2.s, z15.s + fmla z31.s, p0/m, z3.s, z15.s +.endm + +.macro KERNELv2x8_SUB + ld1w z0.s, p0/z, [pA1] + ld1w z1.s, p0/z, [pA1, vec_len, lsl #2] + add pA1, pA1, vec_len, lsl #3 // pA1 = pA1 + vec_len * 4 * 2 + + ld1rw z8.s, p0/z, [pB] + ld1rw z9.s, p0/z, [pB, 4] + ld1rw z10.s, p0/z, [pB, 8] + ld1rw z11.s, p0/z, [pB, 12] + ld1rw z12.s, p0/z, [pB, 16] + ld1rw z13.s, p0/z, [pB, 20] + ld1rw z14.s, p0/z, [pB, 24] + ld1rw z15.s, p0/z, [pB, 28] + + add pB, pB, 32 + + fmla z16.s, p0/m, z0.s, z8.s + fmla z17.s, p0/m, z1.s, z8.s + fmla z18.s, p0/m, z0.s, z9.s + fmla z19.s, p0/m, z1.s, z9.s + fmla z20.s, p0/m, z0.s, z10.s + fmla z21.s, p0/m, z1.s, z10.s + fmla z22.s, p0/m, z0.s, z11.s + fmla z23.s, p0/m, z1.s, z11.s + fmla z24.s, p0/m, z0.s, z12.s + fmla z25.s, p0/m, z1.s, z12.s + fmla z26.s, p0/m, z0.s, z13.s + fmla z27.s, p0/m, z1.s, z13.s + fmla z28.s, p0/m, z0.s, z14.s + fmla z29.s, p0/m, z1.s, z14.s + fmla z30.s, p0/m, z0.s, z15.s + fmla z31.s, p0/m, z1.s, z15.s +.endm + +.macro SAVEv2x8 + add pCRow1, pCRow0, LDC + ld1w z8.s, p0/z, [pCRow0] + ld1w z9.s, p0/z, [pCRow0, #1, mul vl] + fmla z8.s, p0/m, z16.s, alphaZ + fmla z9.s, p0/m, z17.s, alphaZ + st1w z8.s, p0, [pCRow0] + st1w z9.s, p0, [pCRow0, #1, mul vl] + + add pCRow2, pCRow1, LDC + ld1w z10.s, p0/z, [pCRow1] + ld1w z11.s, p0/z, [pCRow1, #1, mul vl] + fmla z10.s, p0/m, z18.s, alphaZ + fmla z11.s, p0/m, z19.s, alphaZ + st1w z10.s, p0, [pCRow1] + st1w z11.s, p0, [pCRow1, #1, mul vl] + + add pCRow1, pCRow2, LDC + ld1w z12.s, p0/z, [pCRow2] + ld1w z13.s, p0/z, [pCRow2, #1, mul vl] + fmla z12.s, p0/m, z20.s, alphaZ + fmla z13.s, p0/m, z21.s, alphaZ + st1w z12.s, p0, [pCRow2] + st1w z13.s, p0, [pCRow2, #1, mul vl] + + add pCRow2, pCRow1, LDC + ld1w z14.s, p0/z, [pCRow1] + ld1w z15.s, p0/z, [pCRow1, #1, mul vl] + fmla z14.s, p0/m, z22.s, alphaZ + fmla z15.s, p0/m, z23.s, alphaZ + st1w z14.s, p0, [pCRow1] + st1w z15.s, p0, [pCRow1, #1, mul vl] + + add pCRow1, pCRow2, LDC + ld1w z8.s, p0/z, [pCRow2] + ld1w z9.s, p0/z, [pCRow2, #1, mul vl] + fmla z8.s, p0/m, z24.s, alphaZ + fmla z9.s, p0/m, z25.s, alphaZ + st1w z8.s, p0, [pCRow2] + st1w z9.s, p0, [pCRow2, #1, mul vl] + + add pCRow2, pCRow1, LDC + ld1w z10.s, p0/z, [pCRow1] + ld1w z11.s, p0/z, [pCRow1, #1, mul vl] + fmla z10.s, p0/m, z26.s, alphaZ + fmla z11.s, p0/m, z27.s, alphaZ + st1w z10.s, p0, [pCRow1] + st1w z11.s, p0, [pCRow1, #1, mul vl] + + add pCRow1, pCRow2, LDC + ld1w z12.s, p0/z, [pCRow2] + ld1w z13.s, p0/z, [pCRow2, #1, mul vl] + fmla z12.s, p0/m, z28.s, alphaZ + fmla z13.s, p0/m, z29.s, alphaZ + st1w z12.s, p0, [pCRow2] + st1w z13.s, p0, [pCRow2, #1, mul vl] + + ld1w z14.s, p0/z, [pCRow1] + ld1w z15.s, p0/z, [pCRow1, #1, mul vl] + fmla z14.s, p0/m, z30.s, alphaZ + fmla z15.s, p0/m, z31.s, alphaZ + st1w z14.s, p0, [pCRow1] + st1w z15.s, p0, [pCRow1, #1, mul vl] + + add pCRow0, pCRow0, vec_len, lsl #3 // pC = pC + vec_len * 4 * 2 + +.endm + +.macro INITv2x4 + dup z16.s, #0 + dup z17.s, #0 + dup z18.s, #0 + dup z19.s, #0 + dup z20.s, #0 + dup z21.s, #0 + dup z22.s, #0 + dup z23.s, #0 +.endm + +.macro KERNELv2x4_SUB + ld1w z0.s, p0/z, [pA1] + ld1w z1.s, p0/z, [pA1, vec_len, lsl #2] + add pA1, pA1, vec_len, lsl #3 // pA1 = pA1 + vec_len * 4 * 2 + + ld1rw z8.s, p0/z, [pB] + ld1rw z9.s, p0/z, [pB, 4] + ld1rw z10.s, p0/z, [pB, 8] + ld1rw z11.s, p0/z, [pB, 12] + + add pB, pB, 16 + + fmla z16.s, p0/m, z0.s, z8.s + fmla z17.s, p0/m, z1.s, z8.s + fmla z18.s, p0/m, z0.s, z9.s + fmla z19.s, p0/m, z1.s, z9.s + fmla z20.s, p0/m, z0.s, z10.s + fmla z21.s, p0/m, z1.s, z10.s + fmla z22.s, p0/m, z0.s, z11.s + fmla z23.s, p0/m, z1.s, z11.s +.endm + +.macro SAVEv2x4 + add pCRow1, pCRow0, LDC + ld1w z8.s, p0/z, [pCRow0] + ld1w z9.s, p0/z, [pCRow0, #1, mul vl] + fmla z8.s, p0/m, z16.s, alphaZ + fmla z9.s, p0/m, z17.s, alphaZ + st1w z8.s, p0, [pCRow0] + st1w z9.s, p0, [pCRow0, #1, mul vl] + + add pCRow2, pCRow1, LDC + ld1w z10.s, p0/z, [pCRow1] + ld1w z11.s, p0/z, [pCRow1, #1, mul vl] + fmla z10.s, p0/m, z18.s, alphaZ + fmla z11.s, p0/m, z19.s, alphaZ + st1w z10.s, p0, [pCRow1] + st1w z11.s, p0, [pCRow1, #1, mul vl] + + add pCRow1, pCRow2, LDC + ld1w z12.s, p0/z, [pCRow2] + ld1w z13.s, p0/z, [pCRow2, #1, mul vl] + fmla z12.s, p0/m, z20.s, alphaZ + fmla z13.s, p0/m, z21.s, alphaZ + st1w z12.s, p0, [pCRow2] + st1w z13.s, p0, [pCRow2, #1, mul vl] + + ld1w z14.s, p0/z, [pCRow1] + ld1w z15.s, p0/z, [pCRow1, #1, mul vl] + fmla z14.s, p0/m, z22.s, alphaZ + fmla z15.s, p0/m, z23.s, alphaZ + st1w z14.s, p0, [pCRow1] + st1w z15.s, p0, [pCRow1, #1, mul vl] + + add pCRow0, pCRow0, vec_len, lsl #3 // pC = pC + vec_len * 4 * 2 + +.endm + +.macro INITv2x2 + dup z16.s, #0 + dup z17.s, #0 + dup z18.s, #0 + dup z19.s, #0 +.endm + +.macro KERNELv2x2_SUB + ld1w z0.s, p0/z, [pA1] + ld1w z1.s, p0/z, [pA1, vec_len, lsl #2] + add pA1, pA1, vec_len, lsl #3 // pA1 = pA1 + vec_len * 4 * 2 + + ld1rw z8.s, p0/z, [pB] + ld1rw z9.s, p0/z, [pB, 4] + + add pB, pB, 8 + + fmla z16.s, p0/m, z0.s, z8.s + fmla z17.s, p0/m, z1.s, z8.s + fmla z18.s, p0/m, z0.s, z9.s + fmla z19.s, p0/m, z1.s, z9.s +.endm + +.macro SAVEv2x2 + add pCRow1, pCRow0, LDC + ld1w z8.s, p0/z, [pCRow0] + ld1w z9.s, p0/z, [pCRow0, #1, mul vl] + fmla z8.s, p0/m, z16.s, alphaZ + fmla z9.s, p0/m, z17.s, alphaZ + st1w z8.s, p0, [pCRow0] + st1w z9.s, p0, [pCRow0, #1, mul vl] + + ld1w z10.s, p0/z, [pCRow1] + ld1w z11.s, p0/z, [pCRow1, #1, mul vl] + fmla z10.s, p0/m, z18.s, alphaZ + fmla z11.s, p0/m, z19.s, alphaZ + st1w z10.s, p0, [pCRow1] + st1w z11.s, p0, [pCRow1, #1, mul vl] + + add pCRow0, pCRow0, vec_len, lsl #3 // pC = pC + vec_len * 4 * 2 +.endm + +.macro INITv2x1 + dup z16.s, #0 + dup z17.s, #0 +.endm + +.macro KERNELv2x1_SUB + ld1w z0.s, p0/z, [pA1] + ld1w z1.s, p0/z, [pA1, vec_len, lsl #2] + add pA1, pA1, vec_len, lsl #3 // pA1 = pA1 + vec_len * 4 * 2 + + ld1rw z8.s, p0/z, [pB] + + add pB, pB, 4 + + fmla z16.s, p0/m, z0.s, z8.s + fmla z17.s, p0/m, z1.s, z8.s +.endm + +.macro SAVEv2x1 + add pCRow1, pCRow0, LDC + ld1w z8.s, p0/z, [pCRow0] + ld1w z9.s, p0/z, [pCRow0, #1, mul vl] + fmla z8.s, p0/m, z16.s, alphaZ + fmla z9.s, p0/m, z17.s, alphaZ + st1w z8.s, p0, [pCRow0] + st1w z9.s, p0, [pCRow0, #1, mul vl] + + add pCRow0, pCRow0, vec_len, lsl #3 // pC = pC + vec_len * 4 * 2 + +.endm + +.macro INITv1x8 + dup z16.s, #0 + dup z17.s, #0 + dup z18.s, #0 + dup z19.s, #0 + dup z20.s, #0 + dup z21.s, #0 + dup z22.s, #0 + dup z23.s, #0 +.endm + +.macro KERNELv1x8_I + ld1w z0.s, p1/z, [pA1] + ld1w z1.s, p1/z, [pA1, lanes, lsl #2] // next one + add pA1, pA1, lanes, lsl #3 // pA1 = pA1 + lanes * 2 * 4 + + ld1rw z8.s, p0/z, [pB] + ld1rw z9.s, p0/z, [pB, 4] + ld1rw z10.s, p0/z, [pB, 8] + ld1rw z11.s, p0/z, [pB, 12] + ld1rw z12.s, p0/z, [pB, 16] + ld1rw z13.s, p0/z, [pB, 20] + ld1rw z14.s, p0/z, [pB, 24] + ld1rw z15.s, p0/z, [pB, 28] + + add pB, pB, 32 + + fmla z16.s, p1/m, z0.s, z8.s + ld1rw z8.s, p0/z, [pB] + fmla z17.s, p1/m, z0.s, z9.s + ld1rw z9.s, p0/z, [pB, 4] + fmla z18.s, p1/m, z0.s, z10.s + ld1rw z10.s, p0/z, [pB, 8] + fmla z19.s, p1/m, z0.s, z11.s + ld1rw z11.s, p0/z, [pB, 12] + fmla z20.s, p1/m, z0.s, z12.s + ld1rw z12.s, p0/z, [pB, 16] + fmla z21.s, p1/m, z0.s, z13.s + ld1rw z13.s, p0/z, [pB, 20] + fmla z22.s, p1/m, z0.s, z14.s + ld1rw z14.s, p0/z, [pB, 24] + fmla z23.s, p1/m, z0.s, z15.s + ld1rw z15.s, p0/z, [pB, 28] + + add pB, pB, 32 +.endm + +.macro KERNELv1x8_M1 + ld1w z1.s, p1/z, [pA1] + add pA1, pA1, lanes, lsl #2 // pA1 = pA1 + lanes * 4 + + fmla z16.s, p1/m, z0.s, z8.s + ld1rw z8.s, p0/z, [pB] + fmla z17.s, p1/m, z0.s, z9.s + ld1rw z9.s, p0/z, [pB, 4] + fmla z18.s, p1/m, z0.s, z10.s + ld1rw z10.s, p0/z, [pB, 8] + fmla z19.s, p1/m, z0.s, z11.s + ld1rw z11.s, p0/z, [pB, 12] + fmla z20.s, p1/m, z0.s, z12.s + ld1rw z12.s, p0/z, [pB, 16] + fmla z21.s, p1/m, z0.s, z13.s + ld1rw z13.s, p0/z, [pB, 20] + fmla z22.s, p1/m, z0.s, z14.s + ld1rw z14.s, p0/z, [pB, 24] + fmla z23.s, p1/m, z0.s, z15.s + ld1rw z15.s, p0/z, [pB, 28] + + add pB, pB, 32 +.endm + +.macro KERNELv1x8_M2 + ld1w z0.s, p1/z, [pA1] + add pA1, pA1, lanes, lsl #2 // pA1 = pA1 + lanes * 4 + + fmla z16.s, p1/m, z1.s, z8.s + ld1rw z8.s, p0/z, [pB] + fmla z17.s, p1/m, z1.s, z9.s + ld1rw z9.s, p0/z, [pB, 4] + fmla z18.s, p1/m, z1.s, z10.s + ld1rw z10.s, p0/z, [pB, 8] + fmla z19.s, p1/m, z1.s, z11.s + ld1rw z11.s, p0/z, [pB, 12] + fmla z20.s, p1/m, z1.s, z12.s + ld1rw z12.s, p0/z, [pB, 16] + fmla z21.s, p1/m, z1.s, z13.s + ld1rw z13.s, p0/z, [pB, 20] + fmla z22.s, p1/m, z1.s, z14.s + ld1rw z14.s, p0/z, [pB, 24] + fmla z23.s, p1/m, z1.s, z15.s + ld1rw z15.s, p0/z, [pB, 28] + + add pB, pB, 32 +.endm + +.macro KERNELv1x8_E + fmla z16.s, p1/m, z1.s, z8.s + fmla z17.s, p1/m, z1.s, z9.s + fmla z18.s, p1/m, z1.s, z10.s + fmla z19.s, p1/m, z1.s, z11.s + fmla z20.s, p1/m, z1.s, z12.s + fmla z21.s, p1/m, z1.s, z13.s + fmla z22.s, p1/m, z1.s, z14.s + fmla z23.s, p1/m, z1.s, z15.s +.endm + +.macro KERNELv1x8_SUB + ld1w z0.s, p1/z, [pA1] + add pA1, pA1, lanes, lsl #2 // pA1 = pA1 + lanes * 4 + + ld1rw z8.s, p0/z, [pB] + ld1rw z9.s, p0/z, [pB, 4] + ld1rw z10.s, p0/z, [pB, 8] + ld1rw z11.s, p0/z, [pB, 12] + ld1rw z12.s, p0/z, [pB, 16] + ld1rw z13.s, p0/z, [pB, 20] + ld1rw z14.s, p0/z, [pB, 24] + ld1rw z15.s, p0/z, [pB, 28] + + add pB, pB, 32 + + fmla z16.s, p1/m, z0.s, z8.s + fmla z17.s, p1/m, z0.s, z9.s + fmla z18.s, p1/m, z0.s, z10.s + fmla z19.s, p1/m, z0.s, z11.s + fmla z20.s, p1/m, z0.s, z12.s + fmla z21.s, p1/m, z0.s, z13.s + fmla z22.s, p1/m, z0.s, z14.s + fmla z23.s, p1/m, z0.s, z15.s + +.endm + +.macro SAVEv1x8 + add pCRow1, pCRow0, LDC + ld1w z24.s, p1/z, [pCRow0] + fmla z24.s, p1/m, z16.s, alphaZ + st1w z24.s, p1, [pCRow0] + + add pCRow2, pCRow1, LDC + ld1w z25.s, p1/z, [pCRow1] + fmla z25.s, p1/m, z17.s, alphaZ + st1w z25.s, p1, [pCRow1] + + add pCRow1, pCRow2, LDC + ld1w z26.s, p1/z, [pCRow2] + fmla z26.s, p1/m, z18.s, alphaZ + st1w z26.s, p1, [pCRow2] + + add pCRow2, pCRow1, LDC + ld1w z27.s, p1/z, [pCRow1] + fmla z27.s, p1/m, z19.s, alphaZ + st1w z27.s, p1, [pCRow1] + + add pCRow1, pCRow2, LDC + ld1w z28.s, p1/z, [pCRow2] + fmla z28.s, p1/m, z20.s, alphaZ + st1w z28.s, p1, [pCRow2] + + add pCRow2, pCRow1, LDC + ld1w z29.s, p1/z, [pCRow1] + fmla z29.s, p1/m, z21.s, alphaZ + st1w z29.s, p1, [pCRow1] + + add pCRow1, pCRow2, LDC + ld1w z30.s, p1/z, [pCRow2] + fmla z30.s, p1/m, z22.s, alphaZ + st1w z30.s, p1, [pCRow2] + + ld1w z31.s, p1/z, [pCRow1] + fmla z31.s, p1/m, z23.s, alphaZ + st1w z31.s, p1, [pCRow1] + + add pCRow0, pCRow0, lanes, lsl #2 // pC = pC + lanes * 4 + +.endm + +/******************************************************************************/ + +.macro INITv1x4 + dup z16.s, #0 + dup z17.s, #0 + dup z18.s, #0 + dup z19.s, #0 +.endm + +.macro KERNELv1x4_SUB + ld1w z0.s, p1/z, [pA1] + add pA1, pA1, lanes, lsl #2 // pA1 = pA1 + lanes * 4 + + ld1rw z8.s, p0/z, [pB] + ld1rw z9.s, p0/z, [pB, 4] + ld1rw z10.s, p0/z, [pB, 8] + ld1rw z11.s, p0/z, [pB, 12] + + add pB, pB, 16 + + fmla z16.s, p1/m, z0.s, z8.s + fmla z17.s, p1/m, z0.s, z9.s + fmla z18.s, p1/m, z0.s, z10.s + fmla z19.s, p1/m, z0.s, z11.s + +.endm + +.macro SAVEv1x4 + add pCRow1, pCRow0, LDC + ld1w z24.s, p1/z, [pCRow0] + fmla z24.s, p1/m, z16.s, alphaZ + st1w z24.s, p1, [pCRow0] + + add pCRow2, pCRow1, LDC + ld1w z25.s, p1/z, [pCRow1] + fmla z25.s, p1/m, z17.s, alphaZ + st1w z25.s, p1, [pCRow1] + + add pCRow1, pCRow2, LDC + ld1w z26.s, p1/z, [pCRow2] + fmla z26.s, p1/m, z18.s, alphaZ + st1w z26.s, p1, [pCRow2] + + ld1w z27.s, p1/z, [pCRow1] + fmla z27.s, p1/m, z19.s, alphaZ + st1w z27.s, p1, [pCRow1] + + add pCRow0, pCRow0, lanes, lsl #2 // pC = pC + lanes * 4 + +.endm + +/******************************************************************************/ + +.macro INITv1x2 + dup z16.s, #0 + dup z17.s, #0 +.endm + +.macro KERNELv1x2_SUB + ld1w z0.s, p1/z, [pA1] + add pA1, pA1, lanes, lsl #2 // pA1 = pA1 + lanes * 4 + + ld1rw z8.s, p0/z, [pB] + ld1rw z9.s, p0/z, [pB, 4] + + add pB, pB, 8 + + fmla z16.s, p1/m, z0.s, z8.s + fmla z17.s, p1/m, z0.s, z9.s + +.endm + +.macro SAVEv1x2 + add pCRow1, pCRow0, LDC + ld1w z24.s, p1/z, [pCRow0] + fmla z24.s, p1/m, z16.s, alphaZ + st1w z24.s, p1, [pCRow0] + + ld1w z25.s, p1/z, [pCRow1] + fmla z25.s, p1/m, z17.s, alphaZ + st1w z25.s, p1, [pCRow1] + + add pCRow0, pCRow0, lanes, lsl #2 // pC = pC + lanes * 4 + +.endm + +/******************************************************************************/ + +.macro INITv1x1 + dup z16.s, #0 +.endm + +.macro KERNELv1x1_SUB + ld1w z0.s, p1/z, [pA1] + add pA1, pA1, lanes, lsl #2 // pA1 = pA1 + lanes * 4 + + ld1rw z8.s, p0/z, [pB] + + add pB, pB, 4 + + fmla z16.s, p1/m, z0.s, z8.s + +.endm + +.macro SAVEv1x1 + ld1w z24.s, p1/z, [pCRow0] + fmla z24.s, p1/m, z16.s, alphaZ + st1w z24.s, p1, [pCRow0] + + + add pCRow0, pCRow0, lanes, lsl #2 // pC = pC + lanes * 4 + +.endm + + +/******************************************************************************* +* End of macro definitions +*******************************************************************************/ + + PROLOGUE + + .align 5 + add sp, sp, #-(11 * 16) + stp d8, d9, [sp, #(0 * 16)] + stp d10, d11, [sp, #(1 * 16)] + stp d12, d13, [sp, #(2 * 16)] + stp d14, d15, [sp, #(3 * 16)] + stp d16, d17, [sp, #(4 * 16)] + stp x18, x19, [sp, #(5 * 16)] + stp x20, x21, [sp, #(6 * 16)] + stp x22, x23, [sp, #(7 * 16)] + stp x24, x25, [sp, #(8 * 16)] + stp x26, x27, [sp, #(9 * 16)] + str x28, [sp, #(10 * 16)] + + fmov alpha, s0 + dup alphaZ, alpha + cntw vec_len + lsl vec_lenx2, vec_len, #1 + + lsl LDC, LDC, #2 // ldc = ldc * 8 + ptrue p0.s // create true predicate + + mov pB, origPB +// Loop over N + mov counterJ, origN + asr counterJ, counterJ, #3 // J = J / 8 + cmp counterJ, #0 + ble .Lsgemm_kernel_L4_BEGIN + +/******************************************************************************/ +/* Repeat this as long as there are 8 left in N */ + + .align 5 +.Lsgemm_kernel_L8_BEGIN: + mov pCRow0, pC + + add pC, pC, LDC, lsl #3 // add 8 x LDC + + mov pA1, origPA // pA1 = start of A array + +.Lsgemm_kernel_L8_Mv2_BEGIN: + + mov counterI, #0 + cmp origM, vec_lenx2 // Check if M < 2*SVE_LEN + blt .Lsgemm_kernel_L8_Mv1_BEGIN + + mov counterI, origM + +/* Until we have at least 2*SVE_LEN iters left in M, we do them with V2*8 kernel */ + + .align 5 +.Lsgemm_kernel_L8_Mv2_20: + + mov pB, origPB + INITv2x8 // fill with zeros + + asr counterL , origK, #3 // L = K / 8 + cmp counterL , #2 // is there at least 4 to do? + blt .Lsgemm_kernel_L8_Mv2_32 + + KERNELv2x8_I + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_M2 + + subs counterL, counterL, #2 // subtract 2 + ble .Lsgemm_kernel_L8_Mv2_22a + + .align 5 +.Lsgemm_kernel_L8_Mv2_22: + + KERNELv2x8_M1 + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_M2 + + subs counterL, counterL, #1 + bgt .Lsgemm_kernel_L8_Mv2_22 + + .align 5 +.Lsgemm_kernel_L8_Mv2_22a: + + KERNELv2x8_M1 + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_E + + b .Lsgemm_kernel_L8_Mv2_44 + + .align 5 +.Lsgemm_kernel_L8_Mv2_32: + + tst counterL, #1 + ble .Lsgemm_kernel_L8_Mv2_40 + + KERNELv2x8_I + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_M2 + KERNELv2x8_M1 + KERNELv2x8_E + + + b .Lsgemm_kernel_L8_Mv2_44 + +.Lsgemm_kernel_L8_Mv2_40: + + INITv2x8 + +.Lsgemm_kernel_L8_Mv2_44: + + ands counterL , origK, #7 + ble .Lsgemm_kernel_L8_Mv2_100 + + .align 5 +.Lsgemm_kernel_L8_Mv2_46: + + KERNELv2x8_SUB + + subs counterL, counterL, #1 + bne .Lsgemm_kernel_L8_Mv2_46 + +.Lsgemm_kernel_L8_Mv2_100: + + SAVEv2x8 + +.Lsgemm_kernel_L8_Mv2_END: + sub counterI, counterI, vec_lenx2 + cmp counterI, vec_lenx2 + bge .Lsgemm_kernel_L8_Mv2_20 + sub counterI, origM, counterI + + cmp counterI, origM + beq .Lsgemm_kernel_L8_END + +////////////////////////////////////////// +// We have less than 2*SVE_LEN left. We do this with V1x8 kernel. +.Lsgemm_kernel_L8_Mv1_BEGIN: + + whilelt p1.s, counterI, origM //SVE instruction + cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension + + .align 5 +.Lsgemm_kernel_L8_Mv1_20: + + mov pB, origPB + INITv1x8 // fill with zeros + + asr counterL , origK, #3 // L = K / 8 + cmp counterL , #2 // is there at least 4 to do? + blt .Lsgemm_kernel_L8_Mv1_32 + + KERNELv1x8_I + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_M2 + + subs counterL, counterL, #2 // subtract 2 + ble .Lsgemm_kernel_L8_Mv1_22a + + .align 5 +.Lsgemm_kernel_L8_Mv1_22: + + KERNELv1x8_M1 + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_M2 + + subs counterL, counterL, #1 + bgt .Lsgemm_kernel_L8_Mv1_22 + + .align 5 +.Lsgemm_kernel_L8_Mv1_22a: + + KERNELv1x8_M1 + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_E + + b .Lsgemm_kernel_L8_Mv1_44 + + .align 5 +.Lsgemm_kernel_L8_Mv1_32: + + tst counterL, #1 + ble .Lsgemm_kernel_L8_Mv1_40 + + KERNELv1x8_I + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_M2 + KERNELv1x8_M1 + KERNELv1x8_E + + + b .Lsgemm_kernel_L8_Mv1_44 + +.Lsgemm_kernel_L8_Mv1_40: + + INITv1x8 + +.Lsgemm_kernel_L8_Mv1_44: + + ands counterL , origK, #7 + ble .Lsgemm_kernel_L8_Mv1_100 + + .align 5 +.Lsgemm_kernel_L8_Mv1_46: + + KERNELv1x8_SUB + + subs counterL, counterL, #1 + bne .Lsgemm_kernel_L8_Mv1_46 + +.Lsgemm_kernel_L8_Mv1_100: + + SAVEv1x8 + +.Lsgemm_kernel_L8_Mv1_END: + + incw counterI + whilelt p1.s, counterI, origM //SVE instruction + cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension + b.any .Lsgemm_kernel_L8_Mv1_20 + +.Lsgemm_kernel_L8_END: + + lsl temp, origK, #5 + add origPB, origPB, temp // B = B + K * 8 * 4 + + subs counterJ, counterJ , #1 // j-- + bgt .Lsgemm_kernel_L8_BEGIN + +/******************************************************************************/ +/* Repeat the same thing if 4 left in N */ + + .align 5 +.Lsgemm_kernel_L4_BEGIN: + + mov counterJ , origN + tst counterJ , #4 + ble .Lsgemm_kernel_L2_BEGIN + + + mov pCRow0, pC + + add pC, pC, LDC, lsl #2 // add 4 x LDC + + mov pA1, origPA // pA1 = start of A array + +.Lsgemm_kernel_L4_Mv2_BEGIN: + + mov counterI, #0 + cmp origM, vec_lenx2 + blt .Lsgemm_kernel_L4_Mv1_BEGIN + + mov counterI, origM + + .align 5 +.Lsgemm_kernel_L4_Mv2_20: + + mov pB, origPB + INITv2x4 // fill with zeros + + asr counterL , origK, #3 // L = K / 8 + cmp counterL , #0 // is there at least 4 to do? + ble .Lsgemm_kernel_L4_Mv2_44 + + .align 5 +.Lsgemm_kernel_L4_Mv2_22: + + KERNELv2x4_SUB + KERNELv2x4_SUB + KERNELv2x4_SUB + KERNELv2x4_SUB + KERNELv2x4_SUB + KERNELv2x4_SUB + KERNELv2x4_SUB + KERNELv2x4_SUB + + subs counterL, counterL, #1 + bgt .Lsgemm_kernel_L4_Mv2_22 + +.Lsgemm_kernel_L4_Mv2_44: + + ands counterL , origK, #7 + ble .Lsgemm_kernel_L4_Mv2_100 + + .align 5 +.Lsgemm_kernel_L4_Mv2_46: + + KERNELv2x4_SUB + + subs counterL, counterL, #1 + bne .Lsgemm_kernel_L4_Mv2_46 + +.Lsgemm_kernel_L4_Mv2_100: + + SAVEv2x4 + +.Lsgemm_kernel_L4_Mv2_END: + sub counterI, counterI, vec_lenx2 + cmp counterI, vec_lenx2 + bge .Lsgemm_kernel_L4_Mv2_20 + sub counterI, origM, counterI + + cmp counterI, origM + beq .Lsgemm_kernel_L4_END + +////////////////////////////////// +// We have less than 2*SVE_LEN left. We do this with V1x4 kernel. +.Lsgemm_kernel_L4_Mv1_BEGIN: + + whilelt p1.s, counterI, origM //SVE instruction + cntp lanes, p0, p1.s // lanes contain number of active SVE lanes in M dimension + + .align 5 +.Lsgemm_kernel_L4_Mv1_20: + + mov pB, origPB + INITv1x4 // fill with zeros + + asr counterL , origK, #3 // L = K / 8 + cmp counterL , #0 // is there at least 4 to do? + ble .Lsgemm_kernel_L4_Mv1_44 + + .align 5 +.Lsgemm_kernel_L4_Mv1_22: + + KERNELv1x4_SUB + KERNELv1x4_SUB + KERNELv1x4_SUB + KERNELv1x4_SUB + KERNELv1x4_SUB + KERNELv1x4_SUB + KERNELv1x4_SUB + KERNELv1x4_SUB + + subs counterL, counterL, #1 + bgt .Lsgemm_kernel_L4_Mv1_22 + +.Lsgemm_kernel_L4_Mv1_44: + + ands counterL , origK, #7 + ble .Lsgemm_kernel_L4_Mv1_100 + + .align 5 +.Lsgemm_kernel_L4_Mv1_46: + + KERNELv1x4_SUB + + subs counterL, counterL, #1 + bne .Lsgemm_kernel_L4_Mv1_46 + +.Lsgemm_kernel_L4_Mv1_100: + + SAVEv1x4 + +.Lsgemm_kernel_L4_Mv1_END: + + incw counterI + whilelt p1.s, counterI, origM //SVE instruction + cntp lanes, p0, p1.s + b.any .Lsgemm_kernel_L4_Mv1_20 + + +.Lsgemm_kernel_L4_END: + lsl temp, origK, #4 + add origPB, origPB, temp // B = B + K * 4 * 4 + +/******************************************************************************/ +/* Repeat the same thing if 2 left in N */ + + .align 5 +.Lsgemm_kernel_L2_BEGIN: + + mov counterJ , origN + tst counterJ , #2 + ble .Lsgemm_kernel_L1_BEGIN + + mov pCRow0, pC + + add pC, pC, LDC, lsl #1 // add 2 x LDC + + mov pA1, origPA // pA1 = start of A array + +.Lsgemm_kernel_L2_Mv2_BEGIN: + + mov counterI, #0 + cmp origM, vec_lenx2 + blt .Lsgemm_kernel_L2_Mv1_BEGIN + + mov counterI, origM + + .align 5 +.Lsgemm_kernel_L2_Mv2_20: + + mov pB, origPB + INITv2x2 // fill with zeros + + asr counterL , origK, #3 // L = K / 8 + cmp counterL , #0 // is there at least 4 to do? + ble .Lsgemm_kernel_L2_Mv2_44 + + .align 5 +.Lsgemm_kernel_L2_Mv2_22: + + KERNELv2x2_SUB + KERNELv2x2_SUB + KERNELv2x2_SUB + KERNELv2x2_SUB + KERNELv2x2_SUB + KERNELv2x2_SUB + KERNELv2x2_SUB + KERNELv2x2_SUB + + subs counterL, counterL, #1 + bgt .Lsgemm_kernel_L2_Mv2_22 + +.Lsgemm_kernel_L2_Mv2_44: + + ands counterL , origK, #7 + ble .Lsgemm_kernel_L2_Mv2_100 + + .align 5 +.Lsgemm_kernel_L2_Mv2_46: + + KERNELv2x2_SUB + + subs counterL, counterL, #1 + bne .Lsgemm_kernel_L2_Mv2_46 + +.Lsgemm_kernel_L2_Mv2_100: + + SAVEv2x2 + +.Lsgemm_kernel_L2_Mv2_END: + sub counterI, counterI, vec_lenx2 + cmp counterI, vec_lenx2 + bge .Lsgemm_kernel_L2_Mv2_20 + sub counterI, origM, counterI + + cmp counterI, origM + beq .Lsgemm_kernel_L2_END + + +////////////////////////////////// +// We have less than 2*SVE_LEN left. We do this with V1x2 kernel. +.Lsgemm_kernel_L2_Mv1_BEGIN: + + whilelt p1.s, counterI, origM //SVE instruction + cntp lanes, p0, p1.s + + .align 5 +.Lsgemm_kernel_L2_Mv1_20: + + mov pB, origPB + INITv1x2 // fill with zeros + + asr counterL , origK, #3 // L = K / 8 + cmp counterL , #0 // is there at least 4 to do? + ble .Lsgemm_kernel_L2_Mv1_44 + + .align 5 +.Lsgemm_kernel_L2_Mv1_22: + + KERNELv1x2_SUB + KERNELv1x2_SUB + KERNELv1x2_SUB + KERNELv1x2_SUB + KERNELv1x2_SUB + KERNELv1x2_SUB + KERNELv1x2_SUB + KERNELv1x2_SUB + + subs counterL, counterL, #1 + bgt .Lsgemm_kernel_L2_Mv1_22 + +.Lsgemm_kernel_L2_Mv1_44: + + ands counterL , origK, #7 + ble .Lsgemm_kernel_L2_Mv1_100 + + .align 5 +.Lsgemm_kernel_L2_Mv1_46: + + KERNELv1x2_SUB + + subs counterL, counterL, #1 + bne .Lsgemm_kernel_L2_Mv1_46 + +.Lsgemm_kernel_L2_Mv1_100: + + SAVEv1x2 + +.Lsgemm_kernel_L2_Mv1_END: + + incw counterI + whilelt p1.s, counterI, origM //SVE instruction + cntp lanes, p0, p1.s + b.any .Lsgemm_kernel_L2_Mv1_20 + + +.Lsgemm_kernel_L2_END: + add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4 + +/******************************************************************************/ +/* Repeat the same thing if 1 left in N */ + + .align 5 +.Lsgemm_kernel_L1_BEGIN: + + mov counterJ , origN + tst counterJ , #1 + ble .Lsgemm_kernel_L999 // done + + mov pCRow0, pC + + add pC, pC, LDC // add 1 x LDC + + mov pA1, origPA // pA1 = start of A array + +.Lsgemm_kernel_L1_Mv2_BEGIN: + + mov counterI, #0 + cmp origM, vec_lenx2 + blt .Lsgemm_kernel_L1_Mv1_BEGIN + + mov counterI, origM + + .align 5 +.Lsgemm_kernel_L1_Mv2_20: + + mov pB, origPB + INITv2x1 // fill with zeros + + asr counterL , origK, #3 // L = K / 8 + cmp counterL , #0 // is there at least 8 to do? + ble .Lsgemm_kernel_L1_Mv2_44 + + .align 5 +.Lsgemm_kernel_L1_Mv2_22: + + KERNELv2x1_SUB + KERNELv2x1_SUB + KERNELv2x1_SUB + KERNELv2x1_SUB + KERNELv2x1_SUB + KERNELv2x1_SUB + KERNELv2x1_SUB + KERNELv2x1_SUB + + subs counterL, counterL, #1 + bgt .Lsgemm_kernel_L1_Mv2_22 + +.Lsgemm_kernel_L1_Mv2_44: + + ands counterL , origK, #7 + ble .Lsgemm_kernel_L1_Mv2_100 + + .align 5 +.Lsgemm_kernel_L1_Mv2_46: + + KERNELv2x1_SUB + + subs counterL, counterL, #1 + bgt .Lsgemm_kernel_L1_Mv2_46 + +.Lsgemm_kernel_L1_Mv2_100: + + SAVEv2x1 + +.Lsgemm_kernel_L1_Mv2_END: + sub counterI, counterI, vec_lenx2 + cmp counterI, vec_lenx2 + bge .Lsgemm_kernel_L1_Mv2_20 + sub counterI, origM, counterI + + cmp counterI, origM + beq .Lsgemm_kernel_L1_END + + +////////////////////////////////// +// We have less than 2*SVE_LEN left. We do this with V1x1 kernel. +.Lsgemm_kernel_L1_Mv1_BEGIN: + + whilelt p1.s, counterI, origM //SVE instruction + cntp lanes, p0, p1.s + + .align 5 +.Lsgemm_kernel_L1_Mv1_20: + + mov pB, origPB + INITv1x1 // fill with zeros + + asr counterL , origK, #3 // L = K / 8 + cmp counterL , #0 // is there at least 8 to do? + ble .Lsgemm_kernel_L1_Mv1_44 + + .align 5 +.Lsgemm_kernel_L1_Mv1_22: + + KERNELv1x1_SUB + KERNELv1x1_SUB + KERNELv1x1_SUB + KERNELv1x1_SUB + KERNELv1x1_SUB + KERNELv1x1_SUB + KERNELv1x1_SUB + KERNELv1x1_SUB + + subs counterL, counterL, #1 + bgt .Lsgemm_kernel_L1_Mv1_22 + +.Lsgemm_kernel_L1_Mv1_44: + + ands counterL , origK, #7 + ble .Lsgemm_kernel_L1_Mv1_100 + + .align 5 +.Lsgemm_kernel_L1_Mv1_46: + + KERNELv1x1_SUB + + subs counterL, counterL, #1 + bgt .Lsgemm_kernel_L1_Mv1_46 + +.Lsgemm_kernel_L1_Mv1_100: + + SAVEv1x1 + +.Lsgemm_kernel_L1_Mv1_END: + + incw counterI + whilelt p1.s, counterI, origM //SVE instruction + cntp lanes, p0, p1.s + b.any .Lsgemm_kernel_L1_Mv1_20 + + +.Lsgemm_kernel_L1_END: + +/******************************************************************************/ + +.Lsgemm_kernel_L999: + mov x0, #0 // set return value + ldp d8, d9, [sp, #(0 * 16)] + ldp d10, d11, [sp, #(1 * 16)] + ldp d12, d13, [sp, #(2 * 16)] + ldp d14, d15, [sp, #(3 * 16)] + ldp d16, d17, [sp, #(4 * 16)] + ldp x18, x19, [sp, #(5 * 16)] + ldp x20, x21, [sp, #(6 * 16)] + ldp x22, x23, [sp, #(7 * 16)] + ldp x24, x25, [sp, #(8 * 16)] + ldp x26, x27, [sp, #(9 * 16)] + ldr x28, [sp, #(10 * 16)] + add sp, sp, #(11*16) + ret + + EPILOGUE + diff --git a/kernel/setparam-ref.c b/kernel/setparam-ref.c index a38f25a679..f8251240a7 100644 --- a/kernel/setparam-ref.c +++ b/kernel/setparam-ref.c @@ -246,7 +246,7 @@ gotoblas_t TABLE_NAME = { #endif #endif - sgemm_kernelTS, sgemm_betaTS, + sgemm_kernelTS, sgemm_betaTS, scomm_kernelTS, #if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N sgemm_incopyTS, sgemm_itcopyTS, #else @@ -284,6 +284,7 @@ gotoblas_t TABLE_NAME = { strmm_ounucopyTS, strmm_ounncopyTS, strmm_outucopyTS, strmm_outncopyTS, strmm_olnucopyTS, strmm_olnncopyTS, strmm_oltucopyTS, strmm_oltncopyTS, #endif + scomm_ncopyTS, scomm_tcopyTS, strmm_ounucopyTS, strmm_ounncopyTS, strmm_outucopyTS, strmm_outncopyTS, strmm_olnucopyTS, strmm_olnncopyTS, strmm_oltucopyTS, strmm_oltncopyTS, #if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N @@ -345,7 +346,7 @@ gotoblas_t TABLE_NAME = { NULL, #endif #endif - dgemm_kernelTS, dgemm_betaTS, + dgemm_kernelTS, dgemm_betaTS, dcomm_kernelTS, #if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N dgemm_incopyTS, dgemm_itcopyTS, #else @@ -380,6 +381,7 @@ gotoblas_t TABLE_NAME = { dtrmm_ounucopyTS, dtrmm_ounncopyTS, dtrmm_outucopyTS, dtrmm_outncopyTS, dtrmm_olnucopyTS, dtrmm_olnncopyTS, dtrmm_oltucopyTS, dtrmm_oltncopyTS, #endif + dcomm_ncopyTS, dcomm_tcopyTS, dtrmm_ounucopyTS, dtrmm_ounncopyTS, dtrmm_outucopyTS, dtrmm_outncopyTS, dtrmm_olnucopyTS, dtrmm_olnncopyTS, dtrmm_oltucopyTS, dtrmm_oltncopyTS, #if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N @@ -498,6 +500,7 @@ gotoblas_t TABLE_NAME = { #endif cgemm_kernel_nTS, cgemm_kernel_lTS, cgemm_kernel_rTS, cgemm_kernel_bTS, cgemm_betaTS, + ccomm_kernel_nTS, ccomm_kernel_lTS, ccomm_kernel_rTS, ccomm_kernel_bTS, #if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N cgemm_incopyTS, cgemm_itcopyTS, #else @@ -543,6 +546,7 @@ gotoblas_t TABLE_NAME = { ctrmm_ounucopyTS, ctrmm_ounncopyTS, ctrmm_outucopyTS, ctrmm_outncopyTS, ctrmm_olnucopyTS, ctrmm_olnncopyTS, ctrmm_oltucopyTS, ctrmm_oltncopyTS, #endif + ccomm_ncopyTS, ccomm_tcopyTS, ctrmm_ounucopyTS, ctrmm_ounncopyTS, ctrmm_outucopyTS, ctrmm_outncopyTS, ctrmm_olnucopyTS, ctrmm_olnncopyTS, ctrmm_oltucopyTS, ctrmm_oltncopyTS, @@ -660,6 +664,7 @@ gotoblas_t TABLE_NAME = { #endif zgemm_kernel_nTS, zgemm_kernel_lTS, zgemm_kernel_rTS, zgemm_kernel_bTS, zgemm_betaTS, + zcomm_kernel_nTS, zcomm_kernel_lTS, zcomm_kernel_rTS, zcomm_kernel_bTS, #if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N zgemm_incopyTS, zgemm_itcopyTS, @@ -703,6 +708,7 @@ gotoblas_t TABLE_NAME = { ztrmm_ounucopyTS, ztrmm_ounncopyTS, ztrmm_outucopyTS, ztrmm_outncopyTS, ztrmm_olnucopyTS, ztrmm_olnncopyTS, ztrmm_oltucopyTS, ztrmm_oltncopyTS, #endif + zcomm_ncopyTS, zcomm_tcopyTS, ztrmm_ounucopyTS, ztrmm_ounncopyTS, ztrmm_outucopyTS, ztrmm_outncopyTS, ztrmm_olnucopyTS, ztrmm_olnncopyTS, ztrmm_oltucopyTS, ztrmm_oltncopyTS, From 2e0abd6e45131ca30b937fe2aa8d8bec7bfc4b03 Mon Sep 17 00:00:00 2001 From: Anna Mayne Date: Wed, 17 Dec 2025 11:58:24 +0000 Subject: [PATCH 2/8] Fixed builds and added missing copyright notices. Also fixed formatting of copyright notices added in last commit. --- common_c.h | 44 +++++++++++++++++- common_d.h | 44 +++++++++++++++++- common_macro.h | 30 ++++-------- common_s.h | 44 +++++++++++++++++- common_z.h | 46 +++++++++++++++++-- kernel/arm64/KERNEL.ARMV8SVE | 27 +++++++++++ kernel/arm64/dgemm_kernel_inter_sve_v2x8.S | 53 +++++++++++----------- kernel/arm64/gemm_ncopy_sve_v2x8.c | 53 +++++++++++----------- kernel/arm64/gemm_tcopy_sve_v2x8.c | 53 +++++++++++----------- kernel/arm64/sgemm_kernel_inter_sve_v2x8.S | 53 +++++++++++----------- 10 files changed, 313 insertions(+), 134 deletions(-) diff --git a/common_c.h b/common_c.h index cca27ea073..8f4a869990 100644 --- a/common_c.h +++ b/common_c.h @@ -1,3 +1,42 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + #ifndef COMMON_C_H #define COMMON_C_H @@ -62,6 +101,9 @@ #define CGEMM_ITCOPY cgemm_itcopy #endif +#define CCOMM_NCOPY ccomm_ncopy +#define CCOMM_TCOPY ccomm_tcopy + #define CTRMM_OUNUCOPY ctrmm_ounucopy #define CTRMM_OUNNCOPY ctrmm_ounncopy #define CTRMM_OUTUCOPY ctrmm_outucopy @@ -107,8 +149,6 @@ #define CTRMM_ILNNCOPY ctrmm_ilnncopy #define CTRMM_ILTUCOPY ctrmm_iltucopy #define CTRMM_ILTNCOPY ctrmm_iltncopy -#define CCOMM_NCOPY ccomm_ncopy -#define CCOMM_TCOPY ccomm_tcopy #define CTRSM_IUNUCOPY ctrsm_iunucopy #define CTRSM_IUNNCOPY ctrsm_iunncopy diff --git a/common_d.h b/common_d.h index 342df59284..2e1700cae2 100644 --- a/common_d.h +++ b/common_d.h @@ -1,3 +1,42 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + #ifndef COMMON_D_H #define COMMON_D_H @@ -55,6 +94,9 @@ #define DGEMM_ITCOPY dgemm_itcopy #endif +#define DCOMM_NCOPY dcomm_ncopy +#define DCOMM_TCOPY dcomm_tcopy + #define DTRMM_OUNUCOPY dtrmm_ounucopy #define DTRMM_OUNNCOPY dtrmm_ounncopy #define DTRMM_OUTUCOPY dtrmm_outucopy @@ -100,8 +142,6 @@ #define DTRMM_ILNNCOPY dtrmm_ilnncopy #define DTRMM_ILTUCOPY dtrmm_iltucopy #define DTRMM_ILTNCOPY dtrmm_iltncopy -#define DCOMM_NCOPY dcomm_ncopy -#define DCOMM_TCOPY dcomm_tcopy #define DTRSM_IUNUCOPY dtrsm_iunucopy #define DTRSM_IUNNCOPY dtrsm_iunncopy diff --git a/common_macro.h b/common_macro.h index 411c7d2aa7..b773ab2cde 100644 --- a/common_macro.h +++ b/common_macro.h @@ -391,6 +391,8 @@ #define GEMM_OTCOPY DGEMM_OTCOPY #define GEMM_INCOPY DGEMM_INCOPY #define GEMM_ITCOPY DGEMM_ITCOPY +#define COMM_NCOPY DCOMM_NCOPY +#define COMM_TCOPY DCOMM_TCOPY #ifdef UNIT @@ -407,8 +409,6 @@ #define TRMM_IUTCOPY DTRMM_IUTUCOPY #define TRMM_ILNCOPY DTRMM_ILNUCOPY #define TRMM_ILTCOPY DTRMM_ILTUCOPY -#define COMM_NCOPY DCOMM_NCOPY -#define COMM_TCOPY DCOMM_TCOPY #define TRSM_IUNCOPY DTRSM_IUNUCOPY #define TRSM_IUTCOPY DTRSM_IUTUCOPY @@ -430,8 +430,6 @@ #define TRMM_IUTCOPY DTRMM_IUTNCOPY #define TRMM_ILNCOPY DTRMM_ILNNCOPY #define TRMM_ILTCOPY DTRMM_ILTNCOPY -#define COMM_NCOPY DCOMM_NCOPY -#define COMM_TCOPY DCOMM_TCOPY #define TRSM_IUNCOPY DTRSM_IUNNCOPY #define TRSM_IUTCOPY DTRSM_IUTNCOPY @@ -840,6 +838,8 @@ #define SYMM_RU SSYMM_RU #define SYMM_RL SSYMM_RL +#define COMM_NCOPY SCOMM_NCOPY +#define COMM_TCOPY SCOMM_TCOPY #define HEMM_THREAD_LU SHEMM_THREAD_LU #define HEMM_THREAD_LL SHEMM_THREAD_LL @@ -878,8 +878,6 @@ #define TRMM_IUTCOPY STRMM_IUTUCOPY #define TRMM_ILNCOPY STRMM_ILNUCOPY #define TRMM_ILTCOPY STRMM_ILTUCOPY -#define COMM_NCOPY SCOMM_NCOPY -#define COMM_TCOPY SCOMM_TCOPY #define TRSM_IUNCOPY STRSM_IUNUCOPY #define TRSM_IUTCOPY STRSM_IUTUCOPY @@ -901,8 +899,6 @@ #define TRMM_IUTCOPY STRMM_IUTNCOPY #define TRMM_ILNCOPY STRMM_ILNNCOPY #define TRMM_ILTCOPY STRMM_ILTNCOPY -#define COMM_NCOPY SCOMM_NCOPY -#define COMM_TCOPY SCOMM_TCOPY #define TRSM_IUNCOPY STRSM_IUNNCOPY #define TRSM_IUTCOPY STRSM_IUTNCOPY @@ -1119,6 +1115,8 @@ #define GEMM_OTCOPY SGEMM_OTCOPY #define GEMM_INCOPY SGEMM_INCOPY #define GEMM_ITCOPY SGEMM_ITCOPY +#define COMM_NCOPY SCOMM_NCOPY +#define COMM_TCOPY SCOMM_TCOPY #ifdef UNIT @@ -1135,8 +1133,6 @@ #define TRMM_IUTCOPY STRMM_IUTUCOPY #define TRMM_ILNCOPY STRMM_ILNUCOPY #define TRMM_ILTCOPY STRMM_ILTUCOPY -#define COMM_NCOPY SCOMM_NCOPY -#define COMM_TCOPY SCOMM_TCOPY #define TRSM_IUNCOPY STRSM_IUNUCOPY #define TRSM_IUTCOPY STRSM_IUTUCOPY @@ -1158,8 +1154,6 @@ #define TRMM_IUTCOPY STRMM_IUTNCOPY #define TRMM_ILNCOPY STRMM_ILNNCOPY #define TRMM_ILTCOPY STRMM_ILTNCOPY -#define COMM_NCOPY SCOMM_NCOPY -#define COMM_TCOPY SCOMM_TCOPY #define TRSM_IUNCOPY STRSM_IUNNCOPY #define TRSM_IUTCOPY STRSM_IUTNCOPY @@ -1858,6 +1852,8 @@ #define GEMM_OTCOPY ZGEMM_OTCOPY #define GEMM_INCOPY ZGEMM_INCOPY #define GEMM_ITCOPY ZGEMM_ITCOPY +#define COMM_NCOPY ZCOMM_NCOPY +#define COMM_TCOPY ZCOMM_TCOPY #define GEMM3M_ONCOPYB ZGEMM3M_ONCOPYB #define GEMM3M_ONCOPYR ZGEMM3M_ONCOPYR @@ -1887,8 +1883,6 @@ #define TRMM_IUTCOPY ZTRMM_IUTUCOPY #define TRMM_ILNCOPY ZTRMM_ILNUCOPY #define TRMM_ILTCOPY ZTRMM_ILTUCOPY -#define COMM_NCOPY ZCOMM_NCOPY -#define COMM_TCOPY ZCOMM_TCOPY #define TRSM_IUNCOPY ZTRSM_IUNUCOPY #define TRSM_IUTCOPY ZTRSM_IUTUCOPY @@ -1910,8 +1904,6 @@ #define TRMM_IUTCOPY ZTRMM_IUTNCOPY #define TRMM_ILNCOPY ZTRMM_ILNNCOPY #define TRMM_ILTCOPY ZTRMM_ILTNCOPY -#define COMM_NCOPY ZCOMM_NCOPY -#define COMM_TCOPY ZCOMM_TCOPY #define TRSM_IUNCOPY ZTRSM_IUNNCOPY #define TRSM_IUTCOPY ZTRSM_IUTNCOPY @@ -2334,6 +2326,8 @@ #define GEMM_OTCOPY CGEMM_OTCOPY #define GEMM_INCOPY CGEMM_INCOPY #define GEMM_ITCOPY CGEMM_ITCOPY +#define COMM_NCOPY CCOMM_NCOPY +#define COMM_TCOPY CCOMM_TCOPY #define GEMM3M_ONCOPYB CGEMM3M_ONCOPYB #define GEMM3M_ONCOPYR CGEMM3M_ONCOPYR @@ -2363,8 +2357,6 @@ #define TRMM_IUTCOPY CTRMM_IUTUCOPY #define TRMM_ILNCOPY CTRMM_ILNUCOPY #define TRMM_ILTCOPY CTRMM_ILTUCOPY -#define COMM_NCOPY CCOMM_NCOPY -#define COMM_TCOPY CCOMM_TCOPY #define TRSM_IUNCOPY CTRSM_IUNUCOPY #define TRSM_IUTCOPY CTRSM_IUTUCOPY @@ -2386,8 +2378,6 @@ #define TRMM_IUTCOPY CTRMM_IUTNCOPY #define TRMM_ILNCOPY CTRMM_ILNNCOPY #define TRMM_ILTCOPY CTRMM_ILTNCOPY -#define COMM_NCOPY CCOMM_NCOPY -#define COMM_TCOPY CCOMM_TCOPY #define TRSM_IUNCOPY CTRSM_IUNNCOPY #define TRSM_IUTCOPY CTRSM_IUTNCOPY diff --git a/common_s.h b/common_s.h index e0575130c2..fbe3705e68 100644 --- a/common_s.h +++ b/common_s.h @@ -1,3 +1,42 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + #ifndef COMMON_S_H #define COMMON_S_H @@ -77,6 +116,9 @@ #endif #define SME_SGEMM_KERNEL sme_sgemm_kernel +#define SCOMM_NCOPY scomm_ncopy +#define SCOMM_TCOPY scomm_tcopy + #define STRMM_OUNUCOPY strmm_ounucopy #define STRMM_OUNNCOPY strmm_ounncopy #define STRMM_OUTUCOPY strmm_outucopy @@ -122,8 +164,6 @@ #define STRMM_ILNNCOPY strmm_ilnncopy #define STRMM_ILTUCOPY strmm_iltucopy #define STRMM_ILTNCOPY strmm_iltncopy -#define SCOMM_NCOPY scomm_ncopy -#define SCOMM_TCOPY scomm_tcopy #define STRSM_IUNUCOPY strsm_iunucopy #define STRSM_IUNNCOPY strsm_iunncopy diff --git a/common_z.h b/common_z.h index 475644ef07..9e899394fa 100644 --- a/common_z.h +++ b/common_z.h @@ -1,3 +1,42 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + #ifndef COMMON_Z_H #define COMMON_Z_H @@ -62,6 +101,9 @@ #define ZGEMM_ITCOPY zgemm_itcopy #endif +#define ZCOMM_NCOPY zcomm_ncopy +#define ZCOMM_TCOPY zcomm_tcopy + #define ZTRMM_OUNUCOPY ztrmm_ounucopy #define ZTRMM_OUNNCOPY ztrmm_ounncopy #define ZTRMM_OUTUCOPY ztrmm_outucopy @@ -89,8 +131,6 @@ #define ZTRMM_ILNNCOPY ztrmm_olnncopy #define ZTRMM_ILTUCOPY ztrmm_oltucopy #define ZTRMM_ILTNCOPY ztrmm_oltncopy -#define ZCOMM_NCOPY zcomm_ncopy -#define ZCOMM_TCOPY zcomm_tcopy #define ZTRSM_IUNUCOPY ztrsm_ounucopy #define ZTRSM_IUNNCOPY ztrsm_ounncopy @@ -109,8 +149,6 @@ #define ZTRMM_ILNNCOPY ztrmm_ilnncopy #define ZTRMM_ILTUCOPY ztrmm_iltucopy #define ZTRMM_ILTNCOPY ztrmm_iltncopy -#define ZCOMM_NCOPY zcomm_ncopy -#define ZCOMM_TCOPY zcomm_tcopy #define ZTRSM_IUNUCOPY ztrsm_iunucopy #define ZTRSM_IUNNCOPY ztrsm_iunncopy diff --git a/kernel/arm64/KERNEL.ARMV8SVE b/kernel/arm64/KERNEL.ARMV8SVE index 1c10d8e257..0bc78b1892 100644 --- a/kernel/arm64/KERNEL.ARMV8SVE +++ b/kernel/arm64/KERNEL.ARMV8SVE @@ -1,3 +1,30 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### CSUMKERNEL = csum_thunderx2t99.c ZSUMKERNEL = zsum_thunderx2t99.c diff --git a/kernel/arm64/dgemm_kernel_inter_sve_v2x8.S b/kernel/arm64/dgemm_kernel_inter_sve_v2x8.S index 57532c3511..8de5626e0f 100644 --- a/kernel/arm64/dgemm_kernel_inter_sve_v2x8.S +++ b/kernel/arm64/dgemm_kernel_inter_sve_v2x8.S @@ -1,29 +1,30 @@ -/******************************************************************************* -Copyright (c) 2025, The OpenBLAS Project -All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: -1. Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in -the documentation and/or other materials provided with the -distribution. -3. Neither the name of the OpenBLAS project nor the names of -its contributors may be used to endorse or promote products -derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*******************************************************************************/ +/*******************************************************************************/ +/* Copyright (c) 2025, The OpenBLAS Project */ +/* All rights reserved. */ +/* Redistribution and use in source and binary forms, with or without */ +/* modification, are permitted provided that the following conditions are */ +/* met: */ +/* 1. Redistributions of source code must retain the above copyright */ +/* notice, this list of conditions and the following disclaimer. */ +/* 2. Redistributions in binary form must reproduce the above copyright */ +/* notice, this list of conditions and the following disclaimer in */ +/* the documentation and/or other materials provided with the */ +/* distribution. */ +/* 3. Neither the name of the OpenBLAS project nor the names of */ +/* its contributors may be used to endorse or promote products */ +/* derived from this software without specific prior written permission. */ +/* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" */ +/* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE */ +/* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE */ +/* ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE */ +/* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR */ +/* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF */ +/* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS */ +/* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN */ +/* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE). */ +/* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/*******************************************************************************/ /* This is an SVE dgemm kernel with size 2*SVE_LEN x 8. However, the data layout is the same as for the kernel 1*SVE_LEN x 8. diff --git a/kernel/arm64/gemm_ncopy_sve_v2x8.c b/kernel/arm64/gemm_ncopy_sve_v2x8.c index c810b929d5..8130889d47 100644 --- a/kernel/arm64/gemm_ncopy_sve_v2x8.c +++ b/kernel/arm64/gemm_ncopy_sve_v2x8.c @@ -1,29 +1,30 @@ -/*************************************************************************** -Copyright (c) 2023, 2025, The OpenBLAS Project -All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: -1. Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in -the documentation and/or other materials provided with the -distribution. -3. Neither the name of the OpenBLAS project nor the names of -its contributors may be used to endorse or promote products -derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A00 PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*****************************************************************************/ +/*******************************************************************************/ +/* Copyright (c) 2023, 2025, The OpenBLAS Project */ +/* All rights reserved. */ +/* Redistribution and use in source and binary forms, with or without */ +/* modification, are permitted provided that the following conditions are */ +/* met: */ +/* 1. Redistributions of source code must retain the above copyright */ +/* notice, this list of conditions and the following disclaimer. */ +/* 2. Redistributions in binary form must reproduce the above copyright */ +/* notice, this list of conditions and the following disclaimer in */ +/* the documentation and/or other materials provided with the */ +/* distribution. */ +/* 3. Neither the name of the OpenBLAS project nor the names of */ +/* its contributors may be used to endorse or promote products */ +/* derived from this software without specific prior written permission. */ +/* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" */ +/* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE */ +/* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE */ +/* ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE */ +/* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR */ +/* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF */ +/* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS */ +/* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN */ +/* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE). */ +/* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/*******************************************************************************/ #include #include diff --git a/kernel/arm64/gemm_tcopy_sve_v2x8.c b/kernel/arm64/gemm_tcopy_sve_v2x8.c index e4899fe576..7f350ed67b 100644 --- a/kernel/arm64/gemm_tcopy_sve_v2x8.c +++ b/kernel/arm64/gemm_tcopy_sve_v2x8.c @@ -1,29 +1,30 @@ -/*************************************************************************** -Copyright (c) 2023, 2025, The OpenBLAS Project -All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: -1. Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in -the documentation and/or other materials provided with the -distribution. -3. Neither the name of the OpenBLAS project nor the names of -its contributors may be used to endorse or promote products -derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A00 PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*****************************************************************************/ +/*******************************************************************************/ +/* Copyright (c) 2023, 2025, The OpenBLAS Project */ +/* All rights reserved. */ +/* Redistribution and use in source and binary forms, with or without */ +/* modification, are permitted provided that the following conditions are */ +/* met: */ +/* 1. Redistributions of source code must retain the above copyright */ +/* notice, this list of conditions and the following disclaimer. */ +/* 2. Redistributions in binary form must reproduce the above copyright */ +/* notice, this list of conditions and the following disclaimer in */ +/* the documentation and/or other materials provided with the */ +/* distribution. */ +/* 3. Neither the name of the OpenBLAS project nor the names of */ +/* its contributors may be used to endorse or promote products */ +/* derived from this software without specific prior written permission. */ +/* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" */ +/* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE */ +/* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE */ +/* ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE */ +/* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR */ +/* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF */ +/* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS */ +/* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN */ +/* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE). */ +/* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/*******************************************************************************/ #include #include diff --git a/kernel/arm64/sgemm_kernel_inter_sve_v2x8.S b/kernel/arm64/sgemm_kernel_inter_sve_v2x8.S index 49dee97ef7..2ea1850fe9 100644 --- a/kernel/arm64/sgemm_kernel_inter_sve_v2x8.S +++ b/kernel/arm64/sgemm_kernel_inter_sve_v2x8.S @@ -1,29 +1,30 @@ -/******************************************************************************* -Copyright (c) 2025, The OpenBLAS Project -All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: -1. Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in -the documentation and/or other materials provided with the -distribution. -3. Neither the name of the OpenBLAS project nor the names of -its contributors may be used to endorse or promote products -derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*******************************************************************************/ +/*******************************************************************************/ +/* Copyright (c) 2025, The OpenBLAS Project */ +/* All rights reserved. */ +/* Redistribution and use in source and binary forms, with or without */ +/* modification, are permitted provided that the following conditions are */ +/* met: */ +/* 1. Redistributions of source code must retain the above copyright */ +/* notice, this list of conditions and the following disclaimer. */ +/* 2. Redistributions in binary form must reproduce the above copyright */ +/* notice, this list of conditions and the following disclaimer in */ +/* the documentation and/or other materials provided with the */ +/* distribution. */ +/* 3. Neither the name of the OpenBLAS project nor the names of */ +/* its contributors may be used to endorse or promote products */ +/* derived from this software without specific prior written permission. */ +/* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" */ +/* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE */ +/* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE */ +/* ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE */ +/* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR */ +/* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF */ +/* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS */ +/* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN */ +/* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE). */ +/* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/*******************************************************************************/ /* This is an SVE sgemm kernel with size 2*SVE_LEN x 8. However, the data layout is the same as for the kernel 1*SVE_LEN x 8. From 1cd2f07da5a7c54b53dfaf2616925ab544641a1a Mon Sep 17 00:00:00 2001 From: Anna Mayne Date: Thu, 18 Dec 2025 09:08:21 +0000 Subject: [PATCH 3/8] Accommodate ex and quad precision builds. --- common_macro.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/common_macro.h b/common_macro.h index b773ab2cde..e17ffa6d5d 100644 --- a/common_macro.h +++ b/common_macro.h @@ -149,6 +149,11 @@ #define GEMM_KERNEL_R QGEMM_KERNEL #define GEMM_KERNEL_B QGEMM_KERNEL +#define COMM_KERNEL_N QGEMM_KERNEL +#define COMM_KERNEL_L QGEMM_KERNEL +#define COMM_KERNEL_R QGEMM_KERNEL +#define COMM_KERNEL_B QGEMM_KERNEL + #define TRMM_KERNEL_LN QTRMM_KERNEL_LN #define TRMM_KERNEL_LT QTRMM_KERNEL_LT #define TRMM_KERNEL_LR QTRMM_KERNEL_LN @@ -1541,6 +1546,11 @@ #define GEMM_KERNEL_R XGEMM_KERNEL_R #define GEMM_KERNEL_B XGEMM_KERNEL_B +#define COMM_KERNEL_N XGEMM_KERNEL_N +#define COMM_KERNEL_L XGEMM_KERNEL_L +#define COMM_KERNEL_R XGEMM_KERNEL_R +#define COMM_KERNEL_B XGEMM_KERNEL_B + #define GEMM3M_KERNEL XGEMM3M_KERNEL #define TRMM_KERNEL_LN XTRMM_KERNEL_LN From 857ec4d58312a0362638d422319f9804781f58d8 Mon Sep 17 00:00:00 2001 From: Anna Mayne Date: Thu, 18 Dec 2025 09:45:19 +0000 Subject: [PATCH 4/8] Add new copy functions to ex and quad precision builds. --- common_macro.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/common_macro.h b/common_macro.h index e17ffa6d5d..237523927f 100644 --- a/common_macro.h +++ b/common_macro.h @@ -99,6 +99,8 @@ #define GEMM_OTCOPY QGEMM_OTCOPY #define GEMM_INCOPY QGEMM_INCOPY #define GEMM_ITCOPY QGEMM_ITCOPY +#define COMM_NCOPY QGEMM_INCOPY +#define COMM_TCOPY QGEMM_ITCOPY #ifdef UNIT @@ -834,6 +836,8 @@ #define GEMM_OTCOPY SBGEMM_OTCOPY #define GEMM_INCOPY SBGEMM_INCOPY #define GEMM_ITCOPY SBGEMM_ITCOPY +#define COMM_NCOPY SCOMM_NCOPY +#define COMM_TCOPY SCOMM_TCOPY #define SYMM_THREAD_LU SSYMM_THREAD_LU #define SYMM_THREAD_LL SSYMM_THREAD_LL #define SYMM_THREAD_RU SSYMM_THREAD_RU @@ -843,9 +847,6 @@ #define SYMM_RU SSYMM_RU #define SYMM_RL SSYMM_RL -#define COMM_NCOPY SCOMM_NCOPY -#define COMM_TCOPY SCOMM_TCOPY - #define HEMM_THREAD_LU SHEMM_THREAD_LU #define HEMM_THREAD_LL SHEMM_THREAD_LL #define HEMM_THREAD_RU SHEMM_THREAD_RU @@ -1455,6 +1456,8 @@ #define GEMM_OTCOPY XGEMM_OTCOPY #define GEMM_INCOPY XGEMM_INCOPY #define GEMM_ITCOPY XGEMM_ITCOPY +#define COMM_NCOPY XGEMM_INCOPY +#define COMM_TCOPY XGEMM_ITCOPY #define GEMM3M_ONCOPYB XGEMM3M_ONCOPYB #define GEMM3M_ONCOPYR XGEMM3M_ONCOPYR From d1ddb8558bad3680b2d35a7754e7dc771272ae01 Mon Sep 17 00:00:00 2001 From: Anna Mayne Date: Thu, 18 Dec 2025 15:28:08 +0000 Subject: [PATCH 5/8] Fix CMake build. --- kernel/CMakeLists.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index a66918c9ba..3d420ec1f8 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -341,6 +341,8 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS) GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "" "gemm_kernel" false "" "" false ${float_type}) if (${float_char}COMMKERNEL) GenerateNamedObjects("${KERNELDIR}/${${float_char}COMMKERNEL}" "" "comm_kernel" false "" "" false ${float_type}) + else () + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "" "comm_kernel" false "" "" false ${float_type}) endif () endforeach() @@ -547,6 +549,11 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS) GenerateNamedObjects("${KERNELDIR}/${${float_char}COMMKERNEL}" "CN" "comm_kernel_l" false "" "" false ${float_type}) GenerateNamedObjects("${KERNELDIR}/${${float_char}COMMKERNEL}" "NC" "comm_kernel_r" false "" "" false ${float_type}) GenerateNamedObjects("${KERNELDIR}/${${float_char}COMMKERNEL}" "CC" "comm_kernel_b" false "" "" false ${float_type}) + else () + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "NN" "comm_kernel_n" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "CN" "comm_kernel_l" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "NC" "comm_kernel_r" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "CC" "comm_kernel_b" false "" "" false ${float_type}) endif () GenerateNamedObjects("${KERNELDIR}/${TRMM_KERNEL}" "TRMMKERNEL;LEFT;NN" "trmm_kernel_LN" false "" "" false ${float_type}) From 3fb6e41beab0530df92e0732a59fcb790f4b2c1f Mon Sep 17 00:00:00 2001 From: Anna Mayne Date: Fri, 19 Dec 2025 10:16:02 +0000 Subject: [PATCH 6/8] Fix the CMake build: updates for new copy functions. --- kernel/CMakeLists.txt | 12 ++++++------ kernel/arm64/KERNEL.ARMV8SVE | 8 -------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 3d420ec1f8..074fd5ef06 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -520,19 +520,19 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS) endif () if (${float_char}COMMNCOPY) - GenerateNamedObjects("${KERNELDIR}/${${float_char}COMMNCOPY}" "${float_type}" "${${float_char}COMMNCOPYOBJ}" false "" "" true ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}COMMNCOPY}" "${float_type}" "comm_ncopy" false "" "" false ${float_type}) elseif (${float_char}GEMMINCOPY) - GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMINCOPY}" "${float_type}" "${${float_char}COMMNCOPYOBJ}" false "" "" true ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMINCOPY}" "${float_type}" "comm_ncopy" false "" "" false ${float_type}) else () - GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMONCOPY}" "${float_type}" "${${float_char}COMMNCOPYOBJ}" false "" "" true ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMONCOPY}" "${float_type}" "comm_ncopy" false "" "" false ${float_type}) endif () if (${float_char}COMMTCOPY) - GenerateNamedObjects("${KERNELDIR}/${${float_char}COMMTCOPY}" "${float_type}" "${${float_char}COMMTCOPYOBJ}" false "" "" true ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}COMMTCOPY}" "${float_type}" "comm_tcopy" false "" "" false ${float_type}) elseif (${float_char}GEMMITCOPY) - GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMITCOPY}" "${float_type}" "${${float_char}COMMTCOPYOBJ}" false "" "" true ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMITCOPY}" "${float_type}" "comm_tcopy" false "" "" false ${float_type}) else () - GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMOTCOPY}" "${float_type}" "${${float_char}COMMTCOPYOBJ}" false "" "" true ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMOTCOPY}" "${float_type}" "comm_tcopy" false "" "" false ${float_type}) endif () if (${float_type} STREQUAL "COMPLEX" OR ${float_type} STREQUAL "ZCOMPLEX") diff --git a/kernel/arm64/KERNEL.ARMV8SVE b/kernel/arm64/KERNEL.ARMV8SVE index 0bc78b1892..8457144b38 100644 --- a/kernel/arm64/KERNEL.ARMV8SVE +++ b/kernel/arm64/KERNEL.ARMV8SVE @@ -163,8 +163,6 @@ SGEMMINCOPYOBJ = sgemm_incopy$(TSUFFIX).$(SUFFIX) SGEMMITCOPYOBJ = sgemm_itcopy$(TSUFFIX).$(SUFFIX) SGEMMONCOPYOBJ = sgemm_oncopy$(TSUFFIX).$(SUFFIX) SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX) -SCOMMNCOPYOBJ = scomm_ncopy$(TSUFFIX).$(SUFFIX) -SCOMMTCOPYOBJ = scomm_tcopy$(TSUFFIX).$(SUFFIX) SGEMM_SMALL_M_PERMIT = gemm_small_kernel_permit_sve.c SGEMM_SMALL_K_NT = sgemm_small_kernel_nt_sve.c @@ -199,8 +197,6 @@ DGEMMINCOPYOBJ = dgemm_incopy$(TSUFFIX).$(SUFFIX) DGEMMITCOPYOBJ = dgemm_itcopy$(TSUFFIX).$(SUFFIX) DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX) DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX) -DCOMMNCOPYOBJ = dcomm_ncopy$(TSUFFIX).$(SUFFIX) -DCOMMTCOPYOBJ = dcomm_tcopy$(TSUFFIX).$(SUFFIX) DGEMM_SMALL_M_PERMIT = gemm_small_kernel_permit_sve.c DGEMM_SMALL_K_NT = dgemm_small_kernel_nt_sve.c @@ -235,8 +231,6 @@ CGEMMINCOPYOBJ = cgemm_incopy$(TSUFFIX).$(SUFFIX) CGEMMITCOPYOBJ = cgemm_itcopy$(TSUFFIX).$(SUFFIX) CGEMMONCOPYOBJ = cgemm_oncopy$(TSUFFIX).$(SUFFIX) CGEMMOTCOPYOBJ = cgemm_otcopy$(TSUFFIX).$(SUFFIX) -CCOMMNCOPYOBJ = ccomm_ncopy$(TSUFFIX).$(SUFFIX) -CCOMMTCOPYOBJ = ccomm_tcopy$(TSUFFIX).$(SUFFIX) CTRMMUNCOPY_M = ztrmm_uncopy_sve_v1.c CTRMMLNCOPY_M = ztrmm_lncopy_sve_v1.c @@ -264,8 +258,6 @@ ZGEMMINCOPYOBJ = zgemm_incopy$(TSUFFIX).$(SUFFIX) ZGEMMITCOPYOBJ = zgemm_itcopy$(TSUFFIX).$(SUFFIX) ZGEMMONCOPYOBJ = zgemm_oncopy$(TSUFFIX).$(SUFFIX) ZGEMMOTCOPYOBJ = zgemm_otcopy$(TSUFFIX).$(SUFFIX) -ZCOMMNCOPYOBJ = zcomm_ncopy$(TSUFFIX).$(SUFFIX) -ZCOMMTCOPYOBJ = zcomm_tcopy$(TSUFFIX).$(SUFFIX) ZTRMMUNCOPY_M = ztrmm_uncopy_sve_v1.c ZTRMMLNCOPY_M = ztrmm_lncopy_sve_v1.c From a1454bfb18460665a99367547c4ec5b4490711d0 Mon Sep 17 00:00:00 2001 From: Anna Mayne Date: Mon, 5 Jan 2026 09:51:00 +0000 Subject: [PATCH 7/8] Added to list of contributions. --- CONTRIBUTORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index b6a1884791..52c4bea2cb 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -272,6 +272,7 @@ In chronological order: * Anna Mayne * [2025-11-19] Update thread throttling profile for SGEMV on NEOVERSEV1 and NEOVERSEV2 + * [2026-01-05] Add interleaving to sgemm and dgemm. Disentangle trmm/symm from gemm. * Fadi Arafeh * [2026-03-05] Accelerate SVE128 SBGEMM/BGEMM From d005d50b0be641e5855e8bbb6ac15427ad98d16a Mon Sep 17 00:00:00 2001 From: Anna Mayne Date: Thu, 24 Sep 2026 08:23:57 +0000 Subject: [PATCH 8/8] Responded to review feedback: removed use of COMM and made SYMM/TRMM specific duplicates. Updated copyright notices to 2026 where I had added them. AI assisted. --- CONTRIBUTORS.md | 2 +- common_c.h | 38 +- common_d.h | 20 +- common_level3.h | 60 +- common_macro.h | 116 ++- common_param.h | 58 +- common_s.h | 20 +- common_z.h | 38 +- driver/level3/symm_k.c | 20 +- driver/level3/trmm_L.c | 24 +- driver/level3/trmm_R.c | 38 +- kernel/CMakeLists.txt | 76 +- kernel/Makefile.L3 | 983 ++++++++++++++------- kernel/arm64/KERNEL.ARMV8SVE | 38 +- kernel/arm64/dgemm_kernel_inter_sve_v2x8.S | 2 +- kernel/arm64/gemm_ncopy_sve_v2x8.c | 2 +- kernel/arm64/gemm_tcopy_sve_v2x8.c | 2 +- kernel/arm64/sgemm_kernel_inter_sve_v2x8.S | 2 +- kernel/setparam-ref.c | 24 +- 19 files changed, 1052 insertions(+), 511 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 52c4bea2cb..90601ccdb8 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -272,7 +272,7 @@ In chronological order: * Anna Mayne * [2025-11-19] Update thread throttling profile for SGEMV on NEOVERSEV1 and NEOVERSEV2 - * [2026-01-05] Add interleaving to sgemm and dgemm. Disentangle trmm/symm from gemm. + * [2026-09-24] Add interleaving to sgemm and dgemm. Disentangle trmm/symm from gemm. * Fadi Arafeh * [2026-03-05] Accelerate SVE128 SBGEMM/BGEMM diff --git a/common_c.h b/common_c.h index 8f4a869990..cca915e87e 100644 --- a/common_c.h +++ b/common_c.h @@ -1,6 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ -/* Copyright 2025 The OpenBLAS Project. */ +/* Copyright 2025-2026 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -101,8 +101,10 @@ #define CGEMM_ITCOPY cgemm_itcopy #endif -#define CCOMM_NCOPY ccomm_ncopy -#define CCOMM_TCOPY ccomm_tcopy +#define CSYMM_INCOPY csymm_incopy +#define CSYMM_ITCOPY csymm_itcopy +#define CTRMM_INCOPY ctrmm_incopy +#define CTRMM_ITCOPY ctrmm_itcopy #define CTRMM_OUNUCOPY ctrmm_ounucopy #define CTRMM_OUNNCOPY ctrmm_ounncopy @@ -168,10 +170,14 @@ #define CGEMM_KERNEL_R cgemm_kernel_r #define CGEMM_KERNEL_B cgemm_kernel_b -#define CCOMM_KERNEL_N ccomm_kernel_n -#define CCOMM_KERNEL_L ccomm_kernel_l -#define CCOMM_KERNEL_R ccomm_kernel_r -#define CCOMM_KERNEL_B ccomm_kernel_b +#define CSYMM_KERNEL_N csymm_kernel_n +#define CSYMM_KERNEL_L csymm_kernel_l +#define CSYMM_KERNEL_R csymm_kernel_r +#define CSYMM_KERNEL_B csymm_kernel_b +#define CTRMM_GEMM_KERNEL_N ctrmm_gemm_kernel_n +#define CTRMM_GEMM_KERNEL_L ctrmm_gemm_kernel_l +#define CTRMM_GEMM_KERNEL_R ctrmm_gemm_kernel_r +#define CTRMM_GEMM_KERNEL_B ctrmm_gemm_kernel_b #define CTRMM_KERNEL_LN ctrmm_kernel_LN #define CTRMM_KERNEL_LT ctrmm_kernel_LT @@ -368,8 +374,10 @@ #define CTRMM_IUTNCOPY gotoblas -> ctrmm_iutncopy #define CTRMM_ILNNCOPY gotoblas -> ctrmm_ilnncopy #define CTRMM_ILTNCOPY gotoblas -> ctrmm_iltncopy -#define CCOMM_NCOPY gotoblas -> ccomm_ncopy -#define CCOMM_TCOPY gotoblas -> ccomm_tcopy +#define CSYMM_INCOPY gotoblas -> csymm_incopy +#define CSYMM_ITCOPY gotoblas -> csymm_itcopy +#define CTRMM_INCOPY gotoblas -> ctrmm_incopy +#define CTRMM_ITCOPY gotoblas -> ctrmm_itcopy #define CTRSM_IUNNCOPY gotoblas -> ctrsm_iunncopy #define CTRSM_IUTNCOPY gotoblas -> ctrsm_iutncopy @@ -383,10 +391,14 @@ #define CGEMM_KERNEL_R gotoblas -> cgemm_kernel_r #define CGEMM_KERNEL_B gotoblas -> cgemm_kernel_b -#define CCOMM_KERNEL_N gotoblas -> ccomm_kernel_n -#define CCOMM_KERNEL_L gotoblas -> ccomm_kernel_l -#define CCOMM_KERNEL_R gotoblas -> ccomm_kernel_r -#define CCOMM_KERNEL_B gotoblas -> ccomm_kernel_b +#define CSYMM_KERNEL_N gotoblas -> csymm_kernel_n +#define CSYMM_KERNEL_L gotoblas -> csymm_kernel_l +#define CSYMM_KERNEL_R gotoblas -> csymm_kernel_r +#define CSYMM_KERNEL_B gotoblas -> csymm_kernel_b +#define CTRMM_GEMM_KERNEL_N gotoblas -> ctrmm_gemm_kernel_n +#define CTRMM_GEMM_KERNEL_L gotoblas -> ctrmm_gemm_kernel_l +#define CTRMM_GEMM_KERNEL_R gotoblas -> ctrmm_gemm_kernel_r +#define CTRMM_GEMM_KERNEL_B gotoblas -> ctrmm_gemm_kernel_b #define CTRMM_KERNEL_LN gotoblas -> ctrmm_kernel_LN #define CTRMM_KERNEL_LT gotoblas -> ctrmm_kernel_LT diff --git a/common_d.h b/common_d.h index 2e1700cae2..c86e060d42 100644 --- a/common_d.h +++ b/common_d.h @@ -1,6 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ -/* Copyright 2025 The OpenBLAS Project. */ +/* Copyright 2025-2026 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -94,8 +94,10 @@ #define DGEMM_ITCOPY dgemm_itcopy #endif -#define DCOMM_NCOPY dcomm_ncopy -#define DCOMM_TCOPY dcomm_tcopy +#define DSYMM_INCOPY dsymm_incopy +#define DSYMM_ITCOPY dsymm_itcopy +#define DTRMM_INCOPY dtrmm_incopy +#define DTRMM_ITCOPY dtrmm_itcopy #define DTRMM_OUNUCOPY dtrmm_ounucopy #define DTRMM_OUNNCOPY dtrmm_ounncopy @@ -157,7 +159,8 @@ #define DGEMM_KERNEL dgemm_kernel #define SME_DGEMM_KERNEL sme_dgemm_kernel -#define DCOMM_KERNEL dcomm_kernel +#define DSYMM_KERNEL dsymm_kernel +#define DTRMM_GEMM_KERNEL dtrmm_gemm_kernel #define DTRMM_KERNEL_LN dtrmm_kernel_LN #define DTRMM_KERNEL_LT dtrmm_kernel_LT @@ -283,8 +286,10 @@ #define DTRMM_IUTNCOPY gotoblas -> dtrmm_iutncopy #define DTRMM_ILNNCOPY gotoblas -> dtrmm_ilnncopy #define DTRMM_ILTNCOPY gotoblas -> dtrmm_iltncopy -#define DCOMM_NCOPY gotoblas -> dcomm_ncopy -#define DCOMM_TCOPY gotoblas -> dcomm_tcopy +#define DSYMM_INCOPY gotoblas -> dsymm_incopy +#define DSYMM_ITCOPY gotoblas -> dsymm_itcopy +#define DTRMM_INCOPY gotoblas -> dtrmm_incopy +#define DTRMM_ITCOPY gotoblas -> dtrmm_itcopy #define DTRSM_IUNNCOPY gotoblas -> dtrsm_iunncopy #define DTRSM_IUTNCOPY gotoblas -> dtrsm_iutncopy @@ -294,7 +299,8 @@ #define DGEMM_BETA gotoblas -> dgemm_beta #define DGEMM_KERNEL gotoblas -> dgemm_kernel #define SME_DGEMM_KERNEL gotoblas -> sme_dgemm_kernel -#define DCOMM_KERNEL gotoblas -> dcomm_kernel +#define DSYMM_KERNEL gotoblas -> dsymm_kernel +#define DTRMM_GEMM_KERNEL gotoblas -> dtrmm_gemm_kernel #define DTRMM_KERNEL_LN gotoblas -> dtrmm_kernel_LN #define DTRMM_KERNEL_LT gotoblas -> dtrmm_kernel_LT diff --git a/common_level3.h b/common_level3.h index aeb516d2b4..a161239091 100644 --- a/common_level3.h +++ b/common_level3.h @@ -1,6 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ -/* Copyright 2025 The OpenBLAS Project. */ +/* Copyright 2025-2026 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -335,8 +335,10 @@ int strmm_ilnucopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX int strmm_ilnncopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); int strmm_iltucopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); int strmm_iltncopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); -int scomm_ncopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, float *b); -int scomm_tcopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, float *b); +int ssymm_incopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, float *b); +int ssymm_itcopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, float *b); +int strmm_incopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, float *b); +int strmm_itcopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, float *b); int strmm_olnucopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); int strmm_olnncopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); int strmm_oltucopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); @@ -354,8 +356,10 @@ int dtrmm_ilnucopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG pos int dtrmm_ilnncopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); int dtrmm_iltucopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); int dtrmm_iltncopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); -int dcomm_tcopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, double *b); -int dcomm_ncopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, double *b); +int dsymm_itcopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, double *b); +int dsymm_incopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, double *b); +int dtrmm_itcopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, double *b); +int dtrmm_incopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, double *b); int dtrmm_olnucopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); int dtrmm_olnncopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); int dtrmm_oltucopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); @@ -390,8 +394,10 @@ int ctrmm_ilnucopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX int ctrmm_ilnncopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); int ctrmm_iltucopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); int ctrmm_iltncopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); -int ccomm_tcopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, float *b); -int ccomm_ncopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, float *b); +int csymm_itcopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, float *b); +int csymm_incopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, float *b); +int ctrmm_itcopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, float *b); +int ctrmm_incopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, float *b); int ctrmm_olnucopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); int ctrmm_olnncopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); int ctrmm_oltucopy(BLASLONG m, BLASLONG n, float *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, float *b); @@ -409,8 +415,10 @@ int ztrmm_ilnucopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG pos int ztrmm_ilnncopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); int ztrmm_iltucopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); int ztrmm_iltncopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); -int zcomm_tcopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, double *b); -int zcomm_ncopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, double *b); +int zsymm_itcopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, double *b); +int zsymm_incopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, double *b); +int ztrmm_itcopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, double *b); +int ztrmm_incopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, double *b); int ztrmm_olnucopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); int ztrmm_olnncopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); int ztrmm_oltucopy(BLASLONG m, BLASLONG n, double *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, double *b); @@ -638,8 +646,10 @@ int bgemm_kernel(BLASLONG, BLASLONG, BLASLONG, bfloat16, bfloat16 *, bfloat16 * int sbgemm_kernel(BLASLONG, BLASLONG, BLASLONG, float, bfloat16 *, bfloat16 *, float *, BLASLONG); int sgemm_kernel(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); int dgemm_kernel(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); -int scomm_kernel(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); -int dcomm_kernel(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); +int ssymm_kernel(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); +int dsymm_kernel(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); +int strmm_gemm_kernel(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); +int dtrmm_gemm_kernel(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); #ifdef QUAD_PRECISION int qgemm_kernel(BLASLONG, BLASLONG, BLASLONG, xidouble *, xidouble *, xidouble *, xdouble *, BLASLONG); @@ -789,15 +799,25 @@ int cgemm3m_kernel(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float int zgemm3m_kernel(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); int xgemm3m_kernel(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG); -int ccomm_kernel_n(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); -int ccomm_kernel_l(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); -int ccomm_kernel_r(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); -int ccomm_kernel_b(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); - -int zcomm_kernel_n(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); -int zcomm_kernel_l(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); -int zcomm_kernel_r(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); -int zcomm_kernel_b(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); +int csymm_kernel_n(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); +int csymm_kernel_l(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); +int csymm_kernel_r(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); +int csymm_kernel_b(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + +int ctrmm_gemm_kernel_n(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); +int ctrmm_gemm_kernel_l(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); +int ctrmm_gemm_kernel_r(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); +int ctrmm_gemm_kernel_b(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + +int zsymm_kernel_n(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); +int zsymm_kernel_l(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); +int zsymm_kernel_r(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); +int zsymm_kernel_b(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + +int ztrmm_gemm_kernel_n(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); +int ztrmm_gemm_kernel_l(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); +int ztrmm_gemm_kernel_r(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); +int ztrmm_gemm_kernel_b(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); int shgemm_nn(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16 *, BLASLONG); int shgemm_nt(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16 *, BLASLONG); diff --git a/common_macro.h b/common_macro.h index 237523927f..9dbe775799 100644 --- a/common_macro.h +++ b/common_macro.h @@ -1,6 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ -/* Copyright 2025 The OpenBLAS Project. */ +/* Copyright 2025-2026 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -99,8 +99,10 @@ #define GEMM_OTCOPY QGEMM_OTCOPY #define GEMM_INCOPY QGEMM_INCOPY #define GEMM_ITCOPY QGEMM_ITCOPY -#define COMM_NCOPY QGEMM_INCOPY -#define COMM_TCOPY QGEMM_ITCOPY +#define SYMM_INCOPY QGEMM_INCOPY +#define SYMM_ITCOPY QGEMM_ITCOPY +#define TRMM_INCOPY QGEMM_INCOPY +#define TRMM_ITCOPY QGEMM_ITCOPY #ifdef UNIT @@ -151,10 +153,14 @@ #define GEMM_KERNEL_R QGEMM_KERNEL #define GEMM_KERNEL_B QGEMM_KERNEL -#define COMM_KERNEL_N QGEMM_KERNEL -#define COMM_KERNEL_L QGEMM_KERNEL -#define COMM_KERNEL_R QGEMM_KERNEL -#define COMM_KERNEL_B QGEMM_KERNEL +#define SYMM_KERNEL_N QGEMM_KERNEL +#define SYMM_KERNEL_L QGEMM_KERNEL +#define SYMM_KERNEL_R QGEMM_KERNEL +#define SYMM_KERNEL_B QGEMM_KERNEL +#define TRMM_GEMM_KERNEL_N QGEMM_KERNEL +#define TRMM_GEMM_KERNEL_L QGEMM_KERNEL +#define TRMM_GEMM_KERNEL_R QGEMM_KERNEL +#define TRMM_GEMM_KERNEL_B QGEMM_KERNEL #define TRMM_KERNEL_LN QTRMM_KERNEL_LN #define TRMM_KERNEL_LT QTRMM_KERNEL_LT @@ -398,8 +404,10 @@ #define GEMM_OTCOPY DGEMM_OTCOPY #define GEMM_INCOPY DGEMM_INCOPY #define GEMM_ITCOPY DGEMM_ITCOPY -#define COMM_NCOPY DCOMM_NCOPY -#define COMM_TCOPY DCOMM_TCOPY +#define SYMM_INCOPY DSYMM_INCOPY +#define SYMM_ITCOPY DSYMM_ITCOPY +#define TRMM_INCOPY DTRMM_INCOPY +#define TRMM_ITCOPY DTRMM_ITCOPY #ifdef UNIT @@ -452,10 +460,14 @@ #define GEMM_KERNEL_R DGEMM_KERNEL #define GEMM_KERNEL_B DGEMM_KERNEL -#define COMM_KERNEL_N DCOMM_KERNEL -#define COMM_KERNEL_L DCOMM_KERNEL -#define COMM_KERNEL_R DCOMM_KERNEL -#define COMM_KERNEL_B DCOMM_KERNEL +#define SYMM_KERNEL_N DSYMM_KERNEL +#define SYMM_KERNEL_L DSYMM_KERNEL +#define SYMM_KERNEL_R DSYMM_KERNEL +#define SYMM_KERNEL_B DSYMM_KERNEL +#define TRMM_GEMM_KERNEL_N DTRMM_GEMM_KERNEL +#define TRMM_GEMM_KERNEL_L DTRMM_GEMM_KERNEL +#define TRMM_GEMM_KERNEL_R DTRMM_GEMM_KERNEL +#define TRMM_GEMM_KERNEL_B DTRMM_GEMM_KERNEL #define TRMM_KERNEL_LN DTRMM_KERNEL_LN #define TRMM_KERNEL_LT DTRMM_KERNEL_LT @@ -836,8 +848,10 @@ #define GEMM_OTCOPY SBGEMM_OTCOPY #define GEMM_INCOPY SBGEMM_INCOPY #define GEMM_ITCOPY SBGEMM_ITCOPY -#define COMM_NCOPY SCOMM_NCOPY -#define COMM_TCOPY SCOMM_TCOPY +#define SYMM_INCOPY SSYMM_INCOPY +#define SYMM_ITCOPY SSYMM_ITCOPY +#define TRMM_INCOPY STRMM_INCOPY +#define TRMM_ITCOPY STRMM_ITCOPY #define SYMM_THREAD_LU SSYMM_THREAD_LU #define SYMM_THREAD_LL SSYMM_THREAD_LL #define SYMM_THREAD_RU SSYMM_THREAD_RU @@ -1121,8 +1135,10 @@ #define GEMM_OTCOPY SGEMM_OTCOPY #define GEMM_INCOPY SGEMM_INCOPY #define GEMM_ITCOPY SGEMM_ITCOPY -#define COMM_NCOPY SCOMM_NCOPY -#define COMM_TCOPY SCOMM_TCOPY +#define SYMM_INCOPY SSYMM_INCOPY +#define SYMM_ITCOPY SSYMM_ITCOPY +#define TRMM_INCOPY STRMM_INCOPY +#define TRMM_ITCOPY STRMM_ITCOPY #ifdef UNIT @@ -1175,10 +1191,14 @@ #define GEMM_KERNEL_R SGEMM_KERNEL #define GEMM_KERNEL_B SGEMM_KERNEL -#define COMM_KERNEL_N SCOMM_KERNEL -#define COMM_KERNEL_L SCOMM_KERNEL -#define COMM_KERNEL_R SCOMM_KERNEL -#define COMM_KERNEL_B SCOMM_KERNEL +#define SYMM_KERNEL_N SSYMM_KERNEL +#define SYMM_KERNEL_L SSYMM_KERNEL +#define SYMM_KERNEL_R SSYMM_KERNEL +#define SYMM_KERNEL_B SSYMM_KERNEL +#define TRMM_GEMM_KERNEL_N STRMM_GEMM_KERNEL +#define TRMM_GEMM_KERNEL_L STRMM_GEMM_KERNEL +#define TRMM_GEMM_KERNEL_R STRMM_GEMM_KERNEL +#define TRMM_GEMM_KERNEL_B STRMM_GEMM_KERNEL #define TRMM_KERNEL_LN STRMM_KERNEL_LN #define TRMM_KERNEL_LT STRMM_KERNEL_LT @@ -1456,8 +1476,10 @@ #define GEMM_OTCOPY XGEMM_OTCOPY #define GEMM_INCOPY XGEMM_INCOPY #define GEMM_ITCOPY XGEMM_ITCOPY -#define COMM_NCOPY XGEMM_INCOPY -#define COMM_TCOPY XGEMM_ITCOPY +#define SYMM_INCOPY XGEMM_INCOPY +#define SYMM_ITCOPY XGEMM_ITCOPY +#define TRMM_INCOPY XGEMM_INCOPY +#define TRMM_ITCOPY XGEMM_ITCOPY #define GEMM3M_ONCOPYB XGEMM3M_ONCOPYB #define GEMM3M_ONCOPYR XGEMM3M_ONCOPYR @@ -1549,10 +1571,14 @@ #define GEMM_KERNEL_R XGEMM_KERNEL_R #define GEMM_KERNEL_B XGEMM_KERNEL_B -#define COMM_KERNEL_N XGEMM_KERNEL_N -#define COMM_KERNEL_L XGEMM_KERNEL_L -#define COMM_KERNEL_R XGEMM_KERNEL_R -#define COMM_KERNEL_B XGEMM_KERNEL_B +#define SYMM_KERNEL_N XGEMM_KERNEL_N +#define SYMM_KERNEL_L XGEMM_KERNEL_L +#define SYMM_KERNEL_R XGEMM_KERNEL_R +#define SYMM_KERNEL_B XGEMM_KERNEL_B +#define TRMM_GEMM_KERNEL_N XGEMM_KERNEL_N +#define TRMM_GEMM_KERNEL_L XGEMM_KERNEL_L +#define TRMM_GEMM_KERNEL_R XGEMM_KERNEL_R +#define TRMM_GEMM_KERNEL_B XGEMM_KERNEL_B #define GEMM3M_KERNEL XGEMM3M_KERNEL @@ -1865,8 +1891,10 @@ #define GEMM_OTCOPY ZGEMM_OTCOPY #define GEMM_INCOPY ZGEMM_INCOPY #define GEMM_ITCOPY ZGEMM_ITCOPY -#define COMM_NCOPY ZCOMM_NCOPY -#define COMM_TCOPY ZCOMM_TCOPY +#define SYMM_INCOPY ZSYMM_INCOPY +#define SYMM_ITCOPY ZSYMM_ITCOPY +#define TRMM_INCOPY ZTRMM_INCOPY +#define TRMM_ITCOPY ZTRMM_ITCOPY #define GEMM3M_ONCOPYB ZGEMM3M_ONCOPYB #define GEMM3M_ONCOPYR ZGEMM3M_ONCOPYR @@ -1960,10 +1988,14 @@ #define GEMM_KERNEL_R ZGEMM_KERNEL_R #define GEMM_KERNEL_B ZGEMM_KERNEL_B -#define COMM_KERNEL_N ZCOMM_KERNEL_N -#define COMM_KERNEL_L ZCOMM_KERNEL_L -#define COMM_KERNEL_R ZCOMM_KERNEL_R -#define COMM_KERNEL_B ZCOMM_KERNEL_B +#define SYMM_KERNEL_N ZSYMM_KERNEL_N +#define SYMM_KERNEL_L ZSYMM_KERNEL_L +#define SYMM_KERNEL_R ZSYMM_KERNEL_R +#define SYMM_KERNEL_B ZSYMM_KERNEL_B +#define TRMM_GEMM_KERNEL_N ZTRMM_GEMM_KERNEL_N +#define TRMM_GEMM_KERNEL_L ZTRMM_GEMM_KERNEL_L +#define TRMM_GEMM_KERNEL_R ZTRMM_GEMM_KERNEL_R +#define TRMM_GEMM_KERNEL_B ZTRMM_GEMM_KERNEL_B #define GEMM3M_KERNEL ZGEMM3M_KERNEL @@ -2339,8 +2371,10 @@ #define GEMM_OTCOPY CGEMM_OTCOPY #define GEMM_INCOPY CGEMM_INCOPY #define GEMM_ITCOPY CGEMM_ITCOPY -#define COMM_NCOPY CCOMM_NCOPY -#define COMM_TCOPY CCOMM_TCOPY +#define SYMM_INCOPY CSYMM_INCOPY +#define SYMM_ITCOPY CSYMM_ITCOPY +#define TRMM_INCOPY CTRMM_INCOPY +#define TRMM_ITCOPY CTRMM_ITCOPY #define GEMM3M_ONCOPYB CGEMM3M_ONCOPYB #define GEMM3M_ONCOPYR CGEMM3M_ONCOPYR @@ -2434,10 +2468,14 @@ #define GEMM_KERNEL_R CGEMM_KERNEL_R #define GEMM_KERNEL_B CGEMM_KERNEL_B -#define COMM_KERNEL_N CCOMM_KERNEL_N -#define COMM_KERNEL_L CCOMM_KERNEL_L -#define COMM_KERNEL_R CCOMM_KERNEL_R -#define COMM_KERNEL_B CCOMM_KERNEL_B +#define SYMM_KERNEL_N CSYMM_KERNEL_N +#define SYMM_KERNEL_L CSYMM_KERNEL_L +#define SYMM_KERNEL_R CSYMM_KERNEL_R +#define SYMM_KERNEL_B CSYMM_KERNEL_B +#define TRMM_GEMM_KERNEL_N CTRMM_GEMM_KERNEL_N +#define TRMM_GEMM_KERNEL_L CTRMM_GEMM_KERNEL_L +#define TRMM_GEMM_KERNEL_R CTRMM_GEMM_KERNEL_R +#define TRMM_GEMM_KERNEL_B CTRMM_GEMM_KERNEL_B #define GEMM3M_KERNEL CGEMM3M_KERNEL diff --git a/common_param.h b/common_param.h index d0dc343c53..ce04ca6aa1 100644 --- a/common_param.h +++ b/common_param.h @@ -1,6 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ -/* Copyright 2023, 2025 The OpenBLAS Project. */ +/* Copyright 2023, 2025-2026 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -282,7 +282,8 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*sgemm_kernel )(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); int (*sgemm_beta )(BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); - int (*scomm_kernel )(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); + int (*ssymm_kernel )(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); + int (*strmm_gemm_kernel)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); int (*sgemm_incopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*sgemm_itcopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); @@ -340,8 +341,10 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*strmm_ilnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*strmm_iltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*strmm_iltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); - int (*scomm_ncopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); - int (*scomm_tcopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); + int (*ssymm_incopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); + int (*ssymm_itcopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); + int (*strmm_incopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); + int (*strmm_itcopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*strmm_ounucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*strmm_ounncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*strmm_outucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); @@ -409,7 +412,8 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL #endif int (*dgemm_kernel )(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); int (*dgemm_beta )(BLASLONG, BLASLONG, BLASLONG, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG); - int (*dcomm_kernel )(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); + int (*dsymm_kernel )(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); + int (*dtrmm_gemm_kernel)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); int (*dgemm_incopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*dgemm_itcopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); @@ -467,8 +471,10 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*dtrmm_ilnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*dtrmm_iltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*dtrmm_iltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); - int (*dcomm_ncopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); - int (*dcomm_tcopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); + int (*dsymm_incopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); + int (*dsymm_itcopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); + int (*dtrmm_incopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); + int (*dtrmm_itcopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*dtrmm_ounucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*dtrmm_ounncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*dtrmm_outucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); @@ -634,10 +640,15 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*cgemm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); int (*cgemm_beta )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); - int (*ccomm_kernel_n )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); - int (*ccomm_kernel_l )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); - int (*ccomm_kernel_r )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); - int (*ccomm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*csymm_kernel_n )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*csymm_kernel_l )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*csymm_kernel_r )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*csymm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + + int (*ctrmm_gemm_kernel_n)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*ctrmm_gemm_kernel_l)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*ctrmm_gemm_kernel_r)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*ctrmm_gemm_kernel_b)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); int (*cgemm_incopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*cgemm_itcopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); @@ -731,8 +742,10 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*ctrmm_ilnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*ctrmm_iltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*ctrmm_iltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); - int (*ccomm_ncopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); - int (*ccomm_tcopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); + int (*csymm_incopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); + int (*csymm_itcopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); + int (*ctrmm_incopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); + int (*ctrmm_itcopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*ctrmm_ounucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*ctrmm_ounncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*ctrmm_outucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); @@ -855,10 +868,15 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*zgemm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); int (*zgemm_beta )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG); - int (*zcomm_kernel_n )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); - int (*zcomm_kernel_l )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); - int (*zcomm_kernel_r )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); - int (*zcomm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*zsymm_kernel_n )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*zsymm_kernel_l )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*zsymm_kernel_r )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*zsymm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + + int (*ztrmm_gemm_kernel_n)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*ztrmm_gemm_kernel_l)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*ztrmm_gemm_kernel_r)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*ztrmm_gemm_kernel_b)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); int (*zgemm_incopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*zgemm_itcopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); @@ -952,8 +970,10 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*ztrmm_ilnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*ztrmm_iltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*ztrmm_iltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); - int (*zcomm_ncopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); - int (*zcomm_tcopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); + int (*zsymm_incopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); + int (*zsymm_itcopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); + int (*ztrmm_incopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); + int (*ztrmm_itcopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*ztrmm_ounucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*ztrmm_ounncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*ztrmm_outucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); diff --git a/common_s.h b/common_s.h index fbe3705e68..c9f79b71ab 100644 --- a/common_s.h +++ b/common_s.h @@ -1,6 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ -/* Copyright 2025 The OpenBLAS Project. */ +/* Copyright 2025-2026 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -116,8 +116,10 @@ #endif #define SME_SGEMM_KERNEL sme_sgemm_kernel -#define SCOMM_NCOPY scomm_ncopy -#define SCOMM_TCOPY scomm_tcopy +#define SSYMM_INCOPY ssymm_incopy +#define SSYMM_ITCOPY ssymm_itcopy +#define STRMM_INCOPY strmm_incopy +#define STRMM_ITCOPY strmm_itcopy #define STRMM_OUNUCOPY strmm_ounucopy #define STRMM_OUNNCOPY strmm_ounncopy @@ -178,7 +180,8 @@ #define SGEMM_BETA sgemm_beta #define SGEMM_KERNEL sgemm_kernel -#define SCOMM_KERNEL scomm_kernel +#define SSYMM_KERNEL ssymm_kernel +#define STRMM_GEMM_KERNEL strmm_gemm_kernel #define STRMM_KERNEL_LN strmm_kernel_LN #define STRMM_KERNEL_LT strmm_kernel_LT @@ -331,8 +334,10 @@ #define STRMM_IUTNCOPY gotoblas -> strmm_iutncopy #define STRMM_ILNNCOPY gotoblas -> strmm_ilnncopy #define STRMM_ILTNCOPY gotoblas -> strmm_iltncopy -#define SCOMM_NCOPY gotoblas -> scomm_ncopy -#define SCOMM_TCOPY gotoblas -> scomm_tcopy +#define SSYMM_INCOPY gotoblas -> ssymm_incopy +#define SSYMM_ITCOPY gotoblas -> ssymm_itcopy +#define STRMM_INCOPY gotoblas -> strmm_incopy +#define STRMM_ITCOPY gotoblas -> strmm_itcopy #define STRSM_IUNNCOPY gotoblas -> strsm_iunncopy #define STRSM_IUTNCOPY gotoblas -> strsm_iutncopy @@ -341,7 +346,8 @@ #define SGEMM_BETA gotoblas -> sgemm_beta #define SGEMM_KERNEL gotoblas -> sgemm_kernel -#define SCOMM_KERNEL gotoblas -> scomm_kernel +#define SSYMM_KERNEL gotoblas -> ssymm_kernel +#define STRMM_GEMM_KERNEL gotoblas -> strmm_gemm_kernel #define STRMM_KERNEL_LN gotoblas -> strmm_kernel_LN #define STRMM_KERNEL_LT gotoblas -> strmm_kernel_LT diff --git a/common_z.h b/common_z.h index 9e899394fa..240f93d530 100644 --- a/common_z.h +++ b/common_z.h @@ -1,6 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ -/* Copyright 2025 The OpenBLAS Project. */ +/* Copyright 2025-2026 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -101,8 +101,10 @@ #define ZGEMM_ITCOPY zgemm_itcopy #endif -#define ZCOMM_NCOPY zcomm_ncopy -#define ZCOMM_TCOPY zcomm_tcopy +#define ZSYMM_INCOPY zsymm_incopy +#define ZSYMM_ITCOPY zsymm_itcopy +#define ZTRMM_INCOPY ztrmm_incopy +#define ZTRMM_ITCOPY ztrmm_itcopy #define ZTRMM_OUNUCOPY ztrmm_ounucopy #define ZTRMM_OUNNCOPY ztrmm_ounncopy @@ -168,10 +170,14 @@ #define ZGEMM_KERNEL_R zgemm_kernel_r #define ZGEMM_KERNEL_B zgemm_kernel_b -#define ZCOMM_KERNEL_N zcomm_kernel_n -#define ZCOMM_KERNEL_L zcomm_kernel_l -#define ZCOMM_KERNEL_R zcomm_kernel_r -#define ZCOMM_KERNEL_B zcomm_kernel_b +#define ZSYMM_KERNEL_N zsymm_kernel_n +#define ZSYMM_KERNEL_L zsymm_kernel_l +#define ZSYMM_KERNEL_R zsymm_kernel_r +#define ZSYMM_KERNEL_B zsymm_kernel_b +#define ZTRMM_GEMM_KERNEL_N ztrmm_gemm_kernel_n +#define ZTRMM_GEMM_KERNEL_L ztrmm_gemm_kernel_l +#define ZTRMM_GEMM_KERNEL_R ztrmm_gemm_kernel_r +#define ZTRMM_GEMM_KERNEL_B ztrmm_gemm_kernel_b #define ZTRMM_KERNEL_LN ztrmm_kernel_LN #define ZTRMM_KERNEL_LT ztrmm_kernel_LT @@ -368,8 +374,10 @@ #define ZTRMM_IUTNCOPY gotoblas -> ztrmm_iutncopy #define ZTRMM_ILNNCOPY gotoblas -> ztrmm_ilnncopy #define ZTRMM_ILTNCOPY gotoblas -> ztrmm_iltncopy -#define ZCOMM_NCOPY gotoblas -> zcomm_ncopy -#define ZCOMM_TCOPY gotoblas -> zcomm_tcopy +#define ZSYMM_INCOPY gotoblas -> zsymm_incopy +#define ZSYMM_ITCOPY gotoblas -> zsymm_itcopy +#define ZTRMM_INCOPY gotoblas -> ztrmm_incopy +#define ZTRMM_ITCOPY gotoblas -> ztrmm_itcopy #define ZTRSM_IUNNCOPY gotoblas -> ztrsm_iunncopy #define ZTRSM_IUTNCOPY gotoblas -> ztrsm_iutncopy @@ -383,10 +391,14 @@ #define ZGEMM_KERNEL_R gotoblas -> zgemm_kernel_r #define ZGEMM_KERNEL_B gotoblas -> zgemm_kernel_b -#define ZCOMM_KERNEL_N gotoblas -> zcomm_kernel_n -#define ZCOMM_KERNEL_L gotoblas -> zcomm_kernel_l -#define ZCOMM_KERNEL_R gotoblas -> zcomm_kernel_r -#define ZCOMM_KERNEL_B gotoblas -> zcomm_kernel_b +#define ZSYMM_KERNEL_N gotoblas -> zsymm_kernel_n +#define ZSYMM_KERNEL_L gotoblas -> zsymm_kernel_l +#define ZSYMM_KERNEL_R gotoblas -> zsymm_kernel_r +#define ZSYMM_KERNEL_B gotoblas -> zsymm_kernel_b +#define ZTRMM_GEMM_KERNEL_N gotoblas -> ztrmm_gemm_kernel_n +#define ZTRMM_GEMM_KERNEL_L gotoblas -> ztrmm_gemm_kernel_l +#define ZTRMM_GEMM_KERNEL_R gotoblas -> ztrmm_gemm_kernel_r +#define ZTRMM_GEMM_KERNEL_B gotoblas -> ztrmm_gemm_kernel_b #define ZTRMM_KERNEL_LN gotoblas -> ztrmm_kernel_LN #define ZTRMM_KERNEL_LT gotoblas -> ztrmm_kernel_LT diff --git a/driver/level3/symm_k.c b/driver/level3/symm_k.c index db2bc172f4..1d96c70357 100644 --- a/driver/level3/symm_k.c +++ b/driver/level3/symm_k.c @@ -1,6 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ -/* Copyright 2025 The OpenBLAS Project. */ +/* Copyright 2025-2026 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -58,35 +58,30 @@ #endif #endif - #ifndef ICOPY_OPERATION #if defined(NN) || defined(NT) || defined(NC) || defined(NR) || \ defined(RN) || defined(RT) || defined(RC) || defined(RR) -#define ICOPY_OPERATION(M, N, A, LDA, X, Y, BUFFER) COMM_TCOPY(M, N, (IFLOAT *)(A) + ((Y) + (X) * (LDA)) * COMPSIZE, LDA, BUFFER); +#define ICOPY_OPERATION(M, N, A, LDA, X, Y, BUFFER) SYMM_ITCOPY(M, N, (IFLOAT *)(A) + ((Y) + (X) * (LDA)) * COMPSIZE, LDA, BUFFER); #else -#define ICOPY_OPERATION(M, N, A, LDA, X, Y, BUFFER) COMM_NCOPY(M, N, (IFLOAT *)(A) + ((X) + (Y) * (LDA)) * COMPSIZE, LDA, BUFFER); +#define ICOPY_OPERATION(M, N, A, LDA, X, Y, BUFFER) SYMM_INCOPY(M, N, (IFLOAT *)(A) + ((X) + (Y) * (LDA)) * COMPSIZE, LDA, BUFFER); #endif #endif - - - //#ifndef KERNEL_FUNC #if defined(NN) || defined(NT) || defined(TN) || defined(TT) -#define KERNEL_FUNC COMM_KERNEL_N +#define KERNEL_FUNC SYMM_KERNEL_N #endif #if defined(CN) || defined(CT) || defined(RN) || defined(RT) -#define KERNEL_FUNC COMM_KERNEL_L +#define KERNEL_FUNC SYMM_KERNEL_L #endif #if defined(NC) || defined(TC) || defined(NR) || defined(TR) -#define KERNEL_FUNC COMM_KERNEL_R +#define KERNEL_FUNC SYMM_KERNEL_R #endif #if defined(CC) || defined(CR) || defined(RC) || defined(RR) -#define KERNEL_FUNC COMM_KERNEL_B +#define KERNEL_FUNC SYMM_KERNEL_B #endif //#endif - #ifndef RSIDE #define K args -> m #ifndef LOWER @@ -103,7 +98,6 @@ #endif #endif - #ifdef THREADED_LEVEL3 #include "level3_thread.c" #else diff --git a/driver/level3/trmm_L.c b/driver/level3/trmm_L.c index 7927532531..5c908d522e 100644 --- a/driver/level3/trmm_L.c +++ b/driver/level3/trmm_L.c @@ -1,6 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ -/* Copyright 2025 The OpenBLAS Project. */ +/* Copyright 2025-2026 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -44,11 +44,11 @@ const static FLOAT dp1 = 1.; #ifdef CONJ -#define COMM_KERNEL COMM_KERNEL_L +#define TRMM_GEMM_KERNEL TRMM_GEMM_KERNEL_L #define TRMM_KERNEL_N TRMM_KERNEL_LR #define TRMM_KERNEL_T TRMM_KERNEL_LC #else -#define COMM_KERNEL COMM_KERNEL_N +#define TRMM_GEMM_KERNEL TRMM_GEMM_KERNEL_N #define TRMM_KERNEL_N TRMM_KERNEL_LN #define TRMM_KERNEL_T TRMM_KERNEL_LT #endif @@ -207,9 +207,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO START_RPCC(); #ifndef TRANSA - COMM_TCOPY(min_l, min_i, a + (ls * lda) * COMPSIZE, lda, sa); + TRMM_ITCOPY(min_l, min_i, a + (ls * lda) * COMPSIZE, lda, sa); #else - COMM_NCOPY(min_l, min_i, a + (ls ) * COMPSIZE, lda, sa); + TRMM_INCOPY(min_l, min_i, a + (ls ) * COMPSIZE, lda, sa); #endif STOP_RPCC(innercost); @@ -232,7 +232,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO START_RPCC(); - COMM_KERNEL(min_i, min_jj, min_l, dp1, + TRMM_GEMM_KERNEL(min_i, min_jj, min_l, dp1, #ifdef COMPLEX ZERO, #endif @@ -252,16 +252,16 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO START_RPCC(); #ifndef TRANSA - COMM_TCOPY(min_l, min_i, a + (is + ls * lda) * COMPSIZE, lda, sa); + TRMM_ITCOPY(min_l, min_i, a + (is + ls * lda) * COMPSIZE, lda, sa); #else - COMM_NCOPY(min_l, min_i, a + (ls + is * lda) * COMPSIZE, lda, sa); + TRMM_INCOPY(min_l, min_i, a + (ls + is * lda) * COMPSIZE, lda, sa); #endif STOP_RPCC(innercost); START_RPCC(); - COMM_KERNEL(min_i, min_j, min_l, dp1, + TRMM_GEMM_KERNEL(min_i, min_j, min_l, dp1, #ifdef COMPLEX ZERO, #endif @@ -467,16 +467,16 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO START_RPCC(); #ifndef TRANSA - COMM_TCOPY(min_l, min_i, a + (is + (ls - min_l) * lda) * COMPSIZE, lda, sa); + TRMM_ITCOPY(min_l, min_i, a + (is + (ls - min_l) * lda) * COMPSIZE, lda, sa); #else - COMM_NCOPY(min_l, min_i, a + ((ls - min_l) + is * lda) * COMPSIZE, lda, sa); + TRMM_INCOPY(min_l, min_i, a + ((ls - min_l) + is * lda) * COMPSIZE, lda, sa); #endif STOP_RPCC(innercost); START_RPCC(); - COMM_KERNEL(min_i, min_j, min_l, dp1, + TRMM_GEMM_KERNEL(min_i, min_j, min_l, dp1, #ifdef COMPLEX ZERO, #endif diff --git a/driver/level3/trmm_R.c b/driver/level3/trmm_R.c index 9c9975339b..522ff4984e 100644 --- a/driver/level3/trmm_R.c +++ b/driver/level3/trmm_R.c @@ -1,6 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ -/* Copyright 2025 The OpenBLAS Project. */ +/* Copyright 2025-2026 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -44,11 +44,11 @@ const static FLOAT dp1 = 1.; #ifdef CONJ -#define COMM_KERNEL COMM_KERNEL_R +#define TRMM_GEMM_KERNEL TRMM_GEMM_KERNEL_R #define TRMM_KERNEL_N TRMM_KERNEL_RR #define TRMM_KERNEL_T TRMM_KERNEL_RC #else -#define COMM_KERNEL COMM_KERNEL_N +#define TRMM_GEMM_KERNEL TRMM_GEMM_KERNEL_N #define TRMM_KERNEL_N TRMM_KERNEL_RN #define TRMM_KERNEL_T TRMM_KERNEL_RT #endif @@ -119,7 +119,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO min_i = m; if (min_i > GEMM_P) min_i = GEMM_P; - COMM_TCOPY(min_l, min_i, b + (ls * ldb) * COMPSIZE, ldb, sa); + TRMM_ITCOPY(min_l, min_i, b + (ls * ldb) * COMPSIZE, ldb, sa); for(jjs = 0; jjs < ls - js; jjs += min_jj){ min_jj = ls - js - jjs; @@ -137,7 +137,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO GEMM_OTCOPY(min_l, min_jj, a + ((js + jjs) + ls * lda) * COMPSIZE, lda, sb + min_l * jjs * COMPSIZE); #endif - COMM_KERNEL(min_i, min_jj, min_l, dp1, + TRMM_GEMM_KERNEL(min_i, min_jj, min_l, dp1, #ifdef COMPLEX ZERO, #endif @@ -174,9 +174,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO min_i = m - is; if (min_i > GEMM_P) min_i = GEMM_P; - COMM_TCOPY(min_l, min_i, b + (is + ls * ldb) * COMPSIZE, ldb, sa); + TRMM_ITCOPY(min_l, min_i, b + (is + ls * ldb) * COMPSIZE, ldb, sa); - COMM_KERNEL(min_i, ls - js, min_l, dp1, + TRMM_GEMM_KERNEL(min_i, ls - js, min_l, dp1, #ifdef COMPLEX ZERO, #endif @@ -200,7 +200,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO min_i = m; if (min_i > GEMM_P) min_i = GEMM_P; - COMM_TCOPY(min_l, min_i, b + (ls * ldb) * COMPSIZE, ldb, sa); + TRMM_ITCOPY(min_l, min_i, b + (ls * ldb) * COMPSIZE, ldb, sa); for(jjs = js; jjs < js + min_j; jjs += min_jj){ min_jj = min_j + js - jjs; @@ -218,7 +218,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO GEMM_OTCOPY(min_l, min_jj, a + (jjs + ls * lda) * COMPSIZE, lda, sb + min_l * (jjs - js) * COMPSIZE); #endif - COMM_KERNEL(min_i, min_jj, min_l, dp1, + TRMM_GEMM_KERNEL(min_i, min_jj, min_l, dp1, #ifdef COMPLEX ZERO, #endif @@ -230,9 +230,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO min_i = m - is; if (min_i > GEMM_P) min_i = GEMM_P; - COMM_TCOPY(min_l, min_i, b + (is + ls * ldb) * COMPSIZE, ldb, sa); + TRMM_ITCOPY(min_l, min_i, b + (is + ls * ldb) * COMPSIZE, ldb, sa); - COMM_KERNEL(min_i, min_j, min_l, dp1, + TRMM_GEMM_KERNEL(min_i, min_j, min_l, dp1, #ifdef COMPLEX ZERO, #endif @@ -255,7 +255,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO min_i = m; if (min_i > GEMM_P) min_i = GEMM_P; - COMM_TCOPY(min_l, min_i, b + (ls * ldb) * COMPSIZE, ldb, sa); + TRMM_ITCOPY(min_l, min_i, b + (ls * ldb) * COMPSIZE, ldb, sa); for(jjs = 0; jjs < min_l; jjs += min_jj){ min_jj = min_l - jjs; @@ -300,7 +300,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO sb + min_l * (min_l + jjs) * COMPSIZE); #endif - COMM_KERNEL(min_i, min_jj, min_l, dp1, + TRMM_GEMM_KERNEL(min_i, min_jj, min_l, dp1, #ifdef COMPLEX ZERO, #endif @@ -313,7 +313,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO min_i = m - is; if (min_i > GEMM_P) min_i = GEMM_P; - COMM_TCOPY(min_l, min_i, b + (is + ls * ldb) * COMPSIZE, ldb, sa); + TRMM_ITCOPY(min_l, min_i, b + (is + ls * ldb) * COMPSIZE, ldb, sa); TRMM_KERNEL_N(min_i, min_l, min_l, dp1, #ifdef COMPLEX @@ -324,7 +324,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO b + (is + ls * ldb) * COMPSIZE, ldb, 0); if (js - ls - min_l > 0) { - COMM_KERNEL(min_i, js - ls - min_l, min_l, dp1, + TRMM_GEMM_KERNEL(min_i, js - ls - min_l, min_l, dp1, #ifdef COMPLEX ZERO, #endif @@ -341,7 +341,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO min_i = m; if (min_i > GEMM_P) min_i = GEMM_P; - COMM_TCOPY(min_l, min_i, b + (ls * ldb) * COMPSIZE, ldb, sa); + TRMM_ITCOPY(min_l, min_i, b + (ls * ldb) * COMPSIZE, ldb, sa); for(jjs = js; jjs < js + min_j; jjs += min_jj){ min_jj = min_j + js - jjs; @@ -359,7 +359,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO GEMM_OTCOPY(min_l, min_jj, a + ((jjs - min_j) + ls * lda) * COMPSIZE, lda, sb + min_l * (jjs - js) * COMPSIZE); #endif - COMM_KERNEL(min_i, min_jj, min_l, dp1, + TRMM_GEMM_KERNEL(min_i, min_jj, min_l, dp1, #ifdef COMPLEX ZERO, #endif @@ -371,9 +371,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO min_i = m - is; if (min_i > GEMM_P) min_i = GEMM_P; - COMM_TCOPY(min_l, min_i, b + (is + ls * ldb) * COMPSIZE, ldb, sa); + TRMM_ITCOPY(min_l, min_i, b + (is + ls * ldb) * COMPSIZE, ldb, sa); - COMM_KERNEL(min_i, min_j, min_l, dp1, + TRMM_GEMM_KERNEL(min_i, min_j, min_l, dp1, #ifdef COMPLEX ZERO, #endif diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 074fd5ef06..6ae74322dd 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2025, The OpenBLAS Project +# Copyright (c) 2025-2026, The OpenBLAS Project # All rights reserved. # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -339,10 +339,15 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS) foreach (float_type SINGLE DOUBLE) string(SUBSTRING ${float_type} 0 1 float_char) GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "" "gemm_kernel" false "" "" false ${float_type}) - if (${float_char}COMMKERNEL) - GenerateNamedObjects("${KERNELDIR}/${${float_char}COMMKERNEL}" "" "comm_kernel" false "" "" false ${float_type}) + if (${float_char}SYMMKERNEL) + GenerateNamedObjects("${KERNELDIR}/${${float_char}SYMMKERNEL}" "" "symm_kernel" false "" "" false ${float_type}) else () - GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "" "comm_kernel" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "" "symm_kernel" false "" "" false ${float_type}) + endif () + if (${float_char}TRMMGEMMKERNEL) + GenerateNamedObjects("${KERNELDIR}/${${float_char}TRMMGEMMKERNEL}" "" "trmm_gemm_kernel" false "" "" false ${float_type}) + else () + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "" "trmm_gemm_kernel" false "" "" false ${float_type}) endif () endforeach() @@ -513,26 +518,40 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS) if (USE_TRMM) set(TRMM_KERNEL "${${float_char}TRMMKERNEL}") - elseif (${float_char}COMMKERNEL) - set(TRMM_KERNEL "${${float_char}COMMKERNEL}") + elseif (${float_char}TRMMGEMMKERNEL) + set(TRMM_KERNEL "${${float_char}TRMMGEMMKERNEL}") else () set(TRMM_KERNEL "${${float_char}GEMMKERNEL}") endif () - if (${float_char}COMMNCOPY) - GenerateNamedObjects("${KERNELDIR}/${${float_char}COMMNCOPY}" "${float_type}" "comm_ncopy" false "" "" false ${float_type}) + if (${float_char}SYMMINCOPY) + GenerateNamedObjects("${KERNELDIR}/${${float_char}SYMMINCOPY}" "${float_type}" "symm_incopy" false "" "" false ${float_type}) + elseif (${float_char}GEMMINCOPY) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMINCOPY}" "${float_type}" "symm_incopy" false "" "" false ${float_type}) + else () + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMONCOPY}" "${float_type}" "symm_incopy" false "" "" false ${float_type}) + endif () + if (${float_char}TRMMINCOPY) + GenerateNamedObjects("${KERNELDIR}/${${float_char}TRMMINCOPY}" "${float_type}" "trmm_incopy" false "" "" false ${float_type}) elseif (${float_char}GEMMINCOPY) - GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMINCOPY}" "${float_type}" "comm_ncopy" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMINCOPY}" "${float_type}" "trmm_incopy" false "" "" false ${float_type}) else () - GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMONCOPY}" "${float_type}" "comm_ncopy" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMONCOPY}" "${float_type}" "trmm_incopy" false "" "" false ${float_type}) endif () - if (${float_char}COMMTCOPY) - GenerateNamedObjects("${KERNELDIR}/${${float_char}COMMTCOPY}" "${float_type}" "comm_tcopy" false "" "" false ${float_type}) + if (${float_char}SYMMITCOPY) + GenerateNamedObjects("${KERNELDIR}/${${float_char}SYMMITCOPY}" "${float_type}" "symm_itcopy" false "" "" false ${float_type}) elseif (${float_char}GEMMITCOPY) - GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMITCOPY}" "${float_type}" "comm_tcopy" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMITCOPY}" "${float_type}" "symm_itcopy" false "" "" false ${float_type}) else () - GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMOTCOPY}" "${float_type}" "comm_tcopy" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMOTCOPY}" "${float_type}" "symm_itcopy" false "" "" false ${float_type}) + endif () + if (${float_char}TRMMITCOPY) + GenerateNamedObjects("${KERNELDIR}/${${float_char}TRMMITCOPY}" "${float_type}" "trmm_itcopy" false "" "" false ${float_type}) + elseif (${float_char}GEMMITCOPY) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMITCOPY}" "${float_type}" "trmm_itcopy" false "" "" false ${float_type}) + else () + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMOTCOPY}" "${float_type}" "trmm_itcopy" false "" "" false ${float_type}) endif () if (${float_type} STREQUAL "COMPLEX" OR ${float_type} STREQUAL "ZCOMPLEX") @@ -544,16 +563,27 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS) GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "NC" "gemm_kernel_r" false "" "" false ${float_type}) GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "CC" "gemm_kernel_b" false "" "" false ${float_type}) - if (${float_char}COMMKERNEL) - GenerateNamedObjects("${KERNELDIR}/${${float_char}COMMKERNEL}" "NN" "comm_kernel_n" false "" "" false ${float_type}) - GenerateNamedObjects("${KERNELDIR}/${${float_char}COMMKERNEL}" "CN" "comm_kernel_l" false "" "" false ${float_type}) - GenerateNamedObjects("${KERNELDIR}/${${float_char}COMMKERNEL}" "NC" "comm_kernel_r" false "" "" false ${float_type}) - GenerateNamedObjects("${KERNELDIR}/${${float_char}COMMKERNEL}" "CC" "comm_kernel_b" false "" "" false ${float_type}) + if (${float_char}SYMMKERNEL) + GenerateNamedObjects("${KERNELDIR}/${${float_char}SYMMKERNEL}" "NN" "symm_kernel_n" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}SYMMKERNEL}" "CN" "symm_kernel_l" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}SYMMKERNEL}" "NC" "symm_kernel_r" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}SYMMKERNEL}" "CC" "symm_kernel_b" false "" "" false ${float_type}) + else () + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "NN" "symm_kernel_n" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "CN" "symm_kernel_l" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "NC" "symm_kernel_r" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "CC" "symm_kernel_b" false "" "" false ${float_type}) + endif () + if (${float_char}TRMMGEMMKERNEL) + GenerateNamedObjects("${KERNELDIR}/${${float_char}TRMMGEMMKERNEL}" "NN" "trmm_gemm_kernel_n" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}TRMMGEMMKERNEL}" "CN" "trmm_gemm_kernel_l" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}TRMMGEMMKERNEL}" "NC" "trmm_gemm_kernel_r" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}TRMMGEMMKERNEL}" "CC" "trmm_gemm_kernel_b" false "" "" false ${float_type}) else () - GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "NN" "comm_kernel_n" false "" "" false ${float_type}) - GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "CN" "comm_kernel_l" false "" "" false ${float_type}) - GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "NC" "comm_kernel_r" false "" "" false ${float_type}) - GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "CC" "comm_kernel_b" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "NN" "trmm_gemm_kernel_n" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "CN" "trmm_gemm_kernel_l" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "NC" "trmm_gemm_kernel_r" false "" "" false ${float_type}) + GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "CC" "trmm_gemm_kernel_b" false "" "" false ${float_type}) endif () GenerateNamedObjects("${KERNELDIR}/${TRMM_KERNEL}" "TRMMKERNEL;LEFT;NN" "trmm_kernel_LN" false "" "" false ${float_type}) diff --git a/kernel/Makefile.L3 b/kernel/Makefile.L3 index dc4c749feb..fe77d8c3e5 100644 --- a/kernel/Makefile.L3 +++ b/kernel/Makefile.L3 @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2025, The OpenBLAS Project +# Copyright (c) 2025-2026, The OpenBLAS Project # All rights reserved. # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -249,7 +249,8 @@ endif ifneq "$(or $(BUILD_SINGLE),$(BUILD_DOUBLE),$(BUILD_COMPLEX))" "" SKERNELOBJS += \ sgemm_kernel$(TSUFFIX).$(SUFFIX) \ - scomm_kernel$(TSUFFIX).$(SUFFIX) \ + ssymm_kernel$(TSUFFIX).$(SUFFIX) \ + strmm_gemm_kernel$(TSUFFIX).$(SUFFIX) \ sgemm_beta$(TSUFFIX).$(SUFFIX) \ $(SGEMMINCOPYOBJ) $(SGEMMITCOPYOBJ) \ $(SGEMMONCOPYOBJ) $(SGEMMOTCOPYOBJ) @@ -316,7 +317,8 @@ ifneq "$(or $(BUILD_DOUBLE),$(BUILD_COMPLEX16))" "" DKERNELOBJS += \ dgemm_beta$(TSUFFIX).$(SUFFIX) \ dgemm_kernel$(TSUFFIX).$(SUFFIX) \ - dcomm_kernel$(TSUFFIX).$(SUFFIX) \ + dsymm_kernel$(TSUFFIX).$(SUFFIX) \ + dtrmm_gemm_kernel$(TSUFFIX).$(SUFFIX) \ $(DGEMMINCOPYOBJ) $(DGEMMITCOPYOBJ) \ $(DGEMMONCOPYOBJ) $(DGEMMOTCOPYOBJ) endif @@ -330,8 +332,10 @@ ifneq "$(or $(BUILD_COMPLEX),$(BUILD_COMPLEX16))" "" CKERNELOBJS += \ cgemm_kernel_n$(TSUFFIX).$(SUFFIX) cgemm_kernel_r$(TSUFFIX).$(SUFFIX) \ cgemm_kernel_l$(TSUFFIX).$(SUFFIX) cgemm_kernel_b$(TSUFFIX).$(SUFFIX) \ - ccomm_kernel_n$(TSUFFIX).$(SUFFIX) ccomm_kernel_r$(TSUFFIX).$(SUFFIX) \ - ccomm_kernel_l$(TSUFFIX).$(SUFFIX) ccomm_kernel_b$(TSUFFIX).$(SUFFIX) \ + csymm_kernel_n$(TSUFFIX).$(SUFFIX) csymm_kernel_r$(TSUFFIX).$(SUFFIX) \ + csymm_kernel_l$(TSUFFIX).$(SUFFIX) csymm_kernel_b$(TSUFFIX).$(SUFFIX) \ + ctrmm_gemm_kernel_n$(TSUFFIX).$(SUFFIX) ctrmm_gemm_kernel_r$(TSUFFIX).$(SUFFIX) \ + ctrmm_gemm_kernel_l$(TSUFFIX).$(SUFFIX) ctrmm_gemm_kernel_b$(TSUFFIX).$(SUFFIX) \ $(CGEMMINCOPYOBJ) $(CGEMMITCOPYOBJ) \ $(CGEMMONCOPYOBJ) $(CGEMMOTCOPYOBJ) endif @@ -340,8 +344,10 @@ ifeq ($(BUILD_COMPLEX16),1) ZKERNELOBJS += \ zgemm_kernel_n$(TSUFFIX).$(SUFFIX) zgemm_kernel_r$(TSUFFIX).$(SUFFIX) \ zgemm_kernel_l$(TSUFFIX).$(SUFFIX) zgemm_kernel_b$(TSUFFIX).$(SUFFIX) \ - zcomm_kernel_n$(TSUFFIX).$(SUFFIX) zcomm_kernel_r$(TSUFFIX).$(SUFFIX) \ - zcomm_kernel_l$(TSUFFIX).$(SUFFIX) zcomm_kernel_b$(TSUFFIX).$(SUFFIX) \ + zsymm_kernel_n$(TSUFFIX).$(SUFFIX) zsymm_kernel_r$(TSUFFIX).$(SUFFIX) \ + zsymm_kernel_l$(TSUFFIX).$(SUFFIX) zsymm_kernel_b$(TSUFFIX).$(SUFFIX) \ + ztrmm_gemm_kernel_n$(TSUFFIX).$(SUFFIX) ztrmm_gemm_kernel_r$(TSUFFIX).$(SUFFIX) \ + ztrmm_gemm_kernel_l$(TSUFFIX).$(SUFFIX) ztrmm_gemm_kernel_b$(TSUFFIX).$(SUFFIX) \ $(ZGEMMINCOPYOBJ) $(ZGEMMITCOPYOBJ) \ $(ZGEMMONCOPYOBJ) $(ZGEMMOTCOPYOBJ) endif @@ -453,7 +459,8 @@ SBLASOBJS += \ strmm_ilnucopy$(TSUFFIX).$(SUFFIX) strmm_ilnncopy$(TSUFFIX).$(SUFFIX) \ strmm_iutucopy$(TSUFFIX).$(SUFFIX) strmm_iutncopy$(TSUFFIX).$(SUFFIX) \ strmm_iltucopy$(TSUFFIX).$(SUFFIX) strmm_iltncopy$(TSUFFIX).$(SUFFIX) \ - scomm_ncopy$(TSUFFIX).$(SUFFIX) scomm_tcopy$(TSUFFIX).$(SUFFIX) \ + ssymm_incopy$(TSUFFIX).$(SUFFIX) ssymm_itcopy$(TSUFFIX).$(SUFFIX) \ + strmm_incopy$(TSUFFIX).$(SUFFIX) strmm_itcopy$(TSUFFIX).$(SUFFIX) \ strmm_ounucopy$(TSUFFIX).$(SUFFIX) strmm_ounncopy$(TSUFFIX).$(SUFFIX) \ strmm_olnucopy$(TSUFFIX).$(SUFFIX) strmm_olnncopy$(TSUFFIX).$(SUFFIX) \ strmm_outucopy$(TSUFFIX).$(SUFFIX) strmm_outncopy$(TSUFFIX).$(SUFFIX) \ @@ -476,7 +483,8 @@ DBLASOBJS += \ dtrmm_ilnucopy$(TSUFFIX).$(SUFFIX) dtrmm_ilnncopy$(TSUFFIX).$(SUFFIX) \ dtrmm_iutucopy$(TSUFFIX).$(SUFFIX) dtrmm_iutncopy$(TSUFFIX).$(SUFFIX) \ dtrmm_iltucopy$(TSUFFIX).$(SUFFIX) dtrmm_iltncopy$(TSUFFIX).$(SUFFIX) \ - dcomm_ncopy$(TSUFFIX).$(SUFFIX) dcomm_tcopy$(TSUFFIX).$(SUFFIX) \ + dsymm_incopy$(TSUFFIX).$(SUFFIX) dsymm_itcopy$(TSUFFIX).$(SUFFIX) \ + dtrmm_incopy$(TSUFFIX).$(SUFFIX) dtrmm_itcopy$(TSUFFIX).$(SUFFIX) \ dtrmm_ounucopy$(TSUFFIX).$(SUFFIX) dtrmm_ounncopy$(TSUFFIX).$(SUFFIX) \ dtrmm_olnucopy$(TSUFFIX).$(SUFFIX) dtrmm_olnncopy$(TSUFFIX).$(SUFFIX) \ dtrmm_outucopy$(TSUFFIX).$(SUFFIX) dtrmm_outncopy$(TSUFFIX).$(SUFFIX) \ @@ -519,7 +527,8 @@ CBLASOBJS += \ ctrmm_ilnucopy$(TSUFFIX).$(SUFFIX) ctrmm_ilnncopy$(TSUFFIX).$(SUFFIX) \ ctrmm_iutucopy$(TSUFFIX).$(SUFFIX) ctrmm_iutncopy$(TSUFFIX).$(SUFFIX) \ ctrmm_iltucopy$(TSUFFIX).$(SUFFIX) ctrmm_iltncopy$(TSUFFIX).$(SUFFIX) \ - ccomm_ncopy$(TSUFFIX).$(SUFFIX) ccomm_tcopy$(TSUFFIX).$(SUFFIX) \ + csymm_incopy$(TSUFFIX).$(SUFFIX) csymm_itcopy$(TSUFFIX).$(SUFFIX) \ + ctrmm_incopy$(TSUFFIX).$(SUFFIX) ctrmm_itcopy$(TSUFFIX).$(SUFFIX) \ ctrmm_ounucopy$(TSUFFIX).$(SUFFIX) ctrmm_ounncopy$(TSUFFIX).$(SUFFIX) \ ctrmm_olnucopy$(TSUFFIX).$(SUFFIX) ctrmm_olnncopy$(TSUFFIX).$(SUFFIX) \ ctrmm_outucopy$(TSUFFIX).$(SUFFIX) ctrmm_outncopy$(TSUFFIX).$(SUFFIX) \ @@ -547,7 +556,8 @@ ZBLASOBJS += \ ztrmm_ilnucopy$(TSUFFIX).$(SUFFIX) ztrmm_ilnncopy$(TSUFFIX).$(SUFFIX) \ ztrmm_iutucopy$(TSUFFIX).$(SUFFIX) ztrmm_iutncopy$(TSUFFIX).$(SUFFIX) \ ztrmm_iltucopy$(TSUFFIX).$(SUFFIX) ztrmm_iltncopy$(TSUFFIX).$(SUFFIX) \ - zcomm_ncopy$(TSUFFIX).$(SUFFIX) zcomm_tcopy$(TSUFFIX).$(SUFFIX) \ + zsymm_incopy$(TSUFFIX).$(SUFFIX) zsymm_itcopy$(TSUFFIX).$(SUFFIX) \ + ztrmm_incopy$(TSUFFIX).$(SUFFIX) ztrmm_itcopy$(TSUFFIX).$(SUFFIX) \ ztrmm_ounucopy$(TSUFFIX).$(SUFFIX) ztrmm_ounncopy$(TSUFFIX).$(SUFFIX) \ ztrmm_olnucopy$(TSUFFIX).$(SUFFIX) ztrmm_olnncopy$(TSUFFIX).$(SUFFIX) \ ztrmm_outucopy$(TSUFFIX).$(SUFFIX) ztrmm_outncopy$(TSUFFIX).$(SUFFIX) \ @@ -936,34 +946,64 @@ endif endif -ifdef SCOMMNCOPY -$(KDIR)scomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SCOMMNCOPY) +ifdef SSYMMINCOPY +$(KDIR)ssymm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SSYMMINCOPY) else ifdef SGEMMINCOPY -$(KDIR)scomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMINCOPY) +$(KDIR)ssymm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMINCOPY) else -$(KDIR)scomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMONCOPY) +$(KDIR)ssymm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMONCOPY) endif ifeq ($(OS), AIX) - $(CC) $(CFLAGS) -S -UDOUBLE -UCOMPLEX $< -o - > scommncopy.s - $(M4_AIX) scommncopy.s > scommncopy_nomacros.s - $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX scommncopy_nomacros.s -o $@ - rm scommncopy.s scommncopy_nomacros.s + $(CC) $(CFLAGS) -S -UDOUBLE -UCOMPLEX $< -o - > ssymm_incopy.s + $(M4_AIX) ssymm_incopy.s > ssymm_incopy_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX ssymm_incopy_nomacros.s -o $@ + rm ssymm_incopy.s ssymm_incopy_nomacros.s +else + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ +endif +ifdef STRMMINCOPY +$(KDIR)strmm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(STRMMINCOPY) +else ifdef SGEMMINCOPY +$(KDIR)strmm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMINCOPY) +else +$(KDIR)strmm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMONCOPY) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -UDOUBLE -UCOMPLEX $< -o - > strmm_incopy.s + $(M4_AIX) strmm_incopy.s > strmm_incopy_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX strmm_incopy_nomacros.s -o $@ + rm strmm_incopy.s strmm_incopy_nomacros.s else $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ endif -ifdef SCOMMTCOPY -$(KDIR)scomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SCOMMTCOPY) +ifdef SSYMMITCOPY +$(KDIR)ssymm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SSYMMITCOPY) else ifdef SGEMMITCOPY -$(KDIR)scomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMITCOPY) +$(KDIR)ssymm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMITCOPY) else -$(KDIR)scomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMOTCOPY) +$(KDIR)ssymm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMOTCOPY) endif ifeq ($(OS), AIX) - $(CC) $(CFLAGS) -S -UDOUBLE -UCOMPLEX $< -o - > scommtcopy.s - $(M4_AIX) scommtcopy.s > scommtcopy_nomacros.s - $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX scommtcopy_nomacros.s -o $@ - rm scommtcopy.s scommtcopy_nomacros.s + $(CC) $(CFLAGS) -S -UDOUBLE -UCOMPLEX $< -o - > ssymm_itcopy.s + $(M4_AIX) ssymm_itcopy.s > ssymm_itcopy_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX ssymm_itcopy_nomacros.s -o $@ + rm ssymm_itcopy.s ssymm_itcopy_nomacros.s +else + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ +endif +ifdef STRMMITCOPY +$(KDIR)strmm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(STRMMITCOPY) +else ifdef SGEMMITCOPY +$(KDIR)strmm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMITCOPY) +else +$(KDIR)strmm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMOTCOPY) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -UDOUBLE -UCOMPLEX $< -o - > strmm_itcopy.s + $(M4_AIX) strmm_itcopy.s > strmm_itcopy_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX strmm_itcopy_nomacros.s -o $@ + rm strmm_itcopy.s strmm_itcopy_nomacros.s else $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ endif @@ -1095,16 +1135,29 @@ else endif -ifdef SCOMMKERNEL -$(KDIR)scomm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SCOMMKERNEL) $(SGEMMDEPEND) +ifdef SSYMMKERNEL +$(KDIR)ssymm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SSYMMKERNEL) $(SGEMMDEPEND) else -$(KDIR)scomm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) $(SGEMMDEPEND) +$(KDIR)ssymm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) $(SGEMMDEPEND) endif ifeq ($(OS), AIX) - $(CC) $(CFLAGS) -S -UDOUBLE -UCOMPLEX $< -o - > scomm_kernel$(TSUFFIX).s - $(M4_AIX) scomm_kernel$(TSUFFIX).s > scomm_kernel$(TSUFFIX)_nomacros.s - $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX scomm_kernel$(TSUFFIX)_nomacros.s -o $@ - rm scomm_kernel$(TSUFFIX).s scomm_kernel$(TSUFFIX)_nomacros.s + $(CC) $(CFLAGS) -S -UDOUBLE -UCOMPLEX $< -o - > ssymm_kernel$(TSUFFIX).s + $(M4_AIX) ssymm_kernel$(TSUFFIX).s > ssymm_kernel$(TSUFFIX)_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX ssymm_kernel$(TSUFFIX)_nomacros.s -o $@ + rm ssymm_kernel$(TSUFFIX).s ssymm_kernel$(TSUFFIX)_nomacros.s +else + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ +endif +ifdef STRMMGEMMKERNEL +$(KDIR)strmm_gemm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(STRMMGEMMKERNEL) $(SGEMMDEPEND) +else +$(KDIR)strmm_gemm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) $(SGEMMDEPEND) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -UDOUBLE -UCOMPLEX $< -o - > strmm_gemm_kernel$(TSUFFIX).s + $(M4_AIX) strmm_gemm_kernel$(TSUFFIX).s > strmm_gemm_kernel$(TSUFFIX)_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX strmm_gemm_kernel$(TSUFFIX)_nomacros.s -o $@ + rm strmm_gemm_kernel$(TSUFFIX).s strmm_gemm_kernel$(TSUFFIX)_nomacros.s else $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ endif @@ -1188,16 +1241,29 @@ else $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ endif -ifdef DCOMMKERNEL -$(KDIR)dcomm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DCOMMKERNEL) $(DGEMMDEPEND) +ifdef DSYMMKERNEL +$(KDIR)dsymm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DSYMMKERNEL) $(DGEMMDEPEND) +else +$(KDIR)dsymm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) $(DGEMMDEPEND) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -DDOUBLE -UCOMPLEX $< -o - > dsymm_kernel$(TSUFFIX).s + $(M4_AIX) dsymm_kernel$(TSUFFIX).s > dsymm_kernel$(TSUFFIX)_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX dsymm_kernel$(TSUFFIX)_nomacros.s -o $@ + rm dsymm_kernel$(TSUFFIX).s dsymm_kernel$(TSUFFIX)_nomacros.s else -$(KDIR)dcomm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) $(DGEMMDEPEND) + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ +endif +ifdef DTRMMGEMMKERNEL +$(KDIR)dtrmm_gemm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DTRMMGEMMKERNEL) $(DGEMMDEPEND) +else +$(KDIR)dtrmm_gemm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) $(DGEMMDEPEND) endif ifeq ($(OS), AIX) - $(CC) $(CFLAGS) -S -DDOUBLE -UCOMPLEX $< -o - > dcomm_kernel$(TSUFFIX).s - $(M4_AIX) dcomm_kernel$(TSUFFIX).s > dcomm_kernel$(TSUFFIX)_nomacros.s - $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX dcomm_kernel$(TSUFFIX)_nomacros.s -o $@ - rm dcomm_kernel$(TSUFFIX).s dcomm_kernel$(TSUFFIX)_nomacros.s + $(CC) $(CFLAGS) -S -DDOUBLE -UCOMPLEX $< -o - > dtrmm_gemm_kernel$(TSUFFIX).s + $(M4_AIX) dtrmm_gemm_kernel$(TSUFFIX).s > dtrmm_gemm_kernel$(TSUFFIX)_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX dtrmm_gemm_kernel$(TSUFFIX)_nomacros.s -o $@ + rm dtrmm_gemm_kernel$(TSUFFIX).s dtrmm_gemm_kernel$(TSUFFIX)_nomacros.s else $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ endif @@ -1245,58 +1311,110 @@ else $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCC $< -o $@ endif -ifdef CCOMMKERNEL -$(KDIR)ccomm_kernel_n$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) $(CGEMMDEPEND) +ifdef CSYMMKERNEL +$(KDIR)csymm_kernel_n$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CSYMMKERNEL) $(CGEMMDEPEND) +else +$(KDIR)csymm_kernel_n$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -UDOUBLE -DCOMPLEX -DNN $< -o - > csymm_kernel_n.s + $(M4_AIX) csymm_kernel_n.s > csymm_kernel_n_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNN csymm_kernel_n_nomacros.s -o $@ + rm csymm_kernel_n.s csymm_kernel_n_nomacros.s else -$(KDIR)ccomm_kernel_n$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNN $< -o $@ +endif +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_gemm_kernel_n$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) $(CGEMMDEPEND) +else +$(KDIR)ctrmm_gemm_kernel_n$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) endif ifeq ($(OS), AIX) - $(CC) $(CFLAGS) -S -UDOUBLE -DCOMPLEX -DNN $< -o - > ccomm_kernel_n.s - $(M4_AIX) ccomm_kernel_n.s > ccomm_kernel_n_nomacros.s - $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNN ccomm_kernel_n_nomacros.s -o $@ - rm ccomm_kernel_n.s ccomm_kernel_n_nomacros.s + $(CC) $(CFLAGS) -S -UDOUBLE -DCOMPLEX -DNN $< -o - > ctrmm_gemm_kernel_n.s + $(M4_AIX) ctrmm_gemm_kernel_n.s > ctrmm_gemm_kernel_n_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNN ctrmm_gemm_kernel_n_nomacros.s -o $@ + rm ctrmm_gemm_kernel_n.s ctrmm_gemm_kernel_n_nomacros.s else $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNN $< -o $@ endif -ifdef CCOMMKERNEL -$(KDIR)ccomm_kernel_l$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) $(CGEMMDEPEND) +ifdef CSYMMKERNEL +$(KDIR)csymm_kernel_l$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CSYMMKERNEL) $(CGEMMDEPEND) else -$(KDIR)ccomm_kernel_l$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +$(KDIR)csymm_kernel_l$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) endif ifeq ($(OS), AIX) - $(CC) $(CFLAGS) -S -UDOUBLE -DCOMPLEX -DCN $< -o - > ccomm_kernel_l.s - $(M4_AIX) ccomm_kernel_l.s > ccomm_kernel_l_nomacros.s - $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCN ccomm_kernel_l_nomacros.s -o $@ - rm ccomm_kernel_l.s ccomm_kernel_l_nomacros.s + $(CC) $(CFLAGS) -S -UDOUBLE -DCOMPLEX -DCN $< -o - > csymm_kernel_l.s + $(M4_AIX) csymm_kernel_l.s > csymm_kernel_l_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCN csymm_kernel_l_nomacros.s -o $@ + rm csymm_kernel_l.s csymm_kernel_l_nomacros.s +else + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCN $< -o $@ +endif +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_gemm_kernel_l$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) $(CGEMMDEPEND) +else +$(KDIR)ctrmm_gemm_kernel_l$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -UDOUBLE -DCOMPLEX -DCN $< -o - > ctrmm_gemm_kernel_l.s + $(M4_AIX) ctrmm_gemm_kernel_l.s > ctrmm_gemm_kernel_l_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCN ctrmm_gemm_kernel_l_nomacros.s -o $@ + rm ctrmm_gemm_kernel_l.s ctrmm_gemm_kernel_l_nomacros.s else $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCN $< -o $@ endif -ifdef CCOMMKERNEL -$(KDIR)ccomm_kernel_r$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) $(CGEMMDEPEND) +ifdef CSYMMKERNEL +$(KDIR)csymm_kernel_r$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CSYMMKERNEL) $(CGEMMDEPEND) else -$(KDIR)ccomm_kernel_r$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +$(KDIR)csymm_kernel_r$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) endif ifeq ($(OS), AIX) - $(CC) $(CFLAGS) -S -UDOUBLE -DCOMPLEX -DNC $< -o - > ccomm_kernel_r.s - $(M4_AIX) ccomm_kernel_r.s > ccomm_kernel_r_nomacros.s - $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNC ccomm_kernel_r_nomacros.s -o $@ - rm ccomm_kernel_r.s ccomm_kernel_r_nomacros.s + $(CC) $(CFLAGS) -S -UDOUBLE -DCOMPLEX -DNC $< -o - > csymm_kernel_r.s + $(M4_AIX) csymm_kernel_r.s > csymm_kernel_r_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNC csymm_kernel_r_nomacros.s -o $@ + rm csymm_kernel_r.s csymm_kernel_r_nomacros.s +else + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNC $< -o $@ +endif +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_gemm_kernel_r$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) $(CGEMMDEPEND) +else +$(KDIR)ctrmm_gemm_kernel_r$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -UDOUBLE -DCOMPLEX -DNC $< -o - > ctrmm_gemm_kernel_r.s + $(M4_AIX) ctrmm_gemm_kernel_r.s > ctrmm_gemm_kernel_r_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNC ctrmm_gemm_kernel_r_nomacros.s -o $@ + rm ctrmm_gemm_kernel_r.s ctrmm_gemm_kernel_r_nomacros.s else $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNC $< -o $@ endif -ifdef CCOMMKERNEL -$(KDIR)ccomm_kernel_b$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) $(CGEMMDEPEND) +ifdef CSYMMKERNEL +$(KDIR)csymm_kernel_b$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CSYMMKERNEL) $(CGEMMDEPEND) +else +$(KDIR)csymm_kernel_b$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -UDOUBLE -DCOMPLEX -DCC $< -o - > csymm_kernel_b.s + $(M4_AIX) csymm_kernel_b.s > csymm_kernel_b_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCC csymm_kernel_b_nomacros.s -o $@ + rm csymm_kernel_b.s csymm_kernel_b_nomacros.s +else + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCC $< -o $@ +endif +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_gemm_kernel_b$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) $(CGEMMDEPEND) else -$(KDIR)ccomm_kernel_b$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +$(KDIR)ctrmm_gemm_kernel_b$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) endif ifeq ($(OS), AIX) - $(CC) $(CFLAGS) -S -UDOUBLE -DCOMPLEX -DCC $< -o - > ccomm_kernel_b.s - $(M4_AIX) ccomm_kernel_b.s > ccomm_kernel_b_nomacros.s - $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCC ccomm_kernel_b_nomacros.s -o $@ - rm ccomm_kernel_b.s ccomm_kernel_b_nomacros.s + $(CC) $(CFLAGS) -S -UDOUBLE -DCOMPLEX -DCC $< -o - > ctrmm_gemm_kernel_b.s + $(M4_AIX) ctrmm_gemm_kernel_b.s > ctrmm_gemm_kernel_b_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCC ctrmm_gemm_kernel_b_nomacros.s -o $@ + rm ctrmm_gemm_kernel_b.s ctrmm_gemm_kernel_b_nomacros.s else $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCC $< -o $@ endif @@ -1349,64 +1467,124 @@ else $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCC $< -o $@ endif -ifdef ZCOMMKERNEL -$(KDIR)zcomm_kernel_n$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) $(ZGEMMDEPEND) +ifdef ZSYMMKERNEL +$(KDIR)zsymm_kernel_n$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZSYMMKERNEL) $(ZGEMMDEPEND) +else +$(KDIR)zsymm_kernel_n$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -DDOUBLE -DCOMPLEX -DNN $< -o - > zsymm_kernel_n.s + $(M4_AIX) zsymm_kernel_n.s > zsymm_kernel_n_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNN zsymm_kernel_n_nomacros.s -o $@ + rm zsymm_kernel_n.s zsymm_kernel_n_nomacros.s +else ifeq ($(CORE),SANDYBRIDGE) + $(CC) $(filter-out -mavx,$(CFLAGS)) -c -DDOUBLE -DCOMPLEX -DNN $< -o $@ +else + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNN $< -o $@ +endif +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_gemm_kernel_n$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) $(ZGEMMDEPEND) else -$(KDIR)zcomm_kernel_n$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +$(KDIR)ztrmm_gemm_kernel_n$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) endif ifeq ($(OS), AIX) - $(CC) $(CFLAGS) -S -DDOUBLE -DCOMPLEX -DNN $< -o - > zcomm_kernel_n.s - $(M4_AIX) zcomm_kernel_n.s > zcomm_kernel_n_nomacros.s - $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNN zcomm_kernel_n_nomacros.s -o $@ - rm zcomm_kernel_n.s zcomm_kernel_n_nomacros.s + $(CC) $(CFLAGS) -S -DDOUBLE -DCOMPLEX -DNN $< -o - > ztrmm_gemm_kernel_n.s + $(M4_AIX) ztrmm_gemm_kernel_n.s > ztrmm_gemm_kernel_n_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNN ztrmm_gemm_kernel_n_nomacros.s -o $@ + rm ztrmm_gemm_kernel_n.s ztrmm_gemm_kernel_n_nomacros.s else ifeq ($(CORE),SANDYBRIDGE) $(CC) $(filter-out -mavx,$(CFLAGS)) -c -DDOUBLE -DCOMPLEX -DNN $< -o $@ else $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNN $< -o $@ endif -ifdef ZCOMMKERNEL -$(KDIR)zcomm_kernel_l$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) $(ZGEMMDEPEND) +ifdef ZSYMMKERNEL +$(KDIR)zsymm_kernel_l$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZSYMMKERNEL) $(ZGEMMDEPEND) else -$(KDIR)zcomm_kernel_l$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +$(KDIR)zsymm_kernel_l$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) endif ifeq ($(OS), AIX) - $(CC) $(CFLAGS) -S -DDOUBLE -DCOMPLEX -DCN $< -o - > zcomm_kernel_l.s - $(M4_AIX) zcomm_kernel_l.s > zcomm_kernel_l_nomacros.s - $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCN zcomm_kernel_l_nomacros.s -o $@ - rm zcomm_kernel_l.s zcomm_kernel_l_nomacros.s + $(CC) $(CFLAGS) -S -DDOUBLE -DCOMPLEX -DCN $< -o - > zsymm_kernel_l.s + $(M4_AIX) zsymm_kernel_l.s > zsymm_kernel_l_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCN zsymm_kernel_l_nomacros.s -o $@ + rm zsymm_kernel_l.s zsymm_kernel_l_nomacros.s +else ifeq ($(CORE),SANDYBRIDGE) + $(CC) $(filter-out -mavx,$(CFLAGS)) -c -DDOUBLE -DCOMPLEX -DCN $< -o $@ +else + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCN $< -o $@ +endif +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_gemm_kernel_l$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) $(ZGEMMDEPEND) +else +$(KDIR)ztrmm_gemm_kernel_l$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -DDOUBLE -DCOMPLEX -DCN $< -o - > ztrmm_gemm_kernel_l.s + $(M4_AIX) ztrmm_gemm_kernel_l.s > ztrmm_gemm_kernel_l_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCN ztrmm_gemm_kernel_l_nomacros.s -o $@ + rm ztrmm_gemm_kernel_l.s ztrmm_gemm_kernel_l_nomacros.s else ifeq ($(CORE),SANDYBRIDGE) $(CC) $(filter-out -mavx,$(CFLAGS)) -c -DDOUBLE -DCOMPLEX -DCN $< -o $@ else $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCN $< -o $@ endif -ifdef ZCOMMKERNEL -$(KDIR)zcomm_kernel_r$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) $(ZGEMMDEPEND) +ifdef ZSYMMKERNEL +$(KDIR)zsymm_kernel_r$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZSYMMKERNEL) $(ZGEMMDEPEND) +else +$(KDIR)zsymm_kernel_r$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -DDOUBLE -DCOMPLEX -DNC $< -o - > zsymm_kernel_r.s + $(M4_AIX) zsymm_kernel_r.s > zsymm_kernel_r_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNC zsymm_kernel_r_nomacros.s -o $@ + rm zsymm_kernel_r.s zsymm_kernel_r_nomacros.s +else ifeq ($(CORE),SANDYBRIDGE) + $(CC) $(filter-out -mavx,$(CFLAGS)) -c -DDOUBLE -DCOMPLEX -DNC $< -o $@ +else + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNC $< -o $@ +endif +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_gemm_kernel_r$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) $(ZGEMMDEPEND) else -$(KDIR)zcomm_kernel_r$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +$(KDIR)ztrmm_gemm_kernel_r$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) endif ifeq ($(OS), AIX) - $(CC) $(CFLAGS) -S -DDOUBLE -DCOMPLEX -DNC $< -o - > zcomm_kernel_r.s - $(M4_AIX) zcomm_kernel_r.s > zcomm_kernel_r_nomacros.s - $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNC zcomm_kernel_r_nomacros.s -o $@ - rm zcomm_kernel_r.s zcomm_kernel_r_nomacros.s + $(CC) $(CFLAGS) -S -DDOUBLE -DCOMPLEX -DNC $< -o - > ztrmm_gemm_kernel_r.s + $(M4_AIX) ztrmm_gemm_kernel_r.s > ztrmm_gemm_kernel_r_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNC ztrmm_gemm_kernel_r_nomacros.s -o $@ + rm ztrmm_gemm_kernel_r.s ztrmm_gemm_kernel_r_nomacros.s else ifeq ($(CORE),SANDYBRIDGE) $(CC) $(filter-out -mavx,$(CFLAGS)) -c -DDOUBLE -DCOMPLEX -DNC $< -o $@ else $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNC $< -o $@ endif -ifdef ZCOMMKERNEL -$(KDIR)zcomm_kernel_b$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) $(ZGEMMDEPEND) +ifdef ZSYMMKERNEL +$(KDIR)zsymm_kernel_b$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZSYMMKERNEL) $(ZGEMMDEPEND) else -$(KDIR)zcomm_kernel_b$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +$(KDIR)zsymm_kernel_b$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) endif ifeq ($(OS), AIX) - $(CC) $(CFLAGS) -S -DDOUBLE -DCOMPLEX -DCC $< -o - > zcomm_kernel_b.s - $(M4_AIX) zcomm_kernel_b.s > zcomm_kernel_b_nomacros.s - $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCC zcomm_kernel_b_nomacros.s -o $@ - rm zcomm_kernel_b.s zcomm_kernel_b_nomacros.s + $(CC) $(CFLAGS) -S -DDOUBLE -DCOMPLEX -DCC $< -o - > zsymm_kernel_b.s + $(M4_AIX) zsymm_kernel_b.s > zsymm_kernel_b_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCC zsymm_kernel_b_nomacros.s -o $@ + rm zsymm_kernel_b.s zsymm_kernel_b_nomacros.s +else ifeq ($(CORE),SANDYBRIDGE) + $(CC) $(filter-out -mavx,$(CFLAGS)) -c -DDOUBLE -DCOMPLEX -DCC $< -o $@ +else + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCC $< -o $@ +endif +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_gemm_kernel_b$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) $(ZGEMMDEPEND) +else +$(KDIR)ztrmm_gemm_kernel_b$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -DDOUBLE -DCOMPLEX -DCC $< -o - > ztrmm_gemm_kernel_b.s + $(M4_AIX) ztrmm_gemm_kernel_b.s > ztrmm_gemm_kernel_b_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCC ztrmm_gemm_kernel_b_nomacros.s -o $@ + rm ztrmm_gemm_kernel_b.s ztrmm_gemm_kernel_b_nomacros.s else ifeq ($(CORE),SANDYBRIDGE) $(CC) $(filter-out -mavx,$(CFLAGS)) -c -DDOUBLE -DCOMPLEX -DCC $< -o $@ else @@ -1728,29 +1906,29 @@ endif else -ifdef SCOMMKERNEL -$(KDIR)strmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SCOMMKERNEL) +ifdef STRMMGEMMKERNEL +$(KDIR)strmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(STRMMGEMMKERNEL) else $(KDIR)strmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -DLEFT -UTRANSA $< -o $@ -ifdef SCOMMKERNEL -$(KDIR)strmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SCOMMKERNEL) +ifdef STRMMGEMMKERNEL +$(KDIR)strmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(STRMMGEMMKERNEL) else $(KDIR)strmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -DLEFT -DTRANSA $< -o $@ -ifdef SCOMMKERNEL -$(KDIR)strmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SCOMMKERNEL) +ifdef STRMMGEMMKERNEL +$(KDIR)strmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(STRMMGEMMKERNEL) else $(KDIR)strmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -UTRANSA $< -o $@ -ifdef SCOMMKERNEL -$(KDIR)strmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SCOMMKERNEL) +ifdef STRMMGEMMKERNEL +$(KDIR)strmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(STRMMGEMMKERNEL) else $(KDIR)strmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) endif @@ -1763,29 +1941,29 @@ else $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o $@ endif -ifdef DCOMMKERNEL -$(KDIR)dtrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DCOMMKERNEL) +ifdef DTRMMGEMMKERNEL +$(KDIR)dtrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DTRMMGEMMKERNEL) else $(KDIR)dtrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -UTRANSA $< -o $@ -ifdef DCOMMKERNEL -$(KDIR)dtrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DCOMMKERNEL) +ifdef DTRMMGEMMKERNEL +$(KDIR)dtrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DTRMMGEMMKERNEL) else $(KDIR)dtrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -DTRANSA $< -o $@ -ifdef DCOMMKERNEL -$(KDIR)dtrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DCOMMKERNEL) +ifdef DTRMMGEMMKERNEL +$(KDIR)dtrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DTRMMGEMMKERNEL) else $(KDIR)dtrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -ULEFT -UTRANSA $< -o $@ -ifdef DCOMMKERNEL -$(KDIR)dtrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DCOMMKERNEL) +ifdef DTRMMGEMMKERNEL +$(KDIR)dtrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DTRMMGEMMKERNEL) else $(KDIR)dtrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) endif @@ -1803,64 +1981,64 @@ $(KDIR)qtrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(QGEMMKERNEL) $(KDIR)qtrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(QGEMMKERNEL) $(CC) $(CFLAGS) -c -DTRMMKERNEL -DXDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o $@ -ifdef CCOMMKERNEL -$(KDIR)ctrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) else $(KDIR)ctrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -UTRANSA -UCONJ -DNN $< -o $@ -ifdef CCOMMKERNEL -$(KDIR)ctrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) else $(KDIR)ctrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -DTRANSA -UCONJ -DNN $< -o $@ -ifdef CCOMMKERNEL -$(KDIR)ctrmm_kernel_LR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_kernel_LR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) else $(KDIR)ctrmm_kernel_LR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -UTRANSA -DCONJ -DCN $< -o $@ -ifdef CCOMMKERNEL -$(KDIR)ctrmm_kernel_LC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_kernel_LC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) else $(KDIR)ctrmm_kernel_LC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -DTRANSA -DCONJ -DCN $< -o $@ -ifdef CCOMMKERNEL -$(KDIR)ctrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) else $(KDIR)ctrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -UTRANSA -UCONJ -DNN $< -o $@ -ifdef CCOMMKERNEL -$(KDIR)ctrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) else $(KDIR)ctrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -DTRANSA -UCONJ -DNN $< -o $@ -ifdef CCOMMKERNEL -$(KDIR)ctrmm_kernel_RR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_kernel_RR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) else $(KDIR)ctrmm_kernel_RR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -UTRANSA -DCONJ -DNC $< -o $@ -ifdef CCOMMKERNEL -$(KDIR)ctrmm_kernel_RC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_kernel_RC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) else $(KDIR)ctrmm_kernel_RC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) endif $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -DTRANSA -DCONJ -DNC $< -o $@ -ifdef ZCOMMKERNEL -$(KDIR)ztrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) else $(KDIR)ztrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) endif @@ -1870,8 +2048,8 @@ else $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -UTRANSA -UCONJ -DNN $< -o $@ endif -ifdef ZCOMMKERNEL -$(KDIR)ztrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) else $(KDIR)ztrmm_kernel_LT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) endif @@ -1881,8 +2059,8 @@ else $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -DTRANSA -UCONJ -DNN $< -o $@ endif -ifdef ZCOMMKERNEL -$(KDIR)ztrmm_kernel_LR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_kernel_LR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) else $(KDIR)ztrmm_kernel_LR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) endif @@ -1892,8 +2070,8 @@ else $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -UTRANSA -DCONJ -DCN $< -o $@ endif -ifdef ZCOMMKERNEL -$(KDIR)ztrmm_kernel_LC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_kernel_LC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) else $(KDIR)ztrmm_kernel_LC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) endif @@ -1903,8 +2081,8 @@ else $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -DTRANSA -DCONJ -DCN $< -o $@ endif -ifdef ZCOMMKERNEL -$(KDIR)ztrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) else $(KDIR)ztrmm_kernel_RN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) endif @@ -1914,8 +2092,8 @@ else $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -UTRANSA -UCONJ -DNN $< -o $@ endif -ifdef ZCOMMKERNEL -$(KDIR)ztrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) else $(KDIR)ztrmm_kernel_RT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) endif @@ -1925,8 +2103,8 @@ else $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -DTRANSA -UCONJ -DNN $< -o $@ endif -ifdef ZCOMMKERNEL -$(KDIR)ztrmm_kernel_RR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_kernel_RR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) else $(KDIR)ztrmm_kernel_RR$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) endif @@ -1936,8 +2114,8 @@ else $(CC) $(CFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -UTRANSA -DCONJ -DNC $< -o $@ endif -ifdef ZCOMMKERNEL -$(KDIR)ztrmm_kernel_RC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_kernel_RC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) else $(KDIR)ztrmm_kernel_RC$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) endif @@ -2234,34 +2412,64 @@ $(KDIR)dtrmm_iltncopy$(TSUFFIX).$(SUFFIX) : generic/trmm_ltcopy_$(DGEMM_UNROLL_M $(CC) -c $(CFLAGS) $(NO_UNINITIALIZED_WARN) -DDOUBLE -UCOMPLEX -UOUTER -DLOWER -UUNIT $< -o $@ endif -ifdef DCOMMNCOPY -$(KDIR)dcomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DCOMMNCOPY) +ifdef DSYMMINCOPY +$(KDIR)dsymm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DSYMMINCOPY) else ifdef DGEMMINCOPY -$(KDIR)dcomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMINCOPY) +$(KDIR)dsymm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMINCOPY) else -$(KDIR)dcomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMONCOPY) +$(KDIR)dsymm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMONCOPY) endif ifeq ($(OS), AIX) - $(CC) $(CFLAGS) -S -DDOUBLE -UCOMPLEX $< -o - > dcomm_ncopy.s - $(M4_AIX) dcomm_ncopy.s > dcomm_ncopy_nomacros.s - $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX dcomm_ncopy_nomacros.s -o $@ - rm dcomm_ncopy.s dcomm_ncopy_nomacros.s + $(CC) $(CFLAGS) -S -DDOUBLE -UCOMPLEX $< -o - > dsymm_incopy.s + $(M4_AIX) dsymm_incopy.s > dsymm_incopy_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX dsymm_incopy_nomacros.s -o $@ + rm dsymm_incopy.s dsymm_incopy_nomacros.s +else + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ +endif +ifdef DTRMMINCOPY +$(KDIR)dtrmm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DTRMMINCOPY) +else ifdef DGEMMINCOPY +$(KDIR)dtrmm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMINCOPY) +else +$(KDIR)dtrmm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMONCOPY) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -DDOUBLE -UCOMPLEX $< -o - > dtrmm_incopy.s + $(M4_AIX) dtrmm_incopy.s > dtrmm_incopy_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX dtrmm_incopy_nomacros.s -o $@ + rm dtrmm_incopy.s dtrmm_incopy_nomacros.s else $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ endif -ifdef DCOMMTCOPY -$(KDIR)dcomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DCOMMTCOPY) +ifdef DSYMMITCOPY +$(KDIR)dsymm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DSYMMITCOPY) else ifdef DGEMMITCOPY -$(KDIR)dcomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMITCOPY) +$(KDIR)dsymm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMITCOPY) else -$(KDIR)dcomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMOTCOPY) +$(KDIR)dsymm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMOTCOPY) endif ifeq ($(OS), AIX) - $(CC) $(CFLAGS) -S -DDOUBLE -UCOMPLEX $< -o - > dcomm_tcopy.s - $(M4_AIX) dcomm_tcopy.s > dcomm_tcopy_nomacros.s - $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX dcomm_tcopy_nomacros.s -o $@ - rm dcomm_tcopy.s dcomm_tcopy_nomacros.s + $(CC) $(CFLAGS) -S -DDOUBLE -UCOMPLEX $< -o - > dsymm_itcopy.s + $(M4_AIX) dsymm_itcopy.s > dsymm_itcopy_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX dsymm_itcopy_nomacros.s -o $@ + rm dsymm_itcopy.s dsymm_itcopy_nomacros.s +else + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ +endif +ifdef DTRMMITCOPY +$(KDIR)dtrmm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DTRMMITCOPY) +else ifdef DGEMMITCOPY +$(KDIR)dtrmm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMITCOPY) +else +$(KDIR)dtrmm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMOTCOPY) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -DDOUBLE -UCOMPLEX $< -o - > dtrmm_itcopy.s + $(M4_AIX) dtrmm_itcopy.s > dtrmm_itcopy_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX dtrmm_itcopy_nomacros.s -o $@ + rm dtrmm_itcopy.s dtrmm_itcopy_nomacros.s else $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ endif @@ -2395,27 +2603,50 @@ $(KDIR)ctrmm_iltncopy$(TSUFFIX).$(SUFFIX) : generic/ztrmm_ltcopy_$(CGEMM_UNROLL_ $(CC) -c $(CFLAGS) $(NO_UNINITIALIZED_WARN) -UDOUBLE -DCOMPLEX -UOUTER -ULOWER -UUNIT $< -o $@ endif -ifdef CCOMMNCOPY -$(KDIR)ccomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMNCOPY) +ifdef CSYMMINCOPY +$(KDIR)csymm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CSYMMINCOPY) +else ifdef CGEMMINCOPY +$(KDIR)csymm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMINCOPY) +else +$(KDIR)csymm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMONCOPY) +endif + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ +ifdef CTRMMINCOPY +$(KDIR)ctrmm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMINCOPY) else ifdef CGEMMINCOPY -$(KDIR)ccomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMINCOPY) +$(KDIR)ctrmm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMINCOPY) else -$(KDIR)ccomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMONCOPY) +$(KDIR)ctrmm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMONCOPY) endif $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ -ifdef CCOMMTCOPY -$(KDIR)ccomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CCOMMTCOPY) +ifdef CSYMMITCOPY +$(KDIR)csymm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CSYMMITCOPY) else ifdef CGEMMITCOPY -$(KDIR)ccomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMITCOPY) +$(KDIR)csymm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMITCOPY) else -$(KDIR)ccomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMOTCOPY) +$(KDIR)csymm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMOTCOPY) endif ifeq ($(OS), AIX) - $(CC) $(CFLAGS) -UDOUBLE -UCOMPLEX -S $< -o - > ccomm_tcopy.s - $(M4_AIX) ccomm_tcopy.s > ccomm_tcopy_nomacros.s - $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX ccomm_tcopy_nomacros.s -o $@ - rm ccomm_tcopy.s ccomm_tcopy_nomacros.s + $(CC) $(CFLAGS) -UDOUBLE -UCOMPLEX -S $< -o - > csymm_itcopy.s + $(M4_AIX) csymm_itcopy.s > csymm_itcopy_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX csymm_itcopy_nomacros.s -o $@ + rm csymm_itcopy.s csymm_itcopy_nomacros.s +else + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ +endif +ifdef CTRMMITCOPY +$(KDIR)ctrmm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CTRMMITCOPY) +else ifdef CGEMMITCOPY +$(KDIR)ctrmm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMITCOPY) +else +$(KDIR)ctrmm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMOTCOPY) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -UDOUBLE -UCOMPLEX -S $< -o - > ctrmm_itcopy.s + $(M4_AIX) ctrmm_itcopy.s > ctrmm_itcopy_nomacros.s + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX ctrmm_itcopy_nomacros.s -o $@ + rm ctrmm_itcopy.s ctrmm_itcopy_nomacros.s else $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ endif @@ -2500,27 +2731,50 @@ $(KDIR)ztrmm_iltncopy$(TSUFFIX).$(SUFFIX) : generic/ztrmm_ltcopy_$(ZGEMM_UNROLL_ $(CC) -c $(CFLAGS) $(NO_UNINITIALIZED_WARN) -DDOUBLE -DCOMPLEX -UOUTER -DLOWER -UUNIT $< -o $@ endif -ifdef ZCOMMNCOPY -$(KDIR)zcomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMNCOPY) +ifdef ZSYMMINCOPY +$(KDIR)zsymm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZSYMMINCOPY) else ifdef ZGEMMINCOPY -$(KDIR)zcomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMINCOPY) +$(KDIR)zsymm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMINCOPY) else -$(KDIR)zcomm_ncopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMONCOPY) +$(KDIR)zsymm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMONCOPY) +endif + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ +ifdef ZTRMMINCOPY +$(KDIR)ztrmm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMINCOPY) +else ifdef ZGEMMINCOPY +$(KDIR)ztrmm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMINCOPY) +else +$(KDIR)ztrmm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMONCOPY) endif $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ -ifdef ZCOMMTCOPY -$(KDIR)zcomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZCOMMTCOPY) +ifdef ZSYMMITCOPY +$(KDIR)zsymm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZSYMMITCOPY) +else ifdef ZGEMMITCOPY +$(KDIR)zsymm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMITCOPY) +else +$(KDIR)zsymm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMOTCOPY) +endif +ifeq ($(OS), AIX) + $(CC) $(CFLAGS) -S -DDOUBLE -UCOMPLEX $< -o - > zsymm_itcopy.s + $(M4_AIX) zsymm_itcopy.s > zsymm_itcopy_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX zsymm_itcopy_nomacros.s -o $@ + rm zsymm_itcopy.s zsymm_itcopy_nomacros.s +else + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ +endif +ifdef ZTRMMITCOPY +$(KDIR)ztrmm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZTRMMITCOPY) else ifdef ZGEMMITCOPY -$(KDIR)zcomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMITCOPY) +$(KDIR)ztrmm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMITCOPY) else -$(KDIR)zcomm_tcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMOTCOPY) +$(KDIR)ztrmm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMOTCOPY) endif ifeq ($(OS), AIX) - $(CC) $(CFLAGS) -S -DDOUBLE -UCOMPLEX $< -o - > zcomm_tcopy.s - $(M4_AIX) zcomm_tcopy.s > zcomm_tcopy_nomacros.s - $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX zcomm_tcopy_nomacros.s -o $@ - rm zcomm_tcopy.s zcomm_tcopy_nomacros.s + $(CC) $(CFLAGS) -S -DDOUBLE -UCOMPLEX $< -o - > ztrmm_itcopy.s + $(M4_AIX) ztrmm_itcopy.s > ztrmm_itcopy_nomacros.s + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX ztrmm_itcopy_nomacros.s -o $@ + rm ztrmm_itcopy.s ztrmm_itcopy_nomacros.s else $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ endif @@ -3601,21 +3855,37 @@ $(SGEMMITCOPYOBJ_P) : $(KERNELDIR)/$(SGEMMITCOPY) endif -ifdef SCOMMNCOPY -$(KDIR)scomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SCOMMNCOPY) +ifdef SSYMMINCOPY +$(KDIR)ssymm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SSYMMINCOPY) +else ifdef SGEMMINCOPY +$(KDIR)ssymm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMINCOPY) +else +$(KDIR)ssymm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMONCOPY) +endif + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ +ifdef STRMMINCOPY +$(KDIR)strmm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(STRMMINCOPY) else ifdef SGEMMINCOPY -$(KDIR)scomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMINCOPY) +$(KDIR)strmm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMINCOPY) else -$(KDIR)scomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMONCOPY) +$(KDIR)strmm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMONCOPY) endif $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ -ifdef SCOMMTCOPY -$(KDIR)scomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SCOMMTCOPY) +ifdef SSYMMITCOPY +$(KDIR)ssymm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SSYMMITCOPY) +else ifdef SGEMMITCOPY +$(KDIR)ssymm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMITCOPY) +else +$(KDIR)ssymm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMOTCOPY) +endif + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ +ifdef STRMMITCOPY +$(KDIR)strmm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(STRMMITCOPY) else ifdef SGEMMITCOPY -$(KDIR)scomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMITCOPY) +$(KDIR)strmm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMITCOPY) else -$(KDIR)scomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMOTCOPY) +$(KDIR)strmm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMOTCOPY) endif $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ @@ -3636,21 +3906,37 @@ $(DGEMMITCOPYOBJ_P) : $(KERNELDIR)/$(DGEMMITCOPY) endif -ifdef DCOMMNCOPY -$(KDIR)dcomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DCOMMNCOPY) +ifdef DSYMMINCOPY +$(KDIR)dsymm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DSYMMINCOPY) else ifdef DGEMMINCOPY -$(KDIR)dcomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMINCOPY) +$(KDIR)dsymm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMINCOPY) else -$(KDIR)dcomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMONCOPY) +$(KDIR)dsymm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMONCOPY) +endif + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ +ifdef DTRMMINCOPY +$(KDIR)dtrmm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DTRMMINCOPY) +else ifdef DGEMMINCOPY +$(KDIR)dtrmm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMINCOPY) +else +$(KDIR)dtrmm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMONCOPY) endif $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ -ifdef DCOMMTCOPY -$(KDIR)dcomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DCOMMTCOPY) +ifdef DSYMMITCOPY +$(KDIR)dsymm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DSYMMITCOPY) else ifdef DGEMMITCOPY -$(KDIR)dcomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMITCOPY) +$(KDIR)dsymm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMITCOPY) else -$(KDIR)dcomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMOTCOPY) +$(KDIR)dsymm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMOTCOPY) +endif + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ +ifdef DTRMMITCOPY +$(KDIR)dtrmm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DTRMMITCOPY) +else ifdef DGEMMITCOPY +$(KDIR)dtrmm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMITCOPY) +else +$(KDIR)dtrmm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMOTCOPY) endif $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ @@ -3690,21 +3976,37 @@ $(CGEMMITCOPYOBJ_P) : $(KERNELDIR)/$(CGEMMITCOPY) endif -ifdef CCOMMNCOPY -$(KDIR)ccomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMNCOPY) +ifdef CSYMMINCOPY +$(KDIR)csymm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CSYMMINCOPY) +else ifdef CGEMMINCOPY +$(KDIR)csymm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMINCOPY) +else +$(KDIR)csymm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMONCOPY) +endif + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ +ifdef CTRMMINCOPY +$(KDIR)ctrmm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CTRMMINCOPY) else ifdef CGEMMINCOPY -$(KDIR)ccomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMINCOPY) +$(KDIR)ctrmm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMINCOPY) else -$(KDIR)ccomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMONCOPY) +$(KDIR)ctrmm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMONCOPY) endif $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ -ifdef CCOMMTCOPY -$(KDIR)ccomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMTCOPY) +ifdef CSYMMITCOPY +$(KDIR)csymm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CSYMMITCOPY) else ifdef CGEMMITCOPY -$(KDIR)ccomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMITCOPY) +$(KDIR)csymm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMITCOPY) else -$(KDIR)ccomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMOTCOPY) +$(KDIR)csymm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMOTCOPY) +endif + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ +ifdef CTRMMITCOPY +$(KDIR)ctrmm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CTRMMITCOPY) +else ifdef CGEMMITCOPY +$(KDIR)ctrmm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMITCOPY) +else +$(KDIR)ctrmm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMOTCOPY) endif $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ @@ -3724,21 +4026,37 @@ $(ZGEMMITCOPYOBJ_P) : $(KERNELDIR)/$(ZGEMMITCOPY) endif -ifdef ZCOMMNCOPY -$(KDIR)zcomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMNCOPY) +ifdef ZSYMMINCOPY +$(KDIR)zsymm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZSYMMINCOPY) +else ifdef ZGEMMINCOPY +$(KDIR)zsymm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMINCOPY) +else +$(KDIR)zsymm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMONCOPY) +endif + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ +ifdef ZTRMMINCOPY +$(KDIR)ztrmm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZTRMMINCOPY) else ifdef ZGEMMINCOPY -$(KDIR)zcomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMINCOPY) +$(KDIR)ztrmm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMINCOPY) else -$(KDIR)zcomm_ncopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMONCOPY) +$(KDIR)ztrmm_incopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMONCOPY) endif $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ -ifdef ZCOMMTCOPY -$(KDIR)zcomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMTCOPY) +ifdef ZSYMMITCOPY +$(KDIR)zsymm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZSYMMITCOPY) +else ifdef ZGEMMITCOPY +$(KDIR)zsymm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMITCOPY) +else +$(KDIR)zsymm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMOTCOPY) +endif + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ +ifdef ZTRMMITCOPY +$(KDIR)ztrmm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZTRMMITCOPY) else ifdef ZGEMMITCOPY -$(KDIR)zcomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMITCOPY) +$(KDIR)ztrmm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMITCOPY) else -$(KDIR)zcomm_tcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMOTCOPY) +$(KDIR)ztrmm_itcopy$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMOTCOPY) endif $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ @@ -3778,20 +4096,32 @@ endif $(KDIR)sgemm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) $(SGEMMDEPEND) $(CC) $(PFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ -ifdef SCOMMKERNEL -$(KDIR)scomm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SCOMMKERNEL) $(SGEMMDEPEND) +ifdef SSYMMKERNEL +$(KDIR)ssymm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SSYMMKERNEL) $(SGEMMDEPEND) else -$(KDIR)scomm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) $(SGEMMDEPEND) +$(KDIR)ssymm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) $(SGEMMDEPEND) +endif + $(CC) $(PFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ +ifdef STRMMGEMMKERNEL +$(KDIR)strmm_gemm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(STRMMGEMMKERNEL) $(SGEMMDEPEND) +else +$(KDIR)strmm_gemm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) $(SGEMMDEPEND) endif $(CC) $(PFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ $(KDIR)dgemm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) $(DGEMMDEPEND) $(CC) $(PFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ -ifdef DCOMMKERNEL -$(KDIR)dcomm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DCOMMKERNEL) $(DGEMMDEPEND) +ifdef DSYMMKERNEL +$(KDIR)dsymm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DSYMMKERNEL) $(DGEMMDEPEND) +else +$(KDIR)dsymm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) $(DGEMMDEPEND) +endif + $(CC) $(PFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ +ifdef DTRMMGEMMKERNEL +$(KDIR)dtrmm_gemm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DTRMMGEMMKERNEL) $(DGEMMDEPEND) else -$(KDIR)dcomm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) $(DGEMMDEPEND) +$(KDIR)dtrmm_gemm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) $(DGEMMDEPEND) endif $(CC) $(PFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ @@ -3817,38 +4147,69 @@ endif $(KDIR)cgemm_kernel_b$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) $(CC) $(PFLAGS) -c -UDOUBLE -DCOMPLEX -DCC $< -o $@ -ifdef CCOMMKERNEL -$(KDIR)ccomm_kernel_n$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) $(CGEMMDEPEND) +ifdef CSYMMKERNEL +$(KDIR)csymm_kernel_n$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CSYMMKERNEL) $(CGEMMDEPEND) else -$(KDIR)ccomm_kernel_n$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +$(KDIR)csymm_kernel_n$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +endif + $(CC) $(PFLAGS) -c -UDOUBLE -DCOMPLEX -DNN $< -o $@ +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_gemm_kernel_n$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) $(CGEMMDEPEND) +else +$(KDIR)ctrmm_gemm_kernel_n$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) endif $(CC) $(PFLAGS) -c -UDOUBLE -DCOMPLEX -DNN $< -o $@ -ifdef CCOMMKERNEL -$(KDIR)ccomm_kernel_l$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) $(CGEMMDEPEND) +ifdef CSYMMKERNEL +$(KDIR)csymm_kernel_l$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CSYMMKERNEL) $(CGEMMDEPEND) else -$(KDIR)ccomm_kernel_l$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +$(KDIR)csymm_kernel_l$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +endif + $(CC) $(PFLAGS) -c -UDOUBLE -DCOMPLEX -DCN $< -o $@ +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_gemm_kernel_l$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) $(CGEMMDEPEND) +else +$(KDIR)ctrmm_gemm_kernel_l$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) endif $(CC) $(PFLAGS) -c -UDOUBLE -DCOMPLEX -DCN $< -o $@ -ifdef CCOMMKERNEL -$(KDIR)ccomm_kernel_r$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) $(CGEMMDEPEND) +ifdef CSYMMKERNEL +$(KDIR)csymm_kernel_r$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CSYMMKERNEL) $(CGEMMDEPEND) else -$(KDIR)ccomm_kernel_r$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +$(KDIR)csymm_kernel_r$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) endif ifeq ($(OS), AIX) - $(CC) $(PFLAGS) -S -UDOUBLE -DCOMPLEX -DNC $< -o - > ccomm_kernel_r.s - $(M4_AIX) ccomm_kernel_r.s > ccomm_kernel_r_nomacros.s - $(CC) $(PFLAGS) -c -UDOUBLE -DCOMPLEX -DNC ccomm_kernel_r_nomacros.s -o $@ - rm ccomm_kernel_r.s ccomm_kernel_r_nomacros.s + $(CC) $(PFLAGS) -S -UDOUBLE -DCOMPLEX -DNC $< -o - > csymm_kernel_r.s + $(M4_AIX) csymm_kernel_r.s > csymm_kernel_r_nomacros.s + $(CC) $(PFLAGS) -c -UDOUBLE -DCOMPLEX -DNC csymm_kernel_r_nomacros.s -o $@ + rm csymm_kernel_r.s csymm_kernel_r_nomacros.s +else + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNC $< -o $@ +endif +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_gemm_kernel_r$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) $(CGEMMDEPEND) +else +$(KDIR)ctrmm_gemm_kernel_r$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +endif +ifeq ($(OS), AIX) + $(CC) $(PFLAGS) -S -UDOUBLE -DCOMPLEX -DNC $< -o - > ctrmm_gemm_kernel_r.s + $(M4_AIX) ctrmm_gemm_kernel_r.s > ctrmm_gemm_kernel_r_nomacros.s + $(CC) $(PFLAGS) -c -UDOUBLE -DCOMPLEX -DNC ctrmm_gemm_kernel_r_nomacros.s -o $@ + rm ctrmm_gemm_kernel_r.s ctrmm_gemm_kernel_r_nomacros.s else $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNC $< -o $@ endif -ifdef CCOMMKERNEL -$(KDIR)ccomm_kernel_b$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) $(CGEMMDEPEND) +ifdef CSYMMKERNEL +$(KDIR)csymm_kernel_b$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CSYMMKERNEL) $(CGEMMDEPEND) else -$(KDIR)ccomm_kernel_b$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +$(KDIR)csymm_kernel_b$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) +endif + $(CC) $(PFLAGS) -c -UDOUBLE -DCOMPLEX -DCC $< -o $@ +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_gemm_kernel_b$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) $(CGEMMDEPEND) +else +$(KDIR)ctrmm_gemm_kernel_b$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) $(CGEMMDEPEND) endif $(CC) $(PFLAGS) -c -UDOUBLE -DCOMPLEX -DCC $< -o $@ @@ -3864,31 +4225,55 @@ $(KDIR)zgemm_kernel_r$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMM $(KDIR)zgemm_kernel_b$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) $(CC) $(PFLAGS) -c -DDOUBLE -DCOMPLEX -DCC $< -o $@ -ifdef ZCOMMKERNEL -$(KDIR)zcomm_kernel_n$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) $(ZGEMMDEPEND) +ifdef ZSYMMKERNEL +$(KDIR)zsymm_kernel_n$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZSYMMKERNEL) $(ZGEMMDEPEND) +else +$(KDIR)zsymm_kernel_n$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +endif + $(CC) $(PFLAGS) -c -DDOUBLE -DCOMPLEX -DNN $< -o $@ +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_gemm_kernel_n$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) $(ZGEMMDEPEND) else -$(KDIR)zcomm_kernel_n$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +$(KDIR)ztrmm_gemm_kernel_n$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) endif $(CC) $(PFLAGS) -c -DDOUBLE -DCOMPLEX -DNN $< -o $@ -ifdef ZCOMMKERNEL -$(KDIR)zcomm_kernel_l$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) $(ZGEMMDEPEND) +ifdef ZSYMMKERNEL +$(KDIR)zsymm_kernel_l$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZSYMMKERNEL) $(ZGEMMDEPEND) +else +$(KDIR)zsymm_kernel_l$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +endif + $(CC) $(PFLAGS) -c -DDOUBLE -DCOMPLEX -DCN $< -o $@ +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_gemm_kernel_l$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) $(ZGEMMDEPEND) else -$(KDIR)zcomm_kernel_l$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +$(KDIR)ztrmm_gemm_kernel_l$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) endif $(CC) $(PFLAGS) -c -DDOUBLE -DCOMPLEX -DCN $< -o $@ -ifdef ZCOMMKERNEL -$(KDIR)zcomm_kernel_r$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) $(ZGEMMDEPEND) +ifdef ZSYMMKERNEL +$(KDIR)zsymm_kernel_r$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZSYMMKERNEL) $(ZGEMMDEPEND) else -$(KDIR)zcomm_kernel_r$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +$(KDIR)zsymm_kernel_r$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +endif + $(CC) $(PFLAGS) -c -DDOUBLE -DCOMPLEX -DNC $< -o $@ +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_gemm_kernel_r$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) $(ZGEMMDEPEND) +else +$(KDIR)ztrmm_gemm_kernel_r$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) endif $(CC) $(PFLAGS) -c -DDOUBLE -DCOMPLEX -DNC $< -o $@ -ifdef ZCOMMKERNEL -$(KDIR)zcomm_kernel_b$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) $(ZGEMMDEPEND) +ifdef ZSYMMKERNEL +$(KDIR)zsymm_kernel_b$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZSYMMKERNEL) $(ZGEMMDEPEND) +else +$(KDIR)zsymm_kernel_b$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +endif + $(CC) $(PFLAGS) -c -DDOUBLE -DCOMPLEX -DCC $< -o $@ +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_gemm_kernel_b$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) $(ZGEMMDEPEND) else -$(KDIR)zcomm_kernel_b$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) +$(KDIR)ztrmm_gemm_kernel_b$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) $(ZGEMMDEPEND) endif $(CC) $(PFLAGS) -c -DDOUBLE -DCOMPLEX -DCC $< -o $@ @@ -3904,29 +4289,29 @@ $(KDIR)xgemm_kernel_r$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(XGEMMKERNEL) $(XGEMM $(KDIR)xgemm_kernel_b$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(XGEMMKERNEL) $(XGEMMDEPEND) $(CC) $(PFLAGS) -c -DXDOUBLE -DCOMPLEX -DCC $< -o $@ -ifdef SCOMMKERNEL -$(KDIR)strmm_kernel_LN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SCOMMKERNEL) +ifdef STRMMGEMMKERNEL +$(KDIR)strmm_kernel_LN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(STRMMGEMMKERNEL) else $(KDIR)strmm_kernel_LN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -DLEFT -UTRANSA $< -o $@ -ifdef SCOMMKERNEL -$(KDIR)strmm_kernel_LT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SCOMMKERNEL) +ifdef STRMMGEMMKERNEL +$(KDIR)strmm_kernel_LT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(STRMMGEMMKERNEL) else $(KDIR)strmm_kernel_LT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -DLEFT -DTRANSA $< -o $@ -ifdef SCOMMKERNEL -$(KDIR)strmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SCOMMKERNEL) +ifdef STRMMGEMMKERNEL +$(KDIR)strmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(STRMMGEMMKERNEL) else $(KDIR)strmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -UTRANSA $< -o $@ -ifdef SCOMMKERNEL -$(KDIR)strmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SCOMMKERNEL) +ifdef STRMMGEMMKERNEL +$(KDIR)strmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(STRMMGEMMKERNEL) else $(KDIR)strmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMMKERNEL) endif @@ -3939,29 +4324,29 @@ else $(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o $@ endif -ifdef DCOMMKERNEL -$(KDIR)dtrmm_kernel_LN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DCOMMKERNEL) +ifdef DTRMMGEMMKERNEL +$(KDIR)dtrmm_kernel_LN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DTRMMGEMMKERNEL) else $(KDIR)dtrmm_kernel_LN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -UTRANSA $< -o $@ -ifdef DCOMMKERNEL -$(KDIR)dtrmm_kernel_LT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DCOMMKERNEL) +ifdef DTRMMGEMMKERNEL +$(KDIR)dtrmm_kernel_LT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DTRMMGEMMKERNEL) else $(KDIR)dtrmm_kernel_LT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -DTRANSA $< -o $@ -ifdef DCOMMKERNEL -$(KDIR)dtrmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DCOMMKERNEL) +ifdef DTRMMGEMMKERNEL +$(KDIR)dtrmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DTRMMGEMMKERNEL) else $(KDIR)dtrmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -DDOUBLE -UCOMPLEX -ULEFT -UTRANSA $< -o $@ -ifdef DCOMMKERNEL -$(KDIR)dtrmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DCOMMKERNEL) +ifdef DTRMMGEMMKERNEL +$(KDIR)dtrmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DTRMMGEMMKERNEL) else $(KDIR)dtrmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DGEMMKERNEL) endif @@ -3979,113 +4364,113 @@ $(KDIR)qtrmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(QGEMMKERNEL) $(KDIR)qtrmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(QGEMMKERNEL) $(CC) $(PFLAGS) -c -DTRMMKERNEL -DXDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o $@ -ifdef CCOMMKERNEL -$(KDIR)ctrmm_kernel_LN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_kernel_LN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) else $(KDIR)ctrmm_kernel_LN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -UTRANSA -UCONJ -DNN $< -o $@ -ifdef CCOMMKERNEL -$(KDIR)ctrmm_kernel_LT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_kernel_LT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) else $(KDIR)ctrmm_kernel_LT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -DTRANSA -UCONJ -DNN $< -o $@ -ifdef CCOMMKERNEL -$(KDIR)ctrmm_kernel_LR$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_kernel_LR$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) else $(KDIR)ctrmm_kernel_LR$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -UTRANSA -DCONJ -DCN $< -o $@ -ifdef CCOMMKERNEL -$(KDIR)ctrmm_kernel_LC$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_kernel_LC$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) else $(KDIR)ctrmm_kernel_LC$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -DLEFT -DTRANSA -DCONJ -DCN $< -o $@ -ifdef CCOMMKERNEL -$(KDIR)ctrmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) else $(KDIR)ctrmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -UTRANSA -UCONJ -DNN $< -o $@ -ifdef CCOMMKERNEL -$(KDIR)ctrmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) else $(KDIR)ctrmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -DTRANSA -UCONJ -DNN $< -o $@ -ifdef CCOMMKERNEL -$(KDIR)ctrmm_kernel_RR$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_kernel_RR$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) else $(KDIR)ctrmm_kernel_RR$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -UTRANSA -DCONJ -DNC $< -o $@ -ifdef CCOMMKERNEL -$(KDIR)ctrmm_kernel_RC$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CCOMMKERNEL) +ifdef CTRMMGEMMKERNEL +$(KDIR)ctrmm_kernel_RC$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CTRMMGEMMKERNEL) else $(KDIR)ctrmm_kernel_RC$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(CGEMMKERNEL) endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -UDOUBLE -DCOMPLEX -ULEFT -DTRANSA -DCONJ -DNC $< -o $@ -ifdef ZCOMMKERNEL -$(KDIR)ztrmm_kernel_LN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_kernel_LN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) else $(KDIR)ztrmm_kernel_LN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -UTRANSA -UCONJ -DNN $< -o $@ -ifdef ZCOMMKERNEL -$(KDIR)ztrmm_kernel_LT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_kernel_LT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) else $(KDIR)ztrmm_kernel_LT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -DTRANSA -UCONJ -DNN $< -o $@ -ifdef ZCOMMKERNEL -$(KDIR)ztrmm_kernel_LR$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_kernel_LR$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) else $(KDIR)ztrmm_kernel_LR$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -UTRANSA -DCONJ -DCN $< -o $@ -ifdef ZCOMMKERNEL -$(KDIR)ztrmm_kernel_LC$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_kernel_LC$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) else $(KDIR)ztrmm_kernel_LC$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -DLEFT -DTRANSA -DCONJ -DCN $< -o $@ -ifdef ZCOMMKERNEL -$(KDIR)ztrmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) else $(KDIR)ztrmm_kernel_RN$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -UTRANSA -UCONJ -DNN $< -o $@ -ifdef ZCOMMKERNEL -$(KDIR)ztrmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) else $(KDIR)ztrmm_kernel_RT$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -DTRANSA -UCONJ -DNN $< -o $@ -ifdef ZCOMMKERNEL -$(KDIR)ztrmm_kernel_RR$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_kernel_RR$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) else $(KDIR)ztrmm_kernel_RR$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) endif $(CC) $(PFLAGS) -c -DTRMMKERNEL -DDOUBLE -DCOMPLEX -ULEFT -UTRANSA -DCONJ -DNC $< -o $@ -ifdef ZCOMMKERNEL -$(KDIR)ztrmm_kernel_RC$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZCOMMKERNEL) +ifdef ZTRMMGEMMKERNEL +$(KDIR)ztrmm_kernel_RC$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZTRMMGEMMKERNEL) else $(KDIR)ztrmm_kernel_RC$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZGEMMKERNEL) endif diff --git a/kernel/arm64/KERNEL.ARMV8SVE b/kernel/arm64/KERNEL.ARMV8SVE index 8457144b38..69da084b6c 100644 --- a/kernel/arm64/KERNEL.ARMV8SVE +++ b/kernel/arm64/KERNEL.ARMV8SVE @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2025, The OpenBLAS Project +# Copyright (c) 2025-2026, The OpenBLAS Project # All rights reserved. # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -151,7 +151,8 @@ DGEMM_BETA = dgemm_beta.S SGEMM_BETA = sgemm_beta.S SGEMMKERNEL = sgemm_kernel_inter_sve_v2x$(SGEMM_UNROLL_N).S -SCOMMKERNEL = sgemm_kernel_sve_v2x$(SGEMM_UNROLL_N).S +SSYMMKERNEL = sgemm_kernel_sve_v2x$(SGEMM_UNROLL_N).S +STRMMGEMMKERNEL = sgemm_kernel_sve_v2x$(SGEMM_UNROLL_N).S STRMMKERNEL = strmm_kernel_sve_v1x$(SGEMM_UNROLL_N).S SGEMMINCOPY = gemm_ncopy_sve_v2x$(SGEMM_UNROLL_N).c @@ -178,14 +179,17 @@ STRMMUNCOPY_M = trmm_uncopy_sve_v1.c STRMMLNCOPY_M = trmm_lncopy_sve_v1.c STRMMUTCOPY_M = trmm_utcopy_sve_v1.c STRMMLTCOPY_M = trmm_ltcopy_sve_v1.c -SCOMMNCOPY = gemm_ncopy_sve_v1x$(SGEMM_UNROLL_N).c -SCOMMTCOPY = gemm_tcopy_sve_v1x$(SGEMM_UNROLL_N).c +SSYMMINCOPY = gemm_ncopy_sve_v1x$(SGEMM_UNROLL_N).c +SSYMMITCOPY = gemm_tcopy_sve_v1x$(SGEMM_UNROLL_N).c +STRMMINCOPY = gemm_ncopy_sve_v1x$(SGEMM_UNROLL_N).c +STRMMITCOPY = gemm_tcopy_sve_v1x$(SGEMM_UNROLL_N).c SSYMMUCOPY_M = symm_ucopy_sve.c SSYMMLCOPY_M = symm_lcopy_sve.c DGEMMKERNEL = dgemm_kernel_inter_sve_v2x$(DGEMM_UNROLL_N).S -DCOMMKERNEL = dgemm_kernel_sve_v2x$(DGEMM_UNROLL_N).S +DSYMMKERNEL = dgemm_kernel_sve_v2x$(DGEMM_UNROLL_N).S +DTRMMGEMMKERNEL = dgemm_kernel_sve_v2x$(DGEMM_UNROLL_N).S DTRMMKERNEL = dtrmm_kernel_sve_v1x$(DGEMM_UNROLL_N).S DGEMMINCOPY = gemm_ncopy_sve_v2x$(DGEMM_UNROLL_N).c @@ -212,14 +216,17 @@ DTRMMUNCOPY_M = trmm_uncopy_sve_v1.c DTRMMLNCOPY_M = trmm_lncopy_sve_v1.c DTRMMUTCOPY_M = trmm_utcopy_sve_v1.c DTRMMLTCOPY_M = trmm_ltcopy_sve_v1.c -DCOMMNCOPY = gemm_ncopy_sve_v1x$(DGEMM_UNROLL_N).c -DCOMMTCOPY = gemm_tcopy_sve_v1x$(DGEMM_UNROLL_N).c +DSYMMINCOPY = gemm_ncopy_sve_v1x$(DGEMM_UNROLL_N).c +DSYMMITCOPY = gemm_tcopy_sve_v1x$(DGEMM_UNROLL_N).c +DTRMMINCOPY = gemm_ncopy_sve_v1x$(DGEMM_UNROLL_N).c +DTRMMITCOPY = gemm_tcopy_sve_v1x$(DGEMM_UNROLL_N).c DSYMMUCOPY_M = symm_ucopy_sve.c DSYMMLCOPY_M = symm_lcopy_sve.c CGEMMKERNEL = cgemm_kernel_sve_v1x$(ZGEMM_UNROLL_N).S -CCOMMKERNEL = cgemm_kernel_sve_v1x$(ZGEMM_UNROLL_N).S +CSYMMKERNEL = cgemm_kernel_sve_v1x$(ZGEMM_UNROLL_N).S +CTRMMGEMMKERNEL = cgemm_kernel_sve_v1x$(ZGEMM_UNROLL_N).S CTRMMKERNEL = ctrmm_kernel_sve_v1x$(ZGEMM_UNROLL_N).S CGEMMINCOPY = gemm_ncopy_complex_sve_v1x$(ZGEMM_UNROLL_N).c @@ -236,8 +243,10 @@ CTRMMUNCOPY_M = ztrmm_uncopy_sve_v1.c CTRMMLNCOPY_M = ztrmm_lncopy_sve_v1.c CTRMMUTCOPY_M = ztrmm_utcopy_sve_v1.c CTRMMLTCOPY_M = ztrmm_ltcopy_sve_v1.c -CCOMMNCOPY = gemm_ncopy_complex_sve_v1x$(ZGEMM_UNROLL_N).c -CCOMMTCOPY = gemm_tcopy_complex_sve_v1x$(ZGEMM_UNROLL_N).c +CSYMMINCOPY = gemm_ncopy_complex_sve_v1x$(ZGEMM_UNROLL_N).c +CSYMMITCOPY = gemm_tcopy_complex_sve_v1x$(ZGEMM_UNROLL_N).c +CTRMMINCOPY = gemm_ncopy_complex_sve_v1x$(ZGEMM_UNROLL_N).c +CTRMMITCOPY = gemm_tcopy_complex_sve_v1x$(ZGEMM_UNROLL_N).c CHEMMLTCOPY_M = zhemm_ltcopy_sve.c CHEMMUTCOPY_M = zhemm_utcopy_sve.c @@ -246,7 +255,8 @@ CSYMMUCOPY_M = zsymm_ucopy_sve.c CSYMMLCOPY_M = zsymm_lcopy_sve.c ZGEMMKERNEL = zgemm_kernel_sve_v1x$(ZGEMM_UNROLL_N).S -ZCOMMKERNEL = zgemm_kernel_sve_v1x$(ZGEMM_UNROLL_N).S +ZSYMMKERNEL = zgemm_kernel_sve_v1x$(ZGEMM_UNROLL_N).S +ZTRMMGEMMKERNEL = zgemm_kernel_sve_v1x$(ZGEMM_UNROLL_N).S ZTRMMKERNEL = ztrmm_kernel_sve_v1x$(ZGEMM_UNROLL_N).S ZGEMMINCOPY = gemm_ncopy_complex_sve_v1x$(ZGEMM_UNROLL_N).c @@ -263,8 +273,10 @@ ZTRMMUNCOPY_M = ztrmm_uncopy_sve_v1.c ZTRMMLNCOPY_M = ztrmm_lncopy_sve_v1.c ZTRMMUTCOPY_M = ztrmm_utcopy_sve_v1.c ZTRMMLTCOPY_M = ztrmm_ltcopy_sve_v1.c -ZCOMMNCOPY = gemm_ncopy_complex_sve_v1x$(ZGEMM_UNROLL_N).c -ZCOMMTCOPY = gemm_tcopy_complex_sve_v1x$(ZGEMM_UNROLL_N).c +ZSYMMINCOPY = gemm_ncopy_complex_sve_v1x$(ZGEMM_UNROLL_N).c +ZSYMMITCOPY = gemm_tcopy_complex_sve_v1x$(ZGEMM_UNROLL_N).c +ZTRMMINCOPY = gemm_ncopy_complex_sve_v1x$(ZGEMM_UNROLL_N).c +ZTRMMITCOPY = gemm_tcopy_complex_sve_v1x$(ZGEMM_UNROLL_N).c ZHEMMLTCOPY_M = zhemm_ltcopy_sve.c ZHEMMUTCOPY_M = zhemm_utcopy_sve.c diff --git a/kernel/arm64/dgemm_kernel_inter_sve_v2x8.S b/kernel/arm64/dgemm_kernel_inter_sve_v2x8.S index 8de5626e0f..dfad8dbfea 100644 --- a/kernel/arm64/dgemm_kernel_inter_sve_v2x8.S +++ b/kernel/arm64/dgemm_kernel_inter_sve_v2x8.S @@ -1,5 +1,5 @@ /*******************************************************************************/ -/* Copyright (c) 2025, The OpenBLAS Project */ +/* Copyright (c) 2025-2026, The OpenBLAS Project */ /* All rights reserved. */ /* Redistribution and use in source and binary forms, with or without */ /* modification, are permitted provided that the following conditions are */ diff --git a/kernel/arm64/gemm_ncopy_sve_v2x8.c b/kernel/arm64/gemm_ncopy_sve_v2x8.c index 8130889d47..3192b70482 100644 --- a/kernel/arm64/gemm_ncopy_sve_v2x8.c +++ b/kernel/arm64/gemm_ncopy_sve_v2x8.c @@ -1,5 +1,5 @@ /*******************************************************************************/ -/* Copyright (c) 2023, 2025, The OpenBLAS Project */ +/* Copyright (c) 2023, 2025-2026, The OpenBLAS Project */ /* All rights reserved. */ /* Redistribution and use in source and binary forms, with or without */ /* modification, are permitted provided that the following conditions are */ diff --git a/kernel/arm64/gemm_tcopy_sve_v2x8.c b/kernel/arm64/gemm_tcopy_sve_v2x8.c index 7f350ed67b..236c856b51 100644 --- a/kernel/arm64/gemm_tcopy_sve_v2x8.c +++ b/kernel/arm64/gemm_tcopy_sve_v2x8.c @@ -1,5 +1,5 @@ /*******************************************************************************/ -/* Copyright (c) 2023, 2025, The OpenBLAS Project */ +/* Copyright (c) 2023, 2025-2026, The OpenBLAS Project */ /* All rights reserved. */ /* Redistribution and use in source and binary forms, with or without */ /* modification, are permitted provided that the following conditions are */ diff --git a/kernel/arm64/sgemm_kernel_inter_sve_v2x8.S b/kernel/arm64/sgemm_kernel_inter_sve_v2x8.S index 2ea1850fe9..f8c4aa61c1 100644 --- a/kernel/arm64/sgemm_kernel_inter_sve_v2x8.S +++ b/kernel/arm64/sgemm_kernel_inter_sve_v2x8.S @@ -1,5 +1,5 @@ /*******************************************************************************/ -/* Copyright (c) 2025, The OpenBLAS Project */ +/* Copyright (c) 2025-2026, The OpenBLAS Project */ /* All rights reserved. */ /* Redistribution and use in source and binary forms, with or without */ /* modification, are permitted provided that the following conditions are */ diff --git a/kernel/setparam-ref.c b/kernel/setparam-ref.c index f8251240a7..6993062ed2 100644 --- a/kernel/setparam-ref.c +++ b/kernel/setparam-ref.c @@ -1,6 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ -/* Copyright 2023, 2025 The OpenBLAS Project. */ +/* Copyright 2023, 2025-2026 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -246,7 +246,7 @@ gotoblas_t TABLE_NAME = { #endif #endif - sgemm_kernelTS, sgemm_betaTS, scomm_kernelTS, + sgemm_kernelTS, sgemm_betaTS, ssymm_kernelTS, strmm_gemm_kernelTS, #if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N sgemm_incopyTS, sgemm_itcopyTS, #else @@ -284,7 +284,8 @@ gotoblas_t TABLE_NAME = { strmm_ounucopyTS, strmm_ounncopyTS, strmm_outucopyTS, strmm_outncopyTS, strmm_olnucopyTS, strmm_olnncopyTS, strmm_oltucopyTS, strmm_oltncopyTS, #endif - scomm_ncopyTS, scomm_tcopyTS, + ssymm_incopyTS, ssymm_itcopyTS, + strmm_incopyTS, strmm_itcopyTS, strmm_ounucopyTS, strmm_ounncopyTS, strmm_outucopyTS, strmm_outncopyTS, strmm_olnucopyTS, strmm_olnncopyTS, strmm_oltucopyTS, strmm_oltncopyTS, #if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N @@ -346,7 +347,7 @@ gotoblas_t TABLE_NAME = { NULL, #endif #endif - dgemm_kernelTS, dgemm_betaTS, dcomm_kernelTS, + dgemm_kernelTS, dgemm_betaTS, dsymm_kernelTS, dtrmm_gemm_kernelTS, #if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N dgemm_incopyTS, dgemm_itcopyTS, #else @@ -381,7 +382,8 @@ gotoblas_t TABLE_NAME = { dtrmm_ounucopyTS, dtrmm_ounncopyTS, dtrmm_outucopyTS, dtrmm_outncopyTS, dtrmm_olnucopyTS, dtrmm_olnncopyTS, dtrmm_oltucopyTS, dtrmm_oltncopyTS, #endif - dcomm_ncopyTS, dcomm_tcopyTS, + dsymm_incopyTS, dsymm_itcopyTS, + dtrmm_incopyTS, dtrmm_itcopyTS, dtrmm_ounucopyTS, dtrmm_ounncopyTS, dtrmm_outucopyTS, dtrmm_outncopyTS, dtrmm_olnucopyTS, dtrmm_olnncopyTS, dtrmm_oltucopyTS, dtrmm_oltncopyTS, #if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N @@ -500,7 +502,8 @@ gotoblas_t TABLE_NAME = { #endif cgemm_kernel_nTS, cgemm_kernel_lTS, cgemm_kernel_rTS, cgemm_kernel_bTS, cgemm_betaTS, - ccomm_kernel_nTS, ccomm_kernel_lTS, ccomm_kernel_rTS, ccomm_kernel_bTS, + csymm_kernel_nTS, csymm_kernel_lTS, csymm_kernel_rTS, csymm_kernel_bTS, + ctrmm_gemm_kernel_nTS, ctrmm_gemm_kernel_lTS, ctrmm_gemm_kernel_rTS, ctrmm_gemm_kernel_bTS, #if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N cgemm_incopyTS, cgemm_itcopyTS, #else @@ -546,7 +549,8 @@ gotoblas_t TABLE_NAME = { ctrmm_ounucopyTS, ctrmm_ounncopyTS, ctrmm_outucopyTS, ctrmm_outncopyTS, ctrmm_olnucopyTS, ctrmm_olnncopyTS, ctrmm_oltucopyTS, ctrmm_oltncopyTS, #endif - ccomm_ncopyTS, ccomm_tcopyTS, + csymm_incopyTS, csymm_itcopyTS, + ctrmm_incopyTS, ctrmm_itcopyTS, ctrmm_ounucopyTS, ctrmm_ounncopyTS, ctrmm_outucopyTS, ctrmm_outncopyTS, ctrmm_olnucopyTS, ctrmm_olnncopyTS, ctrmm_oltucopyTS, ctrmm_oltncopyTS, @@ -664,7 +668,8 @@ gotoblas_t TABLE_NAME = { #endif zgemm_kernel_nTS, zgemm_kernel_lTS, zgemm_kernel_rTS, zgemm_kernel_bTS, zgemm_betaTS, - zcomm_kernel_nTS, zcomm_kernel_lTS, zcomm_kernel_rTS, zcomm_kernel_bTS, + zsymm_kernel_nTS, zsymm_kernel_lTS, zsymm_kernel_rTS, zsymm_kernel_bTS, + ztrmm_gemm_kernel_nTS, ztrmm_gemm_kernel_lTS, ztrmm_gemm_kernel_rTS, ztrmm_gemm_kernel_bTS, #if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N zgemm_incopyTS, zgemm_itcopyTS, @@ -708,7 +713,8 @@ gotoblas_t TABLE_NAME = { ztrmm_ounucopyTS, ztrmm_ounncopyTS, ztrmm_outucopyTS, ztrmm_outncopyTS, ztrmm_olnucopyTS, ztrmm_olnncopyTS, ztrmm_oltucopyTS, ztrmm_oltncopyTS, #endif - zcomm_ncopyTS, zcomm_tcopyTS, + zsymm_incopyTS, zsymm_itcopyTS, + ztrmm_incopyTS, ztrmm_itcopyTS, ztrmm_ounucopyTS, ztrmm_ounncopyTS, ztrmm_outucopyTS, ztrmm_outncopyTS, ztrmm_olnucopyTS, ztrmm_olnncopyTS, ztrmm_oltucopyTS, ztrmm_oltncopyTS,