diff --git a/.gitignore b/.gitignore index 3d6ddb5052..2b9f4c869d 100644 --- a/.gitignore +++ b/.gitignore @@ -120,3 +120,4 @@ CMakeCache.txt CMakeFiles/* .vscode **/__pycache__ +dyn_cores.h diff --git a/Makefile b/Makefile index ae1da5e9fe..8a117f9927 100644 --- a/Makefile +++ b/Makefile @@ -449,7 +449,7 @@ clean :: ifeq ($(OSNAME), Darwin) @rm -rf getarch.dSYM getarch_2nd.dSYM endif - @rm -f Makefile.conf config.h Makefile_kernel.conf config_kernel.h st* *.dylib + @rm -f Makefile.conf config.h Makefile_kernel.conf config_kernel.h dyn_cores.h st* *.dylib @rm -f cblas.tmp cblas.tmp2 @touch $(NETLIB_LAPACK_DIR)/make.inc @$(MAKE) -C $(NETLIB_LAPACK_DIR) clean diff --git a/Makefile.system b/Makefile.system index e8e732e9c8..6c0c4ae0de 100644 --- a/Makefile.system +++ b/Makefile.system @@ -867,6 +867,15 @@ override DYNAMIC_ARCH= endif endif +# dyn_cores.h tells common_param.h which cores this DYNAMIC_ARCH build has. +ifeq ($(DYNAMIC_ARCH), 1) +ifndef OPENBLAS_DYN_CORES +export OPENBLAS_DYN_CORES := $(sort $(DYNAMIC_CORE)) +HASH := \# +DUMMY := $(shell echo '$(HASH)define OPENBLAS_CORE_LIST(X, arg) $(foreach c,$(OPENBLAS_DYN_CORES),X($(c), arg))' > $(TOPDIR)/dyn_cores.h) +endif +endif + ifeq ($(ARCH), ia64) NO_BINARY_MODE = 1 BINARY_DEFINED = 1 @@ -1484,6 +1493,9 @@ endif ifeq ($(DYNAMIC_ARCH), 1) CCOMMON_OPT += -DDYNAMIC_ARCH +ifeq ($(C_COMPILER), $(filter $(C_COMPILER),GCC LSB CLANG INTEL)) +CCOMMON_OPT += -ffunction-sections -fdata-sections +endif endif ifeq ($(DYNAMIC_OLDER), 1) diff --git a/cmake/arch.cmake b/cmake/arch.cmake index 12d7de54e7..e74c353204 100644 --- a/cmake/arch.cmake +++ b/cmake/arch.cmake @@ -129,9 +129,26 @@ if (DYNAMIC_ARCH) message (FATAL_ERROR "Your build directory contains a file config_kernel.h, probably from a previous compilation with make. This will conflict with the cmake compilation and cause strange compiler errors - please remove the file before trying again") endif () + if (EXISTS ${PROJECT_SOURCE_DIR}/dyn_cores.h) + message (FATAL_ERROR "Your source directory contains a file dyn_cores.h, probably from a previous compilation with make. It would be used instead of the one generated for this build - please remove the file before trying again") + endif () + if (NOT DYNAMIC_CORE) message (STATUS "DYNAMIC_ARCH is not supported on this architecture, removing from options") unset(DYNAMIC_ARCH CACHE) + else () + # dyn_cores.h tells common_param.h which cores this build has; + # see the equivalent rule in Makefile.system. + set(DYN_CORES_LIST ${DYNAMIC_CORE}) + list(REMOVE_DUPLICATES DYN_CORES_LIST) + list(SORT DYN_CORES_LIST) + set(DYN_CORES_H "#define OPENBLAS_CORE_LIST(X, arg)") + foreach (DCORE ${DYN_CORES_LIST}) + string(APPEND DYN_CORES_H " X(${DCORE}, arg)") + endforeach () + file(WRITE ${PROJECT_BINARY_DIR}/dyn_cores.h.tmp "${DYN_CORES_H}\n") + configure_file(${PROJECT_BINARY_DIR}/dyn_cores.h.tmp ${PROJECT_BINARY_DIR}/dyn_cores.h COPYONLY) + file(REMOVE ${PROJECT_BINARY_DIR}/dyn_cores.h.tmp) endif () endif () diff --git a/cmake/system.cmake b/cmake/system.cmake index 436f2109aa..5953d27301 100644 --- a/cmake/system.cmake +++ b/cmake/system.cmake @@ -536,6 +536,9 @@ endif () if (DYNAMIC_ARCH) if (X86 OR X86_64 OR ARM64 OR POWER OR RISCV64 OR LOONGARCH64 OR ZARCH) set(CCOMMON_OPT "${CCOMMON_OPT} -DDYNAMIC_ARCH") + if (CMAKE_C_COMPILER_ID MATCHES "GNU|Clang|Intel" AND NOT MSVC) + set(CCOMMON_OPT "${CCOMMON_OPT} -ffunction-sections -fdata-sections") + endif () if (DYNAMIC_OLDER) set(CCOMMON_OPT "${CCOMMON_OPT} -DDYNAMIC_OLDER") endif () diff --git a/common_b.h b/common_b.h index 1921c3a69d..780dcbc240 100644 --- a/common_b.h +++ b/common_b.h @@ -50,17 +50,17 @@ #define BGEMM_KERNEL bgemm_kernel #else -#define BGEMV_N_K gotoblas->bgemv_n -#define BGEMV_T_K gotoblas->bgemv_t +#define BGEMV_N_K OPENBLAS_DISPATCH(bgemv)->bgemv_n +#define BGEMV_T_K OPENBLAS_DISPATCH(bgemv)->bgemv_t -#define BSCAL_K gotoblas->bscal_k +#define BSCAL_K OPENBLAS_DISPATCH(bscal)->bscal_k -#define BGEMM_ONCOPY gotoblas->bgemm_oncopy -#define BGEMM_OTCOPY gotoblas->bgemm_otcopy -#define BGEMM_INCOPY gotoblas->bgemm_incopy -#define BGEMM_ITCOPY gotoblas->bgemm_itcopy -#define BGEMM_BETA gotoblas->bgemm_beta -#define BGEMM_KERNEL gotoblas->bgemm_kernel +#define BGEMM_ONCOPY OPENBLAS_DISPATCH(bgemm)->bgemm_oncopy +#define BGEMM_OTCOPY OPENBLAS_DISPATCH(bgemm)->bgemm_otcopy +#define BGEMM_INCOPY OPENBLAS_DISPATCH(bgemm)->bgemm_incopy +#define BGEMM_ITCOPY OPENBLAS_DISPATCH(bgemm)->bgemm_itcopy +#define BGEMM_BETA OPENBLAS_DISPATCH(bgemm)->bgemm_beta +#define BGEMM_KERNEL OPENBLAS_DISPATCH(bgemm)->bgemm_kernel #endif diff --git a/common_c.h b/common_c.h index f12561127b..40f30afea7 100644 --- a/common_c.h +++ b/common_c.h @@ -290,46 +290,46 @@ #else -#define CAMAX_K gotoblas -> camax_k -#define CAMIN_K gotoblas -> camin_k +#define CAMAX_K OPENBLAS_DISPATCH(camax) -> camax_k +#define CAMIN_K OPENBLAS_DISPATCH(camin) -> camin_k #define CMAX_K gotoblas -> cmax_k #define CMIN_K gotoblas -> cmin_k -#define ICAMAX_K gotoblas -> icamax_k -#define ICAMIN_K gotoblas -> icamin_k +#define ICAMAX_K OPENBLAS_DISPATCH(icamax) -> icamax_k +#define ICAMIN_K OPENBLAS_DISPATCH(icamin) -> icamin_k #define ICMAX_K gotoblas -> icmax_k #define ICMIN_K gotoblas -> icmin_k -#define CASUM_K gotoblas -> casum_k -#define CAXPYU_K gotoblas -> caxpy_k -#define CAXPYC_K gotoblas -> caxpyc_k -#define CCOPY_K gotoblas -> ccopy_k -#define CDOTU_K gotoblas -> cdotu_k -#define CDOTC_K gotoblas -> cdotc_k -#define CNRM2_K gotoblas -> cnrm2_k -#define CSCAL_K gotoblas -> cscal_k -#define CSUM_K gotoblas -> csum_k -#define CSWAP_K gotoblas -> cswap_k -#define CROT_K gotoblas -> csrot_k - -#define CGEMV_N gotoblas -> cgemv_n -#define CGEMV_T gotoblas -> cgemv_t -#define CGEMV_R gotoblas -> cgemv_r -#define CGEMV_C gotoblas -> cgemv_c -#define CGEMV_O gotoblas -> cgemv_o -#define CGEMV_U gotoblas -> cgemv_u -#define CGEMV_S gotoblas -> cgemv_s -#define CGEMV_D gotoblas -> cgemv_d - -#define CGERU_K gotoblas -> cgeru_k -#define CGERC_K gotoblas -> cgerc_k -#define CGERV_K gotoblas -> cgerv_k -#define CGERD_K gotoblas -> cgerd_k - -#define CSYMV_U gotoblas -> csymv_U -#define CSYMV_L gotoblas -> csymv_L -#define CHEMV_U gotoblas -> chemv_U -#define CHEMV_L gotoblas -> chemv_L -#define CHEMV_V gotoblas -> chemv_V -#define CHEMV_M gotoblas -> chemv_M +#define CASUM_K OPENBLAS_DISPATCH(casum) -> casum_k +#define CAXPYU_K OPENBLAS_DISPATCH(caxpy) -> caxpy_k +#define CAXPYC_K OPENBLAS_DISPATCH(caxpyc) -> caxpyc_k +#define CCOPY_K OPENBLAS_DISPATCH(ccopy) -> ccopy_k +#define CDOTU_K OPENBLAS_DISPATCH(cdotu) -> cdotu_k +#define CDOTC_K OPENBLAS_DISPATCH(cdotc) -> cdotc_k +#define CNRM2_K OPENBLAS_DISPATCH(cnrm2) -> cnrm2_k +#define CSCAL_K OPENBLAS_DISPATCH(cscal) -> cscal_k +#define CSUM_K OPENBLAS_DISPATCH(csum) -> csum_k +#define CSWAP_K OPENBLAS_DISPATCH(cswap) -> cswap_k +#define CROT_K OPENBLAS_DISPATCH(csrot) -> csrot_k + +#define CGEMV_N OPENBLAS_DISPATCH(cgemv) -> cgemv_n +#define CGEMV_T OPENBLAS_DISPATCH(cgemv) -> cgemv_t +#define CGEMV_R OPENBLAS_DISPATCH(cgemv) -> cgemv_r +#define CGEMV_C OPENBLAS_DISPATCH(cgemv) -> cgemv_c +#define CGEMV_O OPENBLAS_DISPATCH(cgemv) -> cgemv_o +#define CGEMV_U OPENBLAS_DISPATCH(cgemv) -> cgemv_u +#define CGEMV_S OPENBLAS_DISPATCH(cgemv) -> cgemv_s +#define CGEMV_D OPENBLAS_DISPATCH(cgemv) -> cgemv_d + +#define CGERU_K OPENBLAS_DISPATCH(cgeru) -> cgeru_k +#define CGERC_K OPENBLAS_DISPATCH(cgerc) -> cgerc_k +#define CGERV_K OPENBLAS_DISPATCH(cgerv) -> cgerv_k +#define CGERD_K OPENBLAS_DISPATCH(cgerd) -> cgerd_k + +#define CSYMV_U OPENBLAS_DISPATCH(csymv) -> csymv_U +#define CSYMV_L OPENBLAS_DISPATCH(csymv) -> csymv_L +#define CHEMV_U OPENBLAS_DISPATCH(chemv) -> chemv_U +#define CHEMV_L OPENBLAS_DISPATCH(chemv) -> chemv_L +#define CHEMV_V OPENBLAS_DISPATCH(chemv) -> chemv_V +#define CHEMV_M OPENBLAS_DISPATCH(chemv) -> chemv_M #define CSYMV_THREAD_U csymv_thread_U #define CSYMV_THREAD_L csymv_thread_L @@ -338,208 +338,209 @@ #define CHEMV_THREAD_V chemv_thread_V #define CHEMV_THREAD_M chemv_thread_M -#define CGEMM_ONCOPY gotoblas -> cgemm_oncopy -#define CGEMM_OTCOPY gotoblas -> cgemm_otcopy -#define CGEMM_INCOPY gotoblas -> cgemm_incopy -#define CGEMM_ITCOPY gotoblas -> cgemm_itcopy - -#define CTRMM_OUNUCOPY gotoblas -> ctrmm_ounucopy -#define CTRMM_OUTUCOPY gotoblas -> ctrmm_outucopy -#define CTRMM_OLNUCOPY gotoblas -> ctrmm_olnucopy -#define CTRMM_OLTUCOPY gotoblas -> ctrmm_oltucopy -#define CTRSM_OUNUCOPY gotoblas -> ctrsm_ounucopy -#define CTRSM_OUTUCOPY gotoblas -> ctrsm_outucopy -#define CTRSM_OLNUCOPY gotoblas -> ctrsm_olnucopy -#define CTRSM_OLTUCOPY gotoblas -> ctrsm_oltucopy - -#define CTRMM_IUNUCOPY gotoblas -> ctrmm_iunucopy -#define CTRMM_IUTUCOPY gotoblas -> ctrmm_iutucopy -#define CTRMM_ILNUCOPY gotoblas -> ctrmm_ilnucopy -#define CTRMM_ILTUCOPY gotoblas -> ctrmm_iltucopy -#define CTRSM_IUNUCOPY gotoblas -> ctrsm_iunucopy -#define CTRSM_IUTUCOPY gotoblas -> ctrsm_iutucopy -#define CTRSM_ILNUCOPY gotoblas -> ctrsm_ilnucopy -#define CTRSM_ILTUCOPY gotoblas -> ctrsm_iltucopy - -#define CTRMM_OUNNCOPY gotoblas -> ctrmm_ounncopy -#define CTRMM_OUTNCOPY gotoblas -> ctrmm_outncopy -#define CTRMM_OLNNCOPY gotoblas -> ctrmm_olnncopy -#define CTRMM_OLTNCOPY gotoblas -> ctrmm_oltncopy -#define CTRSM_OUNNCOPY gotoblas -> ctrsm_ounncopy -#define CTRSM_OUTNCOPY gotoblas -> ctrsm_outncopy -#define CTRSM_OLNNCOPY gotoblas -> ctrsm_olnncopy -#define CTRSM_OLTNCOPY gotoblas -> ctrsm_oltncopy - -#define CTRMM_IUNNCOPY gotoblas -> ctrmm_iunncopy -#define CTRMM_IUTNCOPY gotoblas -> ctrmm_iutncopy -#define CTRMM_ILNNCOPY gotoblas -> ctrmm_ilnncopy -#define CTRMM_ILTNCOPY gotoblas -> ctrmm_iltncopy -#define CSYMM_INCOPY gotoblas -> csymm_incopy -#define CSYMM_ITCOPY gotoblas -> csymm_itcopy -#define CTRMM_INCOPY gotoblas -> ctrmm_incopy -#define CTRMM_ITCOPY gotoblas -> ctrmm_itcopy - -#define CTRSM_IUNNCOPY gotoblas -> ctrsm_iunncopy -#define CTRSM_IUTNCOPY gotoblas -> ctrsm_iutncopy -#define CTRSM_ILNNCOPY gotoblas -> ctrsm_ilnncopy -#define CTRSM_ILTNCOPY gotoblas -> ctrsm_iltncopy - -#define SME_CGEMM_KERNEL gotoblas -> sme_cgemm_kernel -#define CGEMM_BETA gotoblas -> cgemm_beta -#define CGEMM_KERNEL_N gotoblas -> cgemm_kernel_n -#define CGEMM_KERNEL_L gotoblas -> cgemm_kernel_l -#define CGEMM_KERNEL_R gotoblas -> cgemm_kernel_r -#define CGEMM_KERNEL_B gotoblas -> cgemm_kernel_b - -#define CSYMM_KERNEL_N gotoblas -> csymm_kernel_n -#define CSYMM_KERNEL_L gotoblas -> csymm_kernel_l -#define CSYMM_KERNEL_R gotoblas -> csymm_kernel_r -#define CSYMM_KERNEL_B gotoblas -> csymm_kernel_b -#define CTRMM_GEMM_KERNEL_N gotoblas -> ctrmm_gemm_kernel_n -#define CTRMM_GEMM_KERNEL_L gotoblas -> ctrmm_gemm_kernel_l -#define CTRMM_GEMM_KERNEL_R gotoblas -> ctrmm_gemm_kernel_r -#define CTRMM_GEMM_KERNEL_B gotoblas -> ctrmm_gemm_kernel_b - -#define CTRMM_KERNEL_LN gotoblas -> ctrmm_kernel_LN -#define CTRMM_KERNEL_LT gotoblas -> ctrmm_kernel_LT -#define CTRMM_KERNEL_LR gotoblas -> ctrmm_kernel_LR -#define CTRMM_KERNEL_LC gotoblas -> ctrmm_kernel_LC -#define CTRMM_KERNEL_RN gotoblas -> ctrmm_kernel_RN -#define CTRMM_KERNEL_RT gotoblas -> ctrmm_kernel_RT -#define CTRMM_KERNEL_RR gotoblas -> ctrmm_kernel_RR -#define CTRMM_KERNEL_RC gotoblas -> ctrmm_kernel_RC - -#define CTRSM_KERNEL_LN gotoblas -> ctrsm_kernel_LN -#define CTRSM_KERNEL_LT gotoblas -> ctrsm_kernel_LT -#define CTRSM_KERNEL_LR gotoblas -> ctrsm_kernel_LR -#define CTRSM_KERNEL_LC gotoblas -> ctrsm_kernel_LC -#define CTRSM_KERNEL_RN gotoblas -> ctrsm_kernel_RN -#define CTRSM_KERNEL_RT gotoblas -> ctrsm_kernel_RT -#define CTRSM_KERNEL_RR gotoblas -> ctrsm_kernel_RR -#define CTRSM_KERNEL_RC gotoblas -> ctrsm_kernel_RC - -#define CSYMM_IUTCOPY gotoblas -> csymm_iutcopy -#define CSYMM_ILTCOPY gotoblas -> csymm_iltcopy -#define CSYMM_OUTCOPY gotoblas -> csymm_outcopy -#define CSYMM_OLTCOPY gotoblas -> csymm_oltcopy - -#define CHEMM_OUTCOPY gotoblas -> chemm_outcopy -#define CHEMM_OLTCOPY gotoblas -> chemm_oltcopy -#define CHEMM_IUTCOPY gotoblas -> chemm_iutcopy -#define CHEMM_ILTCOPY gotoblas -> chemm_iltcopy - -#define CGEMM3M_ONCOPYB gotoblas -> cgemm3m_oncopyb -#define CGEMM3M_ONCOPYR gotoblas -> cgemm3m_oncopyr -#define CGEMM3M_ONCOPYI gotoblas -> cgemm3m_oncopyi -#define CGEMM3M_OTCOPYB gotoblas -> cgemm3m_otcopyb -#define CGEMM3M_OTCOPYR gotoblas -> cgemm3m_otcopyr -#define CGEMM3M_OTCOPYI gotoblas -> cgemm3m_otcopyi - -#define CGEMM3M_INCOPYB gotoblas -> cgemm3m_incopyb -#define CGEMM3M_INCOPYR gotoblas -> cgemm3m_incopyr -#define CGEMM3M_INCOPYI gotoblas -> cgemm3m_incopyi -#define CGEMM3M_ITCOPYB gotoblas -> cgemm3m_itcopyb -#define CGEMM3M_ITCOPYR gotoblas -> cgemm3m_itcopyr -#define CGEMM3M_ITCOPYI gotoblas -> cgemm3m_itcopyi - -#define CSYMM3M_ILCOPYB gotoblas -> csymm3m_ilcopyb -#define CSYMM3M_IUCOPYB gotoblas -> csymm3m_iucopyb -#define CSYMM3M_ILCOPYR gotoblas -> csymm3m_ilcopyr -#define CSYMM3M_IUCOPYR gotoblas -> csymm3m_iucopyr -#define CSYMM3M_ILCOPYI gotoblas -> csymm3m_ilcopyi -#define CSYMM3M_IUCOPYI gotoblas -> csymm3m_iucopyi - -#define CSYMM3M_OLCOPYB gotoblas -> csymm3m_olcopyb -#define CSYMM3M_OUCOPYB gotoblas -> csymm3m_oucopyb -#define CSYMM3M_OLCOPYR gotoblas -> csymm3m_olcopyr -#define CSYMM3M_OUCOPYR gotoblas -> csymm3m_oucopyr -#define CSYMM3M_OLCOPYI gotoblas -> csymm3m_olcopyi -#define CSYMM3M_OUCOPYI gotoblas -> csymm3m_oucopyi - -#define CHEMM3M_ILCOPYB gotoblas -> chemm3m_ilcopyb -#define CHEMM3M_IUCOPYB gotoblas -> chemm3m_iucopyb -#define CHEMM3M_ILCOPYR gotoblas -> chemm3m_ilcopyr -#define CHEMM3M_IUCOPYR gotoblas -> chemm3m_iucopyr -#define CHEMM3M_ILCOPYI gotoblas -> chemm3m_ilcopyi -#define CHEMM3M_IUCOPYI gotoblas -> chemm3m_iucopyi - -#define CHEMM3M_OLCOPYB gotoblas -> chemm3m_olcopyb -#define CHEMM3M_OUCOPYB gotoblas -> chemm3m_oucopyb -#define CHEMM3M_OLCOPYR gotoblas -> chemm3m_olcopyr -#define CHEMM3M_OUCOPYR gotoblas -> chemm3m_oucopyr -#define CHEMM3M_OLCOPYI gotoblas -> chemm3m_olcopyi -#define CHEMM3M_OUCOPYI gotoblas -> chemm3m_oucopyi - -#define CGEMM3M_KERNEL gotoblas -> cgemm3m_kernel - -#define CNEG_TCOPY gotoblas -> cneg_tcopy -#define CLASWP_NCOPY gotoblas -> claswp_ncopy - -#define CAXPBY_K gotoblas -> caxpby_k - -#define COMATCOPY_K_CN gotoblas -> comatcopy_k_cn -#define COMATCOPY_K_RN gotoblas -> comatcopy_k_rn -#define COMATCOPY_K_CT gotoblas -> comatcopy_k_ct -#define COMATCOPY_K_RT gotoblas -> comatcopy_k_rt -#define COMATCOPY_K_CNC gotoblas -> comatcopy_k_cnc -#define COMATCOPY_K_RNC gotoblas -> comatcopy_k_rnc -#define COMATCOPY_K_CTC gotoblas -> comatcopy_k_ctc -#define COMATCOPY_K_RTC gotoblas -> comatcopy_k_rtc - -#define CIMATCOPY_K_CN gotoblas -> cimatcopy_k_cn -#define CIMATCOPY_K_RN gotoblas -> cimatcopy_k_rn -#define CIMATCOPY_K_CT gotoblas -> cimatcopy_k_ct -#define CIMATCOPY_K_RT gotoblas -> cimatcopy_k_rt -#define CIMATCOPY_K_CNC gotoblas -> cimatcopy_k_cnc -#define CIMATCOPY_K_RNC gotoblas -> cimatcopy_k_rnc -#define CIMATCOPY_K_CTC gotoblas -> cimatcopy_k_ctc -#define CIMATCOPY_K_RTC gotoblas -> cimatcopy_k_rtc - -#define CGEADD_K gotoblas -> cgeadd_k - -#define CGEMM_SMALL_MATRIX_PERMIT gotoblas -> cgemm_small_matrix_permit +#define CGEMM_ONCOPY OPENBLAS_DISPATCH(cgemm) -> cgemm_oncopy +#define CGEMM_OTCOPY OPENBLAS_DISPATCH(cgemm) -> cgemm_otcopy +#define CGEMM_INCOPY OPENBLAS_DISPATCH(cgemm) -> cgemm_incopy +#define CGEMM_ITCOPY OPENBLAS_DISPATCH(cgemm) -> cgemm_itcopy + +#define CTRMM_OUNUCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_ounucopy +#define CTRMM_OUTUCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_outucopy +#define CTRMM_OLNUCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_olnucopy +#define CTRMM_OLTUCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_oltucopy +#define CTRSM_OUNUCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_ounucopy +#define CTRSM_OUTUCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_outucopy +#define CTRSM_OLNUCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_olnucopy +#define CTRSM_OLTUCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_oltucopy + +#define CTRMM_IUNUCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_iunucopy +#define CTRMM_IUTUCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_iutucopy +#define CTRMM_ILNUCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_ilnucopy +#define CTRMM_ILTUCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_iltucopy +#define CTRSM_IUNUCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_iunucopy +#define CTRSM_IUTUCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_iutucopy +#define CTRSM_ILNUCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_ilnucopy +#define CTRSM_ILTUCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_iltucopy + +#define CTRMM_OUNNCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_ounncopy +#define CTRMM_OUTNCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_outncopy +#define CTRMM_OLNNCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_olnncopy +#define CTRMM_OLTNCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_oltncopy +#define CTRSM_OUNNCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_ounncopy +#define CTRSM_OUTNCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_outncopy +#define CTRSM_OLNNCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_olnncopy +#define CTRSM_OLTNCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_oltncopy + +#define CTRMM_IUNNCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_iunncopy +#define CTRMM_IUTNCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_iutncopy +#define CTRMM_ILNNCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_ilnncopy +#define CTRMM_ILTNCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_iltncopy +#define CSYMM_INCOPY OPENBLAS_DISPATCH(csymm) -> csymm_incopy +#define CSYMM_ITCOPY OPENBLAS_DISPATCH(csymm) -> csymm_itcopy +#define CTRMM_INCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_incopy +#define CTRMM_ITCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_itcopy + +#define CTRSM_IUNNCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_iunncopy +#define CTRSM_IUTNCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_iutncopy +#define CTRSM_ILNNCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_ilnncopy +#define CTRSM_ILTNCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_iltncopy + +#define SME_CGEMM_KERNEL OPENBLAS_DISPATCH(cgemm) -> sme_cgemm_kernel +#define CGEMM_BETA OPENBLAS_DISPATCH(cgemm) -> cgemm_beta +#define CGEMM_KERNEL_N OPENBLAS_DISPATCH(cgemm) -> cgemm_kernel_n +#define CGEMM_KERNEL_L OPENBLAS_DISPATCH(cgemm) -> cgemm_kernel_l +#define CGEMM_KERNEL_R OPENBLAS_DISPATCH(cgemm) -> cgemm_kernel_r +#define CGEMM_KERNEL_B OPENBLAS_DISPATCH(cgemm) -> cgemm_kernel_b + +#define CSYMM_KERNEL_N OPENBLAS_DISPATCH(csymm) -> csymm_kernel_n +#define CSYMM_KERNEL_L OPENBLAS_DISPATCH(csymm) -> csymm_kernel_l +#define CSYMM_KERNEL_R OPENBLAS_DISPATCH(csymm) -> csymm_kernel_r +#define CSYMM_KERNEL_B OPENBLAS_DISPATCH(csymm) -> csymm_kernel_b +#define CTRMM_GEMM_KERNEL_N OPENBLAS_DISPATCH(ctrmm) -> ctrmm_gemm_kernel_n +#define CTRMM_GEMM_KERNEL_L OPENBLAS_DISPATCH(ctrmm) -> ctrmm_gemm_kernel_l +#define CTRMM_GEMM_KERNEL_R OPENBLAS_DISPATCH(ctrmm) -> ctrmm_gemm_kernel_r +#define CTRMM_GEMM_KERNEL_B OPENBLAS_DISPATCH(ctrmm) -> ctrmm_gemm_kernel_b + +#define CTRMM_KERNEL_LN OPENBLAS_DISPATCH(ctrmm) -> ctrmm_kernel_LN +#define CTRMM_KERNEL_LT OPENBLAS_DISPATCH(ctrmm) -> ctrmm_kernel_LT +#define CTRMM_KERNEL_LR OPENBLAS_DISPATCH(ctrmm) -> ctrmm_kernel_LR +#define CTRMM_KERNEL_LC OPENBLAS_DISPATCH(ctrmm) -> ctrmm_kernel_LC +#define CTRMM_KERNEL_RN OPENBLAS_DISPATCH(ctrmm) -> ctrmm_kernel_RN +#define CTRMM_KERNEL_RT OPENBLAS_DISPATCH(ctrmm) -> ctrmm_kernel_RT +#define CTRMM_KERNEL_RR OPENBLAS_DISPATCH(ctrmm) -> ctrmm_kernel_RR +#define CTRMM_KERNEL_RC OPENBLAS_DISPATCH(ctrmm) -> ctrmm_kernel_RC + +#define CTRSM_KERNEL_LN OPENBLAS_DISPATCH(ctrsm) -> ctrsm_kernel_LN +#define CTRSM_KERNEL_LT OPENBLAS_DISPATCH(ctrsm) -> ctrsm_kernel_LT +#define CTRSM_KERNEL_LR OPENBLAS_DISPATCH(ctrsm) -> ctrsm_kernel_LR +#define CTRSM_KERNEL_LC OPENBLAS_DISPATCH(ctrsm) -> ctrsm_kernel_LC +#define CTRSM_KERNEL_RN OPENBLAS_DISPATCH(ctrsm) -> ctrsm_kernel_RN +#define CTRSM_KERNEL_RT OPENBLAS_DISPATCH(ctrsm) -> ctrsm_kernel_RT +#define CTRSM_KERNEL_RR OPENBLAS_DISPATCH(ctrsm) -> ctrsm_kernel_RR +#define CTRSM_KERNEL_RC OPENBLAS_DISPATCH(ctrsm) -> ctrsm_kernel_RC + +#define CSYMM_IUTCOPY OPENBLAS_DISPATCH(csymm) -> csymm_iutcopy +#define CSYMM_ILTCOPY OPENBLAS_DISPATCH(csymm) -> csymm_iltcopy +#define CSYMM_OUTCOPY OPENBLAS_DISPATCH(csymm) -> csymm_outcopy +#define CSYMM_OLTCOPY OPENBLAS_DISPATCH(csymm) -> csymm_oltcopy + +#define CHEMM_OUTCOPY OPENBLAS_DISPATCH(chemm) -> chemm_outcopy +#define CHEMM_OLTCOPY OPENBLAS_DISPATCH(chemm) -> chemm_oltcopy +#define CHEMM_IUTCOPY OPENBLAS_DISPATCH(chemm) -> chemm_iutcopy +#define CHEMM_ILTCOPY OPENBLAS_DISPATCH(chemm) -> chemm_iltcopy + +#define CGEMM3M_ONCOPYB OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_oncopyb +#define CGEMM3M_ONCOPYR OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_oncopyr +#define CGEMM3M_ONCOPYI OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_oncopyi +#define CGEMM3M_OTCOPYB OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_otcopyb +#define CGEMM3M_OTCOPYR OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_otcopyr +#define CGEMM3M_OTCOPYI OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_otcopyi + +#define CGEMM3M_INCOPYB OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_incopyb +#define CGEMM3M_INCOPYR OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_incopyr +#define CGEMM3M_INCOPYI OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_incopyi +#define CGEMM3M_ITCOPYB OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_itcopyb +#define CGEMM3M_ITCOPYR OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_itcopyr +#define CGEMM3M_ITCOPYI OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_itcopyi + +#define CSYMM3M_ILCOPYB OPENBLAS_DISPATCH(csymm3m) -> csymm3m_ilcopyb +#define CSYMM3M_IUCOPYB OPENBLAS_DISPATCH(csymm3m) -> csymm3m_iucopyb +#define CSYMM3M_ILCOPYR OPENBLAS_DISPATCH(csymm3m) -> csymm3m_ilcopyr +#define CSYMM3M_IUCOPYR OPENBLAS_DISPATCH(csymm3m) -> csymm3m_iucopyr +#define CSYMM3M_ILCOPYI OPENBLAS_DISPATCH(csymm3m) -> csymm3m_ilcopyi +#define CSYMM3M_IUCOPYI OPENBLAS_DISPATCH(csymm3m) -> csymm3m_iucopyi + +#define CSYMM3M_OLCOPYB OPENBLAS_DISPATCH(csymm3m) -> csymm3m_olcopyb +#define CSYMM3M_OUCOPYB OPENBLAS_DISPATCH(csymm3m) -> csymm3m_oucopyb +#define CSYMM3M_OLCOPYR OPENBLAS_DISPATCH(csymm3m) -> csymm3m_olcopyr +#define CSYMM3M_OUCOPYR OPENBLAS_DISPATCH(csymm3m) -> csymm3m_oucopyr +#define CSYMM3M_OLCOPYI OPENBLAS_DISPATCH(csymm3m) -> csymm3m_olcopyi +#define CSYMM3M_OUCOPYI OPENBLAS_DISPATCH(csymm3m) -> csymm3m_oucopyi + +#define CHEMM3M_ILCOPYB OPENBLAS_DISPATCH(chemm3m) -> chemm3m_ilcopyb +#define CHEMM3M_IUCOPYB OPENBLAS_DISPATCH(chemm3m) -> chemm3m_iucopyb +#define CHEMM3M_ILCOPYR OPENBLAS_DISPATCH(chemm3m) -> chemm3m_ilcopyr +#define CHEMM3M_IUCOPYR OPENBLAS_DISPATCH(chemm3m) -> chemm3m_iucopyr +#define CHEMM3M_ILCOPYI OPENBLAS_DISPATCH(chemm3m) -> chemm3m_ilcopyi +#define CHEMM3M_IUCOPYI OPENBLAS_DISPATCH(chemm3m) -> chemm3m_iucopyi + +#define CHEMM3M_OLCOPYB OPENBLAS_DISPATCH(chemm3m) -> chemm3m_olcopyb +#define CHEMM3M_OUCOPYB OPENBLAS_DISPATCH(chemm3m) -> chemm3m_oucopyb +#define CHEMM3M_OLCOPYR OPENBLAS_DISPATCH(chemm3m) -> chemm3m_olcopyr +#define CHEMM3M_OUCOPYR OPENBLAS_DISPATCH(chemm3m) -> chemm3m_oucopyr +#define CHEMM3M_OLCOPYI OPENBLAS_DISPATCH(chemm3m) -> chemm3m_olcopyi +#define CHEMM3M_OUCOPYI OPENBLAS_DISPATCH(chemm3m) -> chemm3m_oucopyi + +#define CGEMM3M_KERNEL OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_kernel + +#define CNEG_TCOPY OPENBLAS_DISPATCH(cneg) -> cneg_tcopy +#define CLASWP_NCOPY OPENBLAS_DISPATCH(claswp) -> claswp_ncopy + +#define CAXPBY_K OPENBLAS_DISPATCH(caxpby) -> caxpby_k + +#define COMATCOPY_K_CN OPENBLAS_DISPATCH(comatcopy) -> comatcopy_k_cn +#define COMATCOPY_K_RN OPENBLAS_DISPATCH(comatcopy) -> comatcopy_k_rn +#define COMATCOPY_K_CT OPENBLAS_DISPATCH(comatcopy) -> comatcopy_k_ct +#define COMATCOPY_K_RT OPENBLAS_DISPATCH(comatcopy) -> comatcopy_k_rt +#define COMATCOPY_K_CNC OPENBLAS_DISPATCH(comatcopy) -> comatcopy_k_cnc +#define COMATCOPY_K_RNC OPENBLAS_DISPATCH(comatcopy) -> comatcopy_k_rnc +#define COMATCOPY_K_CTC OPENBLAS_DISPATCH(comatcopy) -> comatcopy_k_ctc +#define COMATCOPY_K_RTC OPENBLAS_DISPATCH(comatcopy) -> comatcopy_k_rtc + +#define CIMATCOPY_K_CN OPENBLAS_DISPATCH(cimatcopy) -> cimatcopy_k_cn +#define CIMATCOPY_K_RN OPENBLAS_DISPATCH(cimatcopy) -> cimatcopy_k_rn +#define CIMATCOPY_K_CT OPENBLAS_DISPATCH(cimatcopy) -> cimatcopy_k_ct +#define CIMATCOPY_K_RT OPENBLAS_DISPATCH(cimatcopy) -> cimatcopy_k_rt +#define CIMATCOPY_K_CNC OPENBLAS_DISPATCH(cimatcopy) -> cimatcopy_k_cnc +#define CIMATCOPY_K_RNC OPENBLAS_DISPATCH(cimatcopy) -> cimatcopy_k_rnc +#define CIMATCOPY_K_CTC OPENBLAS_DISPATCH(cimatcopy) -> cimatcopy_k_ctc +#define CIMATCOPY_K_RTC OPENBLAS_DISPATCH(cimatcopy) -> cimatcopy_k_rtc + +#define CGEADD_K OPENBLAS_DISPATCH(cgeadd) -> cgeadd_k + +#define CGEMM_SMALL_MATRIX_PERMIT OPENBLAS_DISPATCH(cgemm) -> cgemm_small_matrix_permit #endif -#define CGEMM_SMALL_KERNEL_NN FUNC_OFFSET(cgemm_small_kernel_nn) -#define CGEMM_SMALL_KERNEL_NT FUNC_OFFSET(cgemm_small_kernel_nt) -#define CGEMM_SMALL_KERNEL_NR FUNC_OFFSET(cgemm_small_kernel_nr) -#define CGEMM_SMALL_KERNEL_NC FUNC_OFFSET(cgemm_small_kernel_nc) - -#define CGEMM_SMALL_KERNEL_TN FUNC_OFFSET(cgemm_small_kernel_tn) -#define CGEMM_SMALL_KERNEL_TT FUNC_OFFSET(cgemm_small_kernel_tt) -#define CGEMM_SMALL_KERNEL_TR FUNC_OFFSET(cgemm_small_kernel_tr) -#define CGEMM_SMALL_KERNEL_TC FUNC_OFFSET(cgemm_small_kernel_tc) - -#define CGEMM_SMALL_KERNEL_RN FUNC_OFFSET(cgemm_small_kernel_rn) -#define CGEMM_SMALL_KERNEL_RT FUNC_OFFSET(cgemm_small_kernel_rt) -#define CGEMM_SMALL_KERNEL_RR FUNC_OFFSET(cgemm_small_kernel_rr) -#define CGEMM_SMALL_KERNEL_RC FUNC_OFFSET(cgemm_small_kernel_rc) - -#define CGEMM_SMALL_KERNEL_CN FUNC_OFFSET(cgemm_small_kernel_cn) -#define CGEMM_SMALL_KERNEL_CT FUNC_OFFSET(cgemm_small_kernel_ct) -#define CGEMM_SMALL_KERNEL_CR FUNC_OFFSET(cgemm_small_kernel_cr) -#define CGEMM_SMALL_KERNEL_CC FUNC_OFFSET(cgemm_small_kernel_cc) - -#define CGEMM_SMALL_KERNEL_B0_NN FUNC_OFFSET(cgemm_small_kernel_b0_nn) -#define CGEMM_SMALL_KERNEL_B0_NT FUNC_OFFSET(cgemm_small_kernel_b0_nt) -#define CGEMM_SMALL_KERNEL_B0_NR FUNC_OFFSET(cgemm_small_kernel_b0_nr) -#define CGEMM_SMALL_KERNEL_B0_NC FUNC_OFFSET(cgemm_small_kernel_b0_nc) - -#define CGEMM_SMALL_KERNEL_B0_TN FUNC_OFFSET(cgemm_small_kernel_b0_tn) -#define CGEMM_SMALL_KERNEL_B0_TT FUNC_OFFSET(cgemm_small_kernel_b0_tt) -#define CGEMM_SMALL_KERNEL_B0_TR FUNC_OFFSET(cgemm_small_kernel_b0_tr) -#define CGEMM_SMALL_KERNEL_B0_TC FUNC_OFFSET(cgemm_small_kernel_b0_tc) - -#define CGEMM_SMALL_KERNEL_B0_RN FUNC_OFFSET(cgemm_small_kernel_b0_rn) -#define CGEMM_SMALL_KERNEL_B0_RT FUNC_OFFSET(cgemm_small_kernel_b0_rt) -#define CGEMM_SMALL_KERNEL_B0_RR FUNC_OFFSET(cgemm_small_kernel_b0_rr) -#define CGEMM_SMALL_KERNEL_B0_RC FUNC_OFFSET(cgemm_small_kernel_b0_rc) - -#define CGEMM_SMALL_KERNEL_B0_CN FUNC_OFFSET(cgemm_small_kernel_b0_cn) -#define CGEMM_SMALL_KERNEL_B0_CT FUNC_OFFSET(cgemm_small_kernel_b0_ct) -#define CGEMM_SMALL_KERNEL_B0_CR FUNC_OFFSET(cgemm_small_kernel_b0_cr) -#define CGEMM_SMALL_KERNEL_B0_CC FUNC_OFFSET(cgemm_small_kernel_b0_cc) +#define CGEMM_SMALL_KERNEL_BASE OPENBLAS_DISPATCH_BASE(cgemm) +#define CGEMM_SMALL_KERNEL_NN OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_nn) +#define CGEMM_SMALL_KERNEL_NT OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_nt) +#define CGEMM_SMALL_KERNEL_NR OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_nr) +#define CGEMM_SMALL_KERNEL_NC OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_nc) + +#define CGEMM_SMALL_KERNEL_TN OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_tn) +#define CGEMM_SMALL_KERNEL_TT OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_tt) +#define CGEMM_SMALL_KERNEL_TR OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_tr) +#define CGEMM_SMALL_KERNEL_TC OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_tc) + +#define CGEMM_SMALL_KERNEL_RN OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_rn) +#define CGEMM_SMALL_KERNEL_RT OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_rt) +#define CGEMM_SMALL_KERNEL_RR OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_rr) +#define CGEMM_SMALL_KERNEL_RC OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_rc) + +#define CGEMM_SMALL_KERNEL_CN OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_cn) +#define CGEMM_SMALL_KERNEL_CT OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_ct) +#define CGEMM_SMALL_KERNEL_CR OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_cr) +#define CGEMM_SMALL_KERNEL_CC OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_cc) + +#define CGEMM_SMALL_KERNEL_B0_NN OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_nn) +#define CGEMM_SMALL_KERNEL_B0_NT OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_nt) +#define CGEMM_SMALL_KERNEL_B0_NR OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_nr) +#define CGEMM_SMALL_KERNEL_B0_NC OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_nc) + +#define CGEMM_SMALL_KERNEL_B0_TN OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_tn) +#define CGEMM_SMALL_KERNEL_B0_TT OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_tt) +#define CGEMM_SMALL_KERNEL_B0_TR OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_tr) +#define CGEMM_SMALL_KERNEL_B0_TC OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_tc) + +#define CGEMM_SMALL_KERNEL_B0_RN OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_rn) +#define CGEMM_SMALL_KERNEL_B0_RT OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_rt) +#define CGEMM_SMALL_KERNEL_B0_RR OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_rr) +#define CGEMM_SMALL_KERNEL_B0_RC OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_rc) + +#define CGEMM_SMALL_KERNEL_B0_CN OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_cn) +#define CGEMM_SMALL_KERNEL_B0_CT OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_ct) +#define CGEMM_SMALL_KERNEL_B0_CR OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_cr) +#define CGEMM_SMALL_KERNEL_B0_CC OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_cc) #define CGEMM_NN cgemm_nn diff --git a/common_d.h b/common_d.h index 30fbe17b55..cefd1fc6dc 100644 --- a/common_d.h +++ b/common_d.h @@ -209,150 +209,151 @@ #else -#define DAMAX_K gotoblas -> damax_k -#define DAMIN_K gotoblas -> damin_k -#define DMAX_K gotoblas -> dmax_k -#define DMIN_K gotoblas -> dmin_k -#define IDAMAX_K gotoblas -> idamax_k -#define IDAMIN_K gotoblas -> idamin_k -#define IDMAX_K gotoblas -> idmax_k -#define IDMIN_K gotoblas -> idmin_k -#define DASUM_K gotoblas -> dasum_k -#define DAXPYU_K gotoblas -> daxpy_k -#define DAXPYC_K gotoblas -> daxpy_k -#define DCOPY_K gotoblas -> dcopy_k -#define DDOTU_K gotoblas -> ddot_k -#define DDOTC_K gotoblas -> ddot_k -#define DNRM2_K gotoblas -> dnrm2_k -#define DSCAL_K gotoblas -> dscal_k -#define DSUM_K gotoblas -> dsum_k -#define DSWAP_K gotoblas -> dswap_k -#define DROT_K gotoblas -> drot_k -#define DROTM_K gotoblas -> drotm_k - -#define DGEMV_N gotoblas -> dgemv_n -#define DGEMV_T gotoblas -> dgemv_t -#define DGEMV_R gotoblas -> dgemv_n -#define DGEMV_C gotoblas -> dgemv_t -#define DGEMV_O gotoblas -> dgemv_n -#define DGEMV_U gotoblas -> dgemv_t -#define DGEMV_S gotoblas -> dgemv_n -#define DGEMV_D gotoblas -> dgemv_t - -#define DGERU_K gotoblas -> dger_k -#define DGERC_K gotoblas -> dger_k -#define DGERV_K gotoblas -> dger_k -#define DGERD_K gotoblas -> dger_k - -#define DSYMV_U gotoblas -> dsymv_U -#define DSYMV_L gotoblas -> dsymv_L +#define DAMAX_K OPENBLAS_DISPATCH(damax) -> damax_k +#define DAMIN_K OPENBLAS_DISPATCH(damin) -> damin_k +#define DMAX_K OPENBLAS_DISPATCH(dmax) -> dmax_k +#define DMIN_K OPENBLAS_DISPATCH(dmin) -> dmin_k +#define IDAMAX_K OPENBLAS_DISPATCH(idamax) -> idamax_k +#define IDAMIN_K OPENBLAS_DISPATCH(idamin) -> idamin_k +#define IDMAX_K OPENBLAS_DISPATCH(idmax) -> idmax_k +#define IDMIN_K OPENBLAS_DISPATCH(idmin) -> idmin_k +#define DASUM_K OPENBLAS_DISPATCH(dasum) -> dasum_k +#define DAXPYU_K OPENBLAS_DISPATCH(daxpy) -> daxpy_k +#define DAXPYC_K OPENBLAS_DISPATCH(daxpy) -> daxpy_k +#define DCOPY_K OPENBLAS_DISPATCH(dcopy) -> dcopy_k +#define DDOTU_K OPENBLAS_DISPATCH(ddot) -> ddot_k +#define DDOTC_K OPENBLAS_DISPATCH(ddot) -> ddot_k +#define DNRM2_K OPENBLAS_DISPATCH(dnrm2) -> dnrm2_k +#define DSCAL_K OPENBLAS_DISPATCH(dscal) -> dscal_k +#define DSUM_K OPENBLAS_DISPATCH(dsum) -> dsum_k +#define DSWAP_K OPENBLAS_DISPATCH(dswap) -> dswap_k +#define DROT_K OPENBLAS_DISPATCH(drot) -> drot_k +#define DROTM_K OPENBLAS_DISPATCH(drotm) -> drotm_k + +#define DGEMV_N OPENBLAS_DISPATCH(dgemv) -> dgemv_n +#define DGEMV_T OPENBLAS_DISPATCH(dgemv) -> dgemv_t +#define DGEMV_R OPENBLAS_DISPATCH(dgemv) -> dgemv_n +#define DGEMV_C OPENBLAS_DISPATCH(dgemv) -> dgemv_t +#define DGEMV_O OPENBLAS_DISPATCH(dgemv) -> dgemv_n +#define DGEMV_U OPENBLAS_DISPATCH(dgemv) -> dgemv_t +#define DGEMV_S OPENBLAS_DISPATCH(dgemv) -> dgemv_n +#define DGEMV_D OPENBLAS_DISPATCH(dgemv) -> dgemv_t + +#define DGERU_K OPENBLAS_DISPATCH(dger) -> dger_k +#define DGERC_K OPENBLAS_DISPATCH(dger) -> dger_k +#define DGERV_K OPENBLAS_DISPATCH(dger) -> dger_k +#define DGERD_K OPENBLAS_DISPATCH(dger) -> dger_k + +#define DSYMV_U OPENBLAS_DISPATCH(dsymv) -> dsymv_U +#define DSYMV_L OPENBLAS_DISPATCH(dsymv) -> dsymv_L #define DSYMV_THREAD_U dsymv_thread_U #define DSYMV_THREAD_L dsymv_thread_L -#define DGEMM_ONCOPY gotoblas -> dgemm_oncopy -#define DGEMM_OTCOPY gotoblas -> dgemm_otcopy -#define DGEMM_INCOPY gotoblas -> dgemm_incopy -#define DGEMM_ITCOPY gotoblas -> dgemm_itcopy - -#define DTRMM_OUNUCOPY gotoblas -> dtrmm_ounucopy -#define DTRMM_OUTUCOPY gotoblas -> dtrmm_outucopy -#define DTRMM_OLNUCOPY gotoblas -> dtrmm_olnucopy -#define DTRMM_OLTUCOPY gotoblas -> dtrmm_oltucopy -#define DTRSM_OUNUCOPY gotoblas -> dtrsm_ounucopy -#define DTRSM_OUTUCOPY gotoblas -> dtrsm_outucopy -#define DTRSM_OLNUCOPY gotoblas -> dtrsm_olnucopy -#define DTRSM_OLTUCOPY gotoblas -> dtrsm_oltucopy - -#define DTRMM_IUNUCOPY gotoblas -> dtrmm_iunucopy -#define DTRMM_IUTUCOPY gotoblas -> dtrmm_iutucopy -#define DTRMM_ILNUCOPY gotoblas -> dtrmm_ilnucopy -#define DTRMM_ILTUCOPY gotoblas -> dtrmm_iltucopy -#define DTRSM_IUNUCOPY gotoblas -> dtrsm_iunucopy -#define DTRSM_IUTUCOPY gotoblas -> dtrsm_iutucopy -#define DTRSM_ILNUCOPY gotoblas -> dtrsm_ilnucopy -#define DTRSM_ILTUCOPY gotoblas -> dtrsm_iltucopy - -#define DTRMM_OUNNCOPY gotoblas -> dtrmm_ounncopy -#define DTRMM_OUTNCOPY gotoblas -> dtrmm_outncopy -#define DTRMM_OLNNCOPY gotoblas -> dtrmm_olnncopy -#define DTRMM_OLTNCOPY gotoblas -> dtrmm_oltncopy -#define DTRSM_OUNNCOPY gotoblas -> dtrsm_ounncopy -#define DTRSM_OUTNCOPY gotoblas -> dtrsm_outncopy -#define DTRSM_OLNNCOPY gotoblas -> dtrsm_olnncopy -#define DTRSM_OLTNCOPY gotoblas -> dtrsm_oltncopy - -#define DTRMM_IUNNCOPY gotoblas -> dtrmm_iunncopy -#define DTRMM_IUTNCOPY gotoblas -> dtrmm_iutncopy -#define DTRMM_ILNNCOPY gotoblas -> dtrmm_ilnncopy -#define DTRMM_ILTNCOPY gotoblas -> dtrmm_iltncopy -#define DSYMM_INCOPY gotoblas -> dsymm_incopy -#define DSYMM_ITCOPY gotoblas -> dsymm_itcopy -#define DTRMM_INCOPY gotoblas -> dtrmm_incopy -#define DTRMM_ITCOPY gotoblas -> dtrmm_itcopy - -#define DTRSM_IUNNCOPY gotoblas -> dtrsm_iunncopy -#define DTRSM_IUTNCOPY gotoblas -> dtrsm_iutncopy -#define DTRSM_ILNNCOPY gotoblas -> dtrsm_ilnncopy -#define DTRSM_ILTNCOPY gotoblas -> dtrsm_iltncopy - -#define DGEMM_BETA gotoblas -> dgemm_beta -#define DGEMM_KERNEL gotoblas -> dgemm_kernel -#define SME_DGEMM_KERNEL gotoblas -> sme_dgemm_kernel -#define DSYMM_KERNEL gotoblas -> dsymm_kernel -#define DTRMM_GEMM_KERNEL gotoblas -> dtrmm_gemm_kernel - -#define DTRMM_KERNEL_LN gotoblas -> dtrmm_kernel_LN -#define DTRMM_KERNEL_LT gotoblas -> dtrmm_kernel_LT -#define DTRMM_KERNEL_LR gotoblas -> dtrmm_kernel_LN -#define DTRMM_KERNEL_LC gotoblas -> dtrmm_kernel_LT -#define DTRMM_KERNEL_RN gotoblas -> dtrmm_kernel_RN -#define DTRMM_KERNEL_RT gotoblas -> dtrmm_kernel_RT -#define DTRMM_KERNEL_RR gotoblas -> dtrmm_kernel_RN -#define DTRMM_KERNEL_RC gotoblas -> dtrmm_kernel_RT - -#define DTRSM_KERNEL_LN gotoblas -> dtrsm_kernel_LN -#define DTRSM_KERNEL_LT gotoblas -> dtrsm_kernel_LT -#define DTRSM_KERNEL_LR gotoblas -> dtrsm_kernel_LN -#define DTRSM_KERNEL_LC gotoblas -> dtrsm_kernel_LT -#define DTRSM_KERNEL_RN gotoblas -> dtrsm_kernel_RN -#define DTRSM_KERNEL_RT gotoblas -> dtrsm_kernel_RT -#define DTRSM_KERNEL_RR gotoblas -> dtrsm_kernel_RN -#define DTRSM_KERNEL_RC gotoblas -> dtrsm_kernel_RT - -#define DSYMM_IUTCOPY gotoblas -> dsymm_iutcopy -#define DSYMM_ILTCOPY gotoblas -> dsymm_iltcopy -#define DSYMM_OUTCOPY gotoblas -> dsymm_outcopy -#define DSYMM_OLTCOPY gotoblas -> dsymm_oltcopy - -#define DNEG_TCOPY gotoblas -> dneg_tcopy -#define DLASWP_NCOPY gotoblas -> dlaswp_ncopy - -#define DAXPBY_K gotoblas -> daxpby_k -#define DOMATCOPY_K_CN gotoblas -> domatcopy_k_cn -#define DOMATCOPY_K_RN gotoblas -> domatcopy_k_rn -#define DOMATCOPY_K_CT gotoblas -> domatcopy_k_ct -#define DOMATCOPY_K_RT gotoblas -> domatcopy_k_rt -#define DIMATCOPY_K_CN gotoblas -> dimatcopy_k_cn -#define DIMATCOPY_K_RN gotoblas -> dimatcopy_k_rn -#define DIMATCOPY_K_CT gotoblas -> dimatcopy_k_ct -#define DIMATCOPY_K_RT gotoblas -> dimatcopy_k_rt - -#define DGEADD_K gotoblas -> dgeadd_k - -#define DGEMM_SMALL_MATRIX_PERMIT gotoblas -> dgemm_small_matrix_permit +#define DGEMM_ONCOPY OPENBLAS_DISPATCH(dgemm) -> dgemm_oncopy +#define DGEMM_OTCOPY OPENBLAS_DISPATCH(dgemm) -> dgemm_otcopy +#define DGEMM_INCOPY OPENBLAS_DISPATCH(dgemm) -> dgemm_incopy +#define DGEMM_ITCOPY OPENBLAS_DISPATCH(dgemm) -> dgemm_itcopy + +#define DTRMM_OUNUCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_ounucopy +#define DTRMM_OUTUCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_outucopy +#define DTRMM_OLNUCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_olnucopy +#define DTRMM_OLTUCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_oltucopy +#define DTRSM_OUNUCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_ounucopy +#define DTRSM_OUTUCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_outucopy +#define DTRSM_OLNUCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_olnucopy +#define DTRSM_OLTUCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_oltucopy + +#define DTRMM_IUNUCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_iunucopy +#define DTRMM_IUTUCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_iutucopy +#define DTRMM_ILNUCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_ilnucopy +#define DTRMM_ILTUCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_iltucopy +#define DTRSM_IUNUCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_iunucopy +#define DTRSM_IUTUCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_iutucopy +#define DTRSM_ILNUCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_ilnucopy +#define DTRSM_ILTUCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_iltucopy + +#define DTRMM_OUNNCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_ounncopy +#define DTRMM_OUTNCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_outncopy +#define DTRMM_OLNNCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_olnncopy +#define DTRMM_OLTNCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_oltncopy +#define DTRSM_OUNNCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_ounncopy +#define DTRSM_OUTNCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_outncopy +#define DTRSM_OLNNCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_olnncopy +#define DTRSM_OLTNCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_oltncopy + +#define DTRMM_IUNNCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_iunncopy +#define DTRMM_IUTNCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_iutncopy +#define DTRMM_ILNNCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_ilnncopy +#define DTRMM_ILTNCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_iltncopy +#define DSYMM_INCOPY OPENBLAS_DISPATCH(dsymm) -> dsymm_incopy +#define DSYMM_ITCOPY OPENBLAS_DISPATCH(dsymm) -> dsymm_itcopy +#define DTRMM_INCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_incopy +#define DTRMM_ITCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_itcopy + +#define DTRSM_IUNNCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_iunncopy +#define DTRSM_IUTNCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_iutncopy +#define DTRSM_ILNNCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_ilnncopy +#define DTRSM_ILTNCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_iltncopy + +#define DGEMM_BETA OPENBLAS_DISPATCH(dgemm) -> dgemm_beta +#define DGEMM_KERNEL OPENBLAS_DISPATCH(dgemm) -> dgemm_kernel +#define SME_DGEMM_KERNEL OPENBLAS_DISPATCH(dgemm) -> sme_dgemm_kernel +#define DSYMM_KERNEL OPENBLAS_DISPATCH(dsymm) -> dsymm_kernel +#define DTRMM_GEMM_KERNEL OPENBLAS_DISPATCH(dtrmm) -> dtrmm_gemm_kernel + +#define DTRMM_KERNEL_LN OPENBLAS_DISPATCH(dtrmm) -> dtrmm_kernel_LN +#define DTRMM_KERNEL_LT OPENBLAS_DISPATCH(dtrmm) -> dtrmm_kernel_LT +#define DTRMM_KERNEL_LR OPENBLAS_DISPATCH(dtrmm) -> dtrmm_kernel_LN +#define DTRMM_KERNEL_LC OPENBLAS_DISPATCH(dtrmm) -> dtrmm_kernel_LT +#define DTRMM_KERNEL_RN OPENBLAS_DISPATCH(dtrmm) -> dtrmm_kernel_RN +#define DTRMM_KERNEL_RT OPENBLAS_DISPATCH(dtrmm) -> dtrmm_kernel_RT +#define DTRMM_KERNEL_RR OPENBLAS_DISPATCH(dtrmm) -> dtrmm_kernel_RN +#define DTRMM_KERNEL_RC OPENBLAS_DISPATCH(dtrmm) -> dtrmm_kernel_RT + +#define DTRSM_KERNEL_LN OPENBLAS_DISPATCH(dtrsm) -> dtrsm_kernel_LN +#define DTRSM_KERNEL_LT OPENBLAS_DISPATCH(dtrsm) -> dtrsm_kernel_LT +#define DTRSM_KERNEL_LR OPENBLAS_DISPATCH(dtrsm) -> dtrsm_kernel_LN +#define DTRSM_KERNEL_LC OPENBLAS_DISPATCH(dtrsm) -> dtrsm_kernel_LT +#define DTRSM_KERNEL_RN OPENBLAS_DISPATCH(dtrsm) -> dtrsm_kernel_RN +#define DTRSM_KERNEL_RT OPENBLAS_DISPATCH(dtrsm) -> dtrsm_kernel_RT +#define DTRSM_KERNEL_RR OPENBLAS_DISPATCH(dtrsm) -> dtrsm_kernel_RN +#define DTRSM_KERNEL_RC OPENBLAS_DISPATCH(dtrsm) -> dtrsm_kernel_RT + +#define DSYMM_IUTCOPY OPENBLAS_DISPATCH(dsymm) -> dsymm_iutcopy +#define DSYMM_ILTCOPY OPENBLAS_DISPATCH(dsymm) -> dsymm_iltcopy +#define DSYMM_OUTCOPY OPENBLAS_DISPATCH(dsymm) -> dsymm_outcopy +#define DSYMM_OLTCOPY OPENBLAS_DISPATCH(dsymm) -> dsymm_oltcopy + +#define DNEG_TCOPY OPENBLAS_DISPATCH(dneg) -> dneg_tcopy +#define DLASWP_NCOPY OPENBLAS_DISPATCH(dlaswp) -> dlaswp_ncopy + +#define DAXPBY_K OPENBLAS_DISPATCH(daxpby) -> daxpby_k +#define DOMATCOPY_K_CN OPENBLAS_DISPATCH(domatcopy) -> domatcopy_k_cn +#define DOMATCOPY_K_RN OPENBLAS_DISPATCH(domatcopy) -> domatcopy_k_rn +#define DOMATCOPY_K_CT OPENBLAS_DISPATCH(domatcopy) -> domatcopy_k_ct +#define DOMATCOPY_K_RT OPENBLAS_DISPATCH(domatcopy) -> domatcopy_k_rt +#define DIMATCOPY_K_CN OPENBLAS_DISPATCH(dimatcopy) -> dimatcopy_k_cn +#define DIMATCOPY_K_RN OPENBLAS_DISPATCH(dimatcopy) -> dimatcopy_k_rn +#define DIMATCOPY_K_CT OPENBLAS_DISPATCH(dimatcopy) -> dimatcopy_k_ct +#define DIMATCOPY_K_RT OPENBLAS_DISPATCH(dimatcopy) -> dimatcopy_k_rt + +#define DGEADD_K OPENBLAS_DISPATCH(dgeadd) -> dgeadd_k + +#define DGEMM_SMALL_MATRIX_PERMIT OPENBLAS_DISPATCH(dgemm) -> dgemm_small_matrix_permit #endif -#define DGEMM_SMALL_KERNEL_NN FUNC_OFFSET(dgemm_small_kernel_nn) -#define DGEMM_SMALL_KERNEL_NT FUNC_OFFSET(dgemm_small_kernel_nt) -#define DGEMM_SMALL_KERNEL_TN FUNC_OFFSET(dgemm_small_kernel_tn) -#define DGEMM_SMALL_KERNEL_TT FUNC_OFFSET(dgemm_small_kernel_tt) +#define DGEMM_SMALL_KERNEL_BASE OPENBLAS_DISPATCH_BASE(dgemm) +#define DGEMM_SMALL_KERNEL_NN OPENBLAS_DISPATCH_OFFSET(dgemm, dgemm_small_kernel_nn) +#define DGEMM_SMALL_KERNEL_NT OPENBLAS_DISPATCH_OFFSET(dgemm, dgemm_small_kernel_nt) +#define DGEMM_SMALL_KERNEL_TN OPENBLAS_DISPATCH_OFFSET(dgemm, dgemm_small_kernel_tn) +#define DGEMM_SMALL_KERNEL_TT OPENBLAS_DISPATCH_OFFSET(dgemm, dgemm_small_kernel_tt) -#define DGEMM_SMALL_KERNEL_B0_NN FUNC_OFFSET(dgemm_small_kernel_b0_nn) -#define DGEMM_SMALL_KERNEL_B0_NT FUNC_OFFSET(dgemm_small_kernel_b0_nt) -#define DGEMM_SMALL_KERNEL_B0_TN FUNC_OFFSET(dgemm_small_kernel_b0_tn) -#define DGEMM_SMALL_KERNEL_B0_TT FUNC_OFFSET(dgemm_small_kernel_b0_tt) +#define DGEMM_SMALL_KERNEL_B0_NN OPENBLAS_DISPATCH_OFFSET(dgemm, dgemm_small_kernel_b0_nn) +#define DGEMM_SMALL_KERNEL_B0_NT OPENBLAS_DISPATCH_OFFSET(dgemm, dgemm_small_kernel_b0_nt) +#define DGEMM_SMALL_KERNEL_B0_TN OPENBLAS_DISPATCH_OFFSET(dgemm, dgemm_small_kernel_b0_tn) +#define DGEMM_SMALL_KERNEL_B0_TT OPENBLAS_DISPATCH_OFFSET(dgemm, dgemm_small_kernel_b0_tt) #define DGEMM_NN dgemm_nn diff --git a/common_macro.h b/common_macro.h index 8ed813742b..8b46a5ce9f 100644 --- a/common_macro.h +++ b/common_macro.h @@ -697,6 +697,7 @@ #define GEMM_SMALL_MATRIX_PERMIT DGEMM_SMALL_MATRIX_PERMIT +#define GEMM_SMALL_KERNEL_BASE DGEMM_SMALL_KERNEL_BASE #define GEMM_SMALL_KERNEL_NN DGEMM_SMALL_KERNEL_NN #define GEMM_SMALL_KERNEL_NT DGEMM_SMALL_KERNEL_NT #define GEMM_SMALL_KERNEL_TN DGEMM_SMALL_KERNEL_TN @@ -1103,6 +1104,7 @@ #define GEMM_SMALL_MATRIX_PERMIT SBGEMM_SMALL_MATRIX_PERMIT +#define GEMM_SMALL_KERNEL_BASE SBGEMM_SMALL_KERNEL_BASE #define GEMM_SMALL_KERNEL_NN SBGEMM_SMALL_KERNEL_NN #define GEMM_SMALL_KERNEL_NT SBGEMM_SMALL_KERNEL_NT #define GEMM_SMALL_KERNEL_TN SBGEMM_SMALL_KERNEL_TN @@ -1446,6 +1448,7 @@ #define GEMM_SMALL_MATRIX_PERMIT SGEMM_SMALL_MATRIX_PERMIT +#define GEMM_SMALL_KERNEL_BASE SGEMM_SMALL_KERNEL_BASE #define GEMM_SMALL_KERNEL_NN SGEMM_SMALL_KERNEL_NN #define GEMM_SMALL_KERNEL_NT SGEMM_SMALL_KERNEL_NT #define GEMM_SMALL_KERNEL_TN SGEMM_SMALL_KERNEL_TN @@ -2393,6 +2396,7 @@ #define GEMM_SMALL_MATRIX_PERMIT ZGEMM_SMALL_MATRIX_PERMIT +#define GEMM_SMALL_KERNEL_BASE ZGEMM_SMALL_KERNEL_BASE #define GEMM_SMALL_KERNEL_NN ZGEMM_SMALL_KERNEL_NN #define GEMM_SMALL_KERNEL_NT ZGEMM_SMALL_KERNEL_NT #define GEMM_SMALL_KERNEL_NR ZGEMM_SMALL_KERNEL_NR @@ -2913,6 +2917,7 @@ #define GEMM_SMALL_MATRIX_PERMIT CGEMM_SMALL_MATRIX_PERMIT +#define GEMM_SMALL_KERNEL_BASE CGEMM_SMALL_KERNEL_BASE #define GEMM_SMALL_KERNEL_NN CGEMM_SMALL_KERNEL_NN #define GEMM_SMALL_KERNEL_NT CGEMM_SMALL_KERNEL_NT #define GEMM_SMALL_KERNEL_NR CGEMM_SMALL_KERNEL_NR diff --git a/common_param.h b/common_param.h index ce04ca6aa1..d5fb7ffc20 100644 --- a/common_param.h +++ b/common_param.h @@ -44,7 +44,19 @@ #ifdef DYNAMIC_ARCH +/* OPENBLAS_CORE_LIST(X, arg) expands to X(CORE, arg) for each core in this + build. dyn_cores.h is generated from DYNAMIC_CORE by the build system. */ +#include "dyn_cores.h" + +#define OPENBLAS_CORE_ENUM(core, unused) OPENBLAS_CORE_##core, +enum { OPENBLAS_CORE_LIST(OPENBLAS_CORE_ENUM, ~) OPENBLAS_NUM_CORES }; + +#define OPENBLAS_DISPATCH(group) (openblas_##group##_dispatch[gotoblas->core]) +#define OPENBLAS_DISPATCH_OFFSET(group, func) (size_t)(&((openblas_##group##_dispatch_t *)NULL)->func) +#define OPENBLAS_DISPATCH_BASE(group) ((char *)OPENBLAS_DISPATCH(group)) + typedef struct { + int core; int dtb_entries; int switch_ratio; int divide_rate; @@ -54,90 +66,313 @@ typedef struct { #if BUILD_HFLOAT16 == 1 int shgemm_p, shgemm_q, shgemm_r; int shgemm_unroll_m, shgemm_unroll_n, shgemm_unroll_mn; +#endif +#if BUILD_BFLOAT16 == 1 + int bgemm_p, bgemm_q, bgemm_r; + int bgemm_unroll_m, bgemm_unroll_n, bgemm_unroll_mn; + int bgemm_align_k; + int sbgemm_p, sbgemm_q, sbgemm_r; + int sbgemm_unroll_m, sbgemm_unroll_n, sbgemm_unroll_mn; + int sbgemm_align_k; + int need_amxtile_permission; // 0 default, 1 for device support amx. +#endif +#if (BUILD_SINGLE == 1) || (BUILD_DOUBLE == 1) || (BUILD_COMPLEX == 1) || (BUILD_COMPLEX16 == 1) + int sgemm_p, sgemm_q, sgemm_r; + int sgemm_unroll_m, sgemm_unroll_n, sgemm_unroll_mn; +#endif + int exclusive_cache; +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) + int dgemm_p, dgemm_q, dgemm_r; + int dgemm_unroll_m, dgemm_unroll_n, dgemm_unroll_mn; +#endif +#ifdef EXPRECISION + int qgemm_p, qgemm_q, qgemm_r; + int qgemm_unroll_m, qgemm_unroll_n, qgemm_unroll_mn; +#endif +#if (BUILD_COMPLEX==1) + int cgemm_p, cgemm_q, cgemm_r; + int cgemm_unroll_m, cgemm_unroll_n, cgemm_unroll_mn; + int cgemm3m_p, cgemm3m_q, cgemm3m_r; + int cgemm3m_unroll_m, cgemm3m_unroll_n, cgemm3m_unroll_mn; +#endif +#if (BUILD_COMPLEX16 == 1) + int zgemm_p, zgemm_q, zgemm_r; + int zgemm_unroll_m, zgemm_unroll_n, zgemm_unroll_mn; + int zgemm3m_p, zgemm3m_q, zgemm3m_r; + int zgemm3m_unroll_m, zgemm3m_unroll_n, zgemm3m_unroll_mn; +#endif +#ifdef EXPRECISION + int xgemm_p, xgemm_q, xgemm_r; + int xgemm_unroll_m, xgemm_unroll_n, xgemm_unroll_mn; + int xgemm3m_p, xgemm3m_q, xgemm3m_r; + int xgemm3m_unroll_m, xgemm3m_unroll_n, xgemm3m_unroll_mn; +#endif + void (*init)(void); + int snum_opt, dnum_opt, qnum_opt; +} gotoblas_t; +extern gotoblas_t *gotoblas; + +#if BUILD_HFLOAT16 == 1 +typedef struct { int (*shgemm_kernel )(BLASLONG, BLASLONG, BLASLONG, float, hfloat16 *, hfloat16 *, float *, BLASLONG); int (*shgemm_beta )(BLASLONG, BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BLASLONG, float *, BLASLONG); - int (*shgemm_incopy )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *); int (*shgemm_itcopy )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *); int (*shgemm_oncopy )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *); int (*shgemm_otcopy )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *); +} openblas_shgemm_dispatch_t; +extern const openblas_shgemm_dispatch_t *const openblas_shgemm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_HFLOAT16 == 1 +typedef struct { int (*shgemv_n) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BLASLONG, float, float *, BLASLONG); int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BLASLONG, float, float *, BLASLONG); +} openblas_shgemv_dispatch_t; +extern const openblas_shgemv_dispatch_t *const openblas_shgemv_dispatch[OPENBLAS_NUM_CORES]; #endif - #if BUILD_BFLOAT16 == 1 - int bgemm_p, bgemm_q, bgemm_r; - int bgemm_unroll_m, bgemm_unroll_n, bgemm_unroll_mn; - int bgemm_align_k; - - int sbgemm_p, sbgemm_q, sbgemm_r; - int sbgemm_unroll_m, sbgemm_unroll_n, sbgemm_unroll_mn; - int sbgemm_align_k; - int need_amxtile_permission; // 0 default, 1 for device support amx. - +typedef struct { void (*sbstobf16_k) (BLASLONG, float *, BLASLONG, bfloat16 *, BLASLONG); +} openblas_sbstobf16_dispatch_t; +extern const openblas_sbstobf16_dispatch_t *const openblas_sbstobf16_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { void (*sbdtobf16_k) (BLASLONG, double *, BLASLONG, bfloat16 *, BLASLONG); +} openblas_sbdtobf16_dispatch_t; +extern const openblas_sbdtobf16_dispatch_t *const openblas_sbdtobf16_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { void (*sbf16tos_k) (BLASLONG, bfloat16 *, BLASLONG, float *, BLASLONG); +} openblas_sbf16tos_dispatch_t; +extern const openblas_sbf16tos_dispatch_t *const openblas_sbf16tos_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { void (*dbf16tod_k) (BLASLONG, bfloat16 *, BLASLONG, double *, BLASLONG); +} openblas_dbf16tod_dispatch_t; +extern const openblas_dbf16tod_dispatch_t *const openblas_dbf16tod_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { float (*sbamax_k) (BLASLONG, float *, BLASLONG); +} openblas_sbamax_dispatch_t; +extern const openblas_sbamax_dispatch_t *const openblas_sbamax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { float (*sbamin_k) (BLASLONG, float *, BLASLONG); +} openblas_sbamin_dispatch_t; +extern const openblas_sbamin_dispatch_t *const openblas_sbamin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { float (*sbmax_k) (BLASLONG, float *, BLASLONG); +} openblas_sbmax_dispatch_t; +extern const openblas_sbmax_dispatch_t *const openblas_sbmax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { float (*sbmin_k) (BLASLONG, float *, BLASLONG); +} openblas_sbmin_dispatch_t; +extern const openblas_sbmin_dispatch_t *const openblas_sbmin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { BLASLONG (*isbamax_k)(BLASLONG, float *, BLASLONG); +} openblas_isbamax_dispatch_t; +extern const openblas_isbamax_dispatch_t *const openblas_isbamax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { BLASLONG (*isbamin_k)(BLASLONG, float *, BLASLONG); +} openblas_isbamin_dispatch_t; +extern const openblas_isbamin_dispatch_t *const openblas_isbamin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { BLASLONG (*isbmax_k) (BLASLONG, float *, BLASLONG); +} openblas_isbmax_dispatch_t; +extern const openblas_isbmax_dispatch_t *const openblas_isbmax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { BLASLONG (*isbmin_k) (BLASLONG, float *, BLASLONG); +} openblas_isbmin_dispatch_t; +extern const openblas_isbmin_dispatch_t *const openblas_isbmin_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { float (*sbnrm2_k) (BLASLONG, float *, BLASLONG); +} openblas_sbnrm2_dispatch_t; +extern const openblas_sbnrm2_dispatch_t *const openblas_sbnrm2_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { float (*sbasum_k) (BLASLONG, float *, BLASLONG); +} openblas_sbasum_dispatch_t; +extern const openblas_sbasum_dispatch_t *const openblas_sbasum_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { float (*sbsum_k) (BLASLONG, float *, BLASLONG); +} openblas_sbsum_dispatch_t; +extern const openblas_sbsum_dispatch_t *const openblas_sbsum_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbcopy_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_sbcopy_dispatch_t; +extern const openblas_sbcopy_dispatch_t *const openblas_sbcopy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { float (*sbdot_k) (BLASLONG, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG); +} openblas_sbdot_dispatch_t; +extern const openblas_sbdot_dispatch_t *const openblas_sbdot_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { double (*dsbdot_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_dsbdot_dispatch_t; +extern const openblas_dsbdot_dispatch_t *const openblas_dsbdot_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbrot_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG, float, float); +} openblas_sbrot_dispatch_t; +extern const openblas_sbrot_dispatch_t *const openblas_sbrot_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbrotm_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_sbrotm_dispatch_t; +extern const openblas_sbrotm_dispatch_t *const openblas_sbrotm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*bscal_k) (BLASLONG, BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG); +} openblas_bscal_dispatch_t; +extern const openblas_bscal_dispatch_t *const openblas_bscal_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbaxpy_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_sbaxpy_dispatch_t; +extern const openblas_sbaxpy_dispatch_t *const openblas_sbaxpy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbscal_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_sbscal_dispatch_t; +extern const openblas_sbscal_dispatch_t *const openblas_sbscal_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbswap_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_sbswap_dispatch_t; +extern const openblas_sbswap_dispatch_t *const openblas_sbswap_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*bgemv_n) (BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16, bfloat16 *, BLASLONG); int (*bgemv_t) (BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16, bfloat16 *, BLASLONG); +} openblas_bgemv_dispatch_t; +extern const openblas_bgemv_dispatch_t *const openblas_bgemv_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbgemv_n) (BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float, float *, BLASLONG); int (*sbgemv_t) (BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float, float *, BLASLONG); +} openblas_sbgemv_dispatch_t; +extern const openblas_sbgemv_dispatch_t *const openblas_sbgemv_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbger_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_sbger_dispatch_t; +extern const openblas_sbger_dispatch_t *const openblas_sbger_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbsymv_L) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*sbsymv_U) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_sbsymv_dispatch_t; +extern const openblas_sbsymv_dispatch_t *const openblas_sbsymv_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*bgemm_kernel )(BLASLONG, BLASLONG, BLASLONG, bfloat16, bfloat16 *, bfloat16 *, bfloat16 *, BLASLONG); int (*bgemm_beta )(BLASLONG, BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG); - int (*bgemm_incopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); int (*bgemm_itcopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); int (*bgemm_oncopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); int (*bgemm_otcopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); +} openblas_bgemm_dispatch_t; +extern const openblas_bgemm_dispatch_t *const openblas_bgemm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbgemm_kernel )(BLASLONG, BLASLONG, BLASLONG, float, bfloat16 *, bfloat16 *, float *, BLASLONG); int (*sbgemm_beta )(BLASLONG, BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float *, BLASLONG); - int (*sbgemm_incopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); int (*sbgemm_itcopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); int (*sbgemm_oncopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); int (*sbgemm_otcopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); +#ifdef SMALL_MATRIX_OPT + int (*sbgemm_small_matrix_permit)(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, float alpha, float beta); + int (*sbgemm_small_kernel_nn )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); + int (*sbgemm_small_kernel_nt )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); + int (*sbgemm_small_kernel_tn )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); + int (*sbgemm_small_kernel_tt )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); + int (*sbgemm_small_kernel_b0_nn )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc); + int (*sbgemm_small_kernel_b0_nt )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc); + int (*sbgemm_small_kernel_b0_tn )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc); + int (*sbgemm_small_kernel_b0_tt )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc); +#endif +} openblas_sbgemm_dispatch_t; +extern const openblas_sbgemm_dispatch_t *const openblas_sbgemm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbtrsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); int (*sbtrsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); int (*sbtrsm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); int (*sbtrsm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*sbtrsm_iunucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); int (*sbtrsm_iunncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); int (*sbtrsm_iutucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); @@ -154,12 +389,16 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*sbtrsm_olnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); int (*sbtrsm_oltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); int (*sbtrsm_oltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); +} openblas_sbtrsm_dispatch_t; +extern const openblas_sbtrsm_dispatch_t *const openblas_sbtrsm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbtrmm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); int (*sbtrmm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); int (*sbtrmm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); int (*sbtrmm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*sbtrmm_iunucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*sbtrmm_iunncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*sbtrmm_iutucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); @@ -176,84 +415,185 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*sbtrmm_olnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*sbtrmm_oltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*sbtrmm_oltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); +} openblas_sbtrmm_dispatch_t; +extern const openblas_sbtrmm_dispatch_t *const openblas_sbtrmm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbsymm_iutcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*sbsymm_iltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*sbsymm_outcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*sbsymm_oltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); +} openblas_sbsymm_dispatch_t; +extern const openblas_sbsymm_dispatch_t *const openblas_sbsymm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbneg_tcopy) (BLASLONG, BLASLONG, float *, BLASLONG, float *); - int (*sblaswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, float *, BLASLONG, blasint *, float *); - -#ifdef SMALL_MATRIX_OPT - int (*sbgemm_small_matrix_permit)(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, float alpha, float beta); - - int (*sbgemm_small_kernel_nn )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); - int (*sbgemm_small_kernel_nt )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); - int (*sbgemm_small_kernel_tn )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); - int (*sbgemm_small_kernel_tt )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); - - int (*sbgemm_small_kernel_b0_nn )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc); - int (*sbgemm_small_kernel_b0_nt )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc); - int (*sbgemm_small_kernel_b0_tn )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc); - int (*sbgemm_small_kernel_b0_tt )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc); -#endif +} openblas_sbneg_dispatch_t; +extern const openblas_sbneg_dispatch_t *const openblas_sbneg_dispatch[OPENBLAS_NUM_CORES]; #endif -#if (BUILD_SINGLE == 1) || (BUILD_DOUBLE == 1) || (BUILD_COMPLEX == 1) || (BUILD_COMPLEX16 == 1) - int sgemm_p, sgemm_q, sgemm_r; - int sgemm_unroll_m, sgemm_unroll_n, sgemm_unroll_mn; +#if BUILD_BFLOAT16 == 1 +typedef struct { + int (*sblaswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, float *, BLASLONG, blasint *, float *); +} openblas_sblaswp_dispatch_t; +extern const openblas_sblaswp_dispatch_t *const openblas_sblaswp_dispatch[OPENBLAS_NUM_CORES]; #endif - int exclusive_cache; - #if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +typedef struct { float (*samax_k) (BLASLONG, float *, BLASLONG); +} openblas_samax_dispatch_t; +extern const openblas_samax_dispatch_t *const openblas_samax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +typedef struct { float (*samin_k) (BLASLONG, float *, BLASLONG); +} openblas_samin_dispatch_t; +extern const openblas_samin_dispatch_t *const openblas_samin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +typedef struct { float (*smax_k) (BLASLONG, float *, BLASLONG); +} openblas_smax_dispatch_t; +extern const openblas_smax_dispatch_t *const openblas_smax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +typedef struct { float (*smin_k) (BLASLONG, float *, BLASLONG); +} openblas_smin_dispatch_t; +extern const openblas_smin_dispatch_t *const openblas_smin_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_SINGLE==1) || (BUILD_DOUBLE ==1) || (BUILD_COMPLEX==1) +typedef struct { BLASLONG (*isamax_k)(BLASLONG, float *, BLASLONG); +} openblas_isamax_dispatch_t; +extern const openblas_isamax_dispatch_t *const openblas_isamax_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +typedef struct { BLASLONG (*isamin_k)(BLASLONG, float *, BLASLONG); +} openblas_isamin_dispatch_t; +extern const openblas_isamin_dispatch_t *const openblas_isamin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +typedef struct { BLASLONG (*ismax_k) (BLASLONG, float *, BLASLONG); +} openblas_ismax_dispatch_t; +extern const openblas_ismax_dispatch_t *const openblas_ismax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +typedef struct { BLASLONG (*ismin_k) (BLASLONG, float *, BLASLONG); +} openblas_ismin_dispatch_t; +extern const openblas_ismin_dispatch_t *const openblas_ismin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +typedef struct { float (*snrm2_k) (BLASLONG, float *, BLASLONG); +} openblas_snrm2_dispatch_t; +extern const openblas_snrm2_dispatch_t *const openblas_snrm2_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +typedef struct { float (*sasum_k) (BLASLONG, float *, BLASLONG); +} openblas_sasum_dispatch_t; +extern const openblas_sasum_dispatch_t *const openblas_sasum_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_SINGLE==1) +typedef struct { float (*ssum_k) (BLASLONG, float *, BLASLONG); -#endif +} openblas_ssum_dispatch_t; +extern const openblas_ssum_dispatch_t *const openblas_ssum_dispatch[OPENBLAS_NUM_CORES]; +#endif + #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +typedef struct { int (*scopy_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_scopy_dispatch_t; +extern const openblas_scopy_dispatch_t *const openblas_scopy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +typedef struct { float (*sdot_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG); - //double (*dsdot_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_sdot_dispatch_t; +extern const openblas_sdot_dispatch_t *const openblas_sdot_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +typedef struct { int (*srot_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG, float, float); +} openblas_srot_dispatch_t; +extern const openblas_srot_dispatch_t *const openblas_srot_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +typedef struct { int (*srotm_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_srotm_dispatch_t; +extern const openblas_srotm_dispatch_t *const openblas_srotm_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +typedef struct { int (*saxpy_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_saxpy_dispatch_t; +extern const openblas_saxpy_dispatch_t *const openblas_saxpy_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) || (BUILD_COMPLEX16==1) +typedef struct { int (*sscal_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_sscal_dispatch_t; +extern const openblas_sscal_dispatch_t *const openblas_sscal_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +typedef struct { int (*sswap_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_sswap_dispatch_t; +extern const openblas_sswap_dispatch_t *const openblas_sswap_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +typedef struct { int (*sgemv_n) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*sgemv_t) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_sgemv_dispatch_t; +extern const openblas_sgemv_dispatch_t *const openblas_sgemv_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_SINGLE==1) +typedef struct { int (*sger_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_sger_dispatch_t; +extern const openblas_sger_dispatch_t *const openblas_sger_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE==1) +typedef struct { int (*ssymv_L) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*ssymv_U) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_ssymv_dispatch_t; +extern const openblas_ssymv_dispatch_t *const openblas_ssymv_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +typedef struct { #ifdef ARCH_X86_64 void (*sgemm_direct) (BLASLONG, BLASLONG, BLASLONG, float *, BLASLONG , float *, BLASLONG , float * , BLASLONG); int (*sgemm_direct_performant) (BLASLONG M, BLASLONG N, BLASLONG K); @@ -262,77 +602,62 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL void (*sgemm_direct) (BLASLONG, BLASLONG, BLASLONG, float *, BLASLONG , float *, BLASLONG , float * , BLASLONG); int (*sgemm_direct_performant) (BLASLONG M, BLASLONG N, BLASLONG K); void (*sgemm_direct_alpha_beta) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float * , BLASLONG); - void (*ssymm_direct_alpha_betaLU) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float * , BLASLONG); - void (*ssymm_direct_alpha_betaLL) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float * , BLASLONG); - void (*strmm_direct_LNUN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG); - void (*strmm_direct_LNLN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG); - void (*strmm_direct_LTUN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG); - void (*strmm_direct_LTLN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG); - void (*ssyrk_direct_alpha_betaUN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float, float *, BLASLONG); - void (*ssyrk_direct_alpha_betaUT) (BLASLONG, BLASLONG, float, float *, BLASLONG, float, float *, BLASLONG); - void (*ssyrk_direct_alpha_betaLN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float, float *, BLASLONG); - void (*ssyrk_direct_alpha_betaLT) (BLASLONG, BLASLONG, float, float *, BLASLONG, float, float *, BLASLONG); - void (*ssyr2k_direct_alpha_betaUN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float *, BLASLONG); - void (*ssyr2k_direct_alpha_betaUT) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float *, BLASLONG); - void (*ssyr2k_direct_alpha_betaLN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float *, BLASLONG); - void (*ssyr2k_direct_alpha_betaLT) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float *, BLASLONG); void (*sme_sgemm_kernel) (char*, char*, BLASLONG, BLASLONG, BLASLONG, float*, float *, BLASLONG , float *, BLASLONG ,float*, float *, BLASLONG); #endif - - int (*sgemm_kernel )(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); int (*sgemm_beta )(BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); - int (*ssymm_kernel )(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); - int (*strmm_gemm_kernel)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); - int (*sgemm_incopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*sgemm_itcopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*sgemm_oncopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*sgemm_otcopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); -#endif -#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) #ifdef SMALL_MATRIX_OPT int (*sgemm_small_matrix_permit)(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, float alpha, float beta); - int (*sgemm_small_kernel_nn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); int (*sgemm_small_kernel_nt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); int (*sgemm_small_kernel_tn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); int (*sgemm_small_kernel_tt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); - int (*sgemm_small_kernel_b0_nn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*sgemm_small_kernel_b0_nt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*sgemm_small_kernel_b0_tn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*sgemm_small_kernel_b0_tt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float * C, BLASLONG ldc); #endif +} openblas_sgemm_dispatch_t; +extern const openblas_sgemm_dispatch_t *const openblas_sgemm_dispatch[OPENBLAS_NUM_CORES]; +#endif - int (*strsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*strsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*strsm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*strsm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +typedef struct { +#ifdef ARCH_ARM64 + void (*ssymm_direct_alpha_betaLU) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float * , BLASLONG); + void (*ssymm_direct_alpha_betaLL) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float * , BLASLONG); +#endif + int (*ssymm_kernel )(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); +#if (BUILD_SINGLE==1) + int (*ssymm_incopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); + int (*ssymm_itcopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); + int (*ssymm_iutcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); + int (*ssymm_iltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); + int (*ssymm_outcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); + int (*ssymm_oltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); +#endif +} openblas_ssymm_dispatch_t; +extern const openblas_ssymm_dispatch_t *const openblas_ssymm_dispatch[OPENBLAS_NUM_CORES]; +#endif - int (*strsm_iunucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_iunncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_iutucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_iutncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_ilnucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_ilnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_iltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_iltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_ounucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_ounncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_outucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_outncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_olnucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_olnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_oltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_oltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +typedef struct { +#ifdef ARCH_ARM64 + void (*strmm_direct_LNUN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG); + void (*strmm_direct_LNLN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG); + void (*strmm_direct_LTUN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG); + void (*strmm_direct_LTLN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG); #endif + int (*strmm_gemm_kernel)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); #if (BUILD_SINGLE==1) int (*strmm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); int (*strmm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); int (*strmm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); int (*strmm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*strmm_iunucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*strmm_iunncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*strmm_iutucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); @@ -341,8 +666,6 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*strmm_ilnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*strmm_iltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*strmm_iltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); - int (*ssymm_incopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); - int (*ssymm_itcopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*strmm_incopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*strmm_itcopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*strmm_ounucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); @@ -353,116 +676,280 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*strmm_olnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*strmm_oltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*strmm_oltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); +#endif +} openblas_strmm_dispatch_t; +extern const openblas_strmm_dispatch_t *const openblas_strmm_dispatch[OPENBLAS_NUM_CORES]; +#endif - int (*ssymm_iutcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); - int (*ssymm_iltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); - int (*ssymm_outcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); - int (*ssymm_oltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +#ifdef ARCH_ARM64 +typedef struct { + void (*ssyrk_direct_alpha_betaUN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float, float *, BLASLONG); + void (*ssyrk_direct_alpha_betaUT) (BLASLONG, BLASLONG, float, float *, BLASLONG, float, float *, BLASLONG); + void (*ssyrk_direct_alpha_betaLN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float, float *, BLASLONG); + void (*ssyrk_direct_alpha_betaLT) (BLASLONG, BLASLONG, float, float *, BLASLONG, float, float *, BLASLONG); +} openblas_ssyrk_dispatch_t; +extern const openblas_ssyrk_dispatch_t *const openblas_ssyrk_dispatch[OPENBLAS_NUM_CORES]; +#endif +#endif +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +#ifdef ARCH_ARM64 +typedef struct { + void (*ssyr2k_direct_alpha_betaUN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float *, BLASLONG); + void (*ssyr2k_direct_alpha_betaUT) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float *, BLASLONG); + void (*ssyr2k_direct_alpha_betaLN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float *, BLASLONG); + void (*ssyr2k_direct_alpha_betaLT) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float *, BLASLONG); +} openblas_ssyr2k_dispatch_t; +extern const openblas_ssyr2k_dispatch_t *const openblas_ssyr2k_dispatch[OPENBLAS_NUM_CORES]; +#endif +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +typedef struct { + int (*strsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*strsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*strsm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*strsm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*strsm_iunucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_iunncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_iutucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_iutncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_ilnucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_ilnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_iltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_iltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_ounucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_ounncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_outucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_outncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_olnucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_olnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_oltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_oltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); +} openblas_strsm_dispatch_t; +extern const openblas_strsm_dispatch_t *const openblas_strsm_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE==1) +typedef struct { int (*sneg_tcopy) (BLASLONG, BLASLONG, float *, BLASLONG, float *); +} openblas_sneg_dispatch_t; +extern const openblas_sneg_dispatch_t *const openblas_sneg_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE==1) +typedef struct { int (*slaswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, float *, BLASLONG, blasint *, float *); +} openblas_slaswp_dispatch_t; +extern const openblas_slaswp_dispatch_t *const openblas_slaswp_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) - int dgemm_p, dgemm_q, dgemm_r; - int dgemm_unroll_m, dgemm_unroll_n, dgemm_unroll_mn; +typedef struct { + double (*damax_k) (BLASLONG, double *, BLASLONG); +} openblas_damax_dispatch_t; +extern const openblas_damax_dispatch_t *const openblas_damax_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) - double (*damax_k) (BLASLONG, double *, BLASLONG); +typedef struct { double (*damin_k) (BLASLONG, double *, BLASLONG); +} openblas_damin_dispatch_t; +extern const openblas_damin_dispatch_t *const openblas_damin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { double (*dmax_k) (BLASLONG, double *, BLASLONG); +} openblas_dmax_dispatch_t; +extern const openblas_dmax_dispatch_t *const openblas_dmax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { double (*dmin_k) (BLASLONG, double *, BLASLONG); +} openblas_dmin_dispatch_t; +extern const openblas_dmin_dispatch_t *const openblas_dmin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { BLASLONG (*idamax_k)(BLASLONG, double *, BLASLONG); +} openblas_idamax_dispatch_t; +extern const openblas_idamax_dispatch_t *const openblas_idamax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { BLASLONG (*idamin_k)(BLASLONG, double *, BLASLONG); +} openblas_idamin_dispatch_t; +extern const openblas_idamin_dispatch_t *const openblas_idamin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { BLASLONG (*idmax_k) (BLASLONG, double *, BLASLONG); +} openblas_idmax_dispatch_t; +extern const openblas_idmax_dispatch_t *const openblas_idmax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { BLASLONG (*idmin_k) (BLASLONG, double *, BLASLONG); +} openblas_idmin_dispatch_t; +extern const openblas_idmin_dispatch_t *const openblas_idmin_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { double (*dnrm2_k) (BLASLONG, double *, BLASLONG); +} openblas_dnrm2_dispatch_t; +extern const openblas_dnrm2_dispatch_t *const openblas_dnrm2_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { double (*dasum_k) (BLASLONG, double *, BLASLONG); +} openblas_dasum_dispatch_t; +extern const openblas_dasum_dispatch_t *const openblas_dasum_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_DOUBLE==1) +typedef struct { double (*dsum_k) (BLASLONG, double *, BLASLONG); +} openblas_dsum_dispatch_t; +extern const openblas_dsum_dispatch_t *const openblas_dsum_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { int (*dcopy_k) (BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_dcopy_dispatch_t; +extern const openblas_dcopy_dispatch_t *const openblas_dcopy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { double (*ddot_k) (BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_ddot_dispatch_t; +extern const openblas_ddot_dispatch_t *const openblas_ddot_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) +typedef struct { double (*dsdot_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_dsdot_dispatch_t; +extern const openblas_dsdot_dispatch_t *const openblas_dsdot_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { int (*drot_k) (BLASLONG, double *, BLASLONG, double *, BLASLONG, double, double); +} openblas_drot_dispatch_t; +extern const openblas_drot_dispatch_t *const openblas_drot_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { int (*drotm_k) (BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); +} openblas_drotm_dispatch_t; +extern const openblas_drotm_dispatch_t *const openblas_drotm_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { int (*daxpy_k) (BLASLONG, BLASLONG, BLASLONG, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_daxpy_dispatch_t; +extern const openblas_daxpy_dispatch_t *const openblas_daxpy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { int (*dscal_k) (BLASLONG, BLASLONG, BLASLONG, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_dscal_dispatch_t; +extern const openblas_dscal_dispatch_t *const openblas_dscal_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { int (*dswap_k) (BLASLONG, BLASLONG, BLASLONG, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_dswap_dispatch_t; +extern const openblas_dswap_dispatch_t *const openblas_dswap_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { int (*dgemv_n) (BLASLONG, BLASLONG, BLASLONG, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); int (*dgemv_t) (BLASLONG, BLASLONG, BLASLONG, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); +} openblas_dgemv_dispatch_t; +extern const openblas_dgemv_dispatch_t *const openblas_dgemv_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_DOUBLE==1) +typedef struct { int (*dger_k) (BLASLONG, BLASLONG, BLASLONG, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); +} openblas_dger_dispatch_t; +extern const openblas_dger_dispatch_t *const openblas_dger_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_DOUBLE==1) +typedef struct { int (*dsymv_L) (BLASLONG, BLASLONG, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); int (*dsymv_U) (BLASLONG, BLASLONG, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); +} openblas_dsymv_dispatch_t; +extern const openblas_dsymv_dispatch_t *const openblas_dsymv_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { #ifdef ARCH_ARM64 void (*sme_dgemm_kernel) (const char*, const char*, const BLASLONG, const BLASLONG, const BLASLONG, const double*, const double *, const BLASLONG , const double *, const BLASLONG ,const double*, double *, const BLASLONG); #endif int (*dgemm_kernel )(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); int (*dgemm_beta )(BLASLONG, BLASLONG, BLASLONG, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG); - int (*dsymm_kernel )(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); - int (*dtrmm_gemm_kernel)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); - int (*dgemm_incopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*dgemm_itcopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*dgemm_oncopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*dgemm_otcopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); -#endif -#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) #ifdef SMALL_MATRIX_OPT int (*dgemm_small_matrix_permit)(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, double alpha, double beta); - int (*dgemm_small_kernel_nn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double beta, double * C, BLASLONG ldc); int (*dgemm_small_kernel_nt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double beta, double * C, BLASLONG ldc); int (*dgemm_small_kernel_tn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double beta, double * C, BLASLONG ldc); int (*dgemm_small_kernel_tt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double beta, double * C, BLASLONG ldc); - int (*dgemm_small_kernel_b0_nn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*dgemm_small_kernel_b0_nt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*dgemm_small_kernel_b0_tn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*dgemm_small_kernel_b0_tt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double * C, BLASLONG ldc); #endif +} openblas_dgemm_dispatch_t; +extern const openblas_dgemm_dispatch_t *const openblas_dgemm_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { + int (*dsymm_kernel )(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); +#if (BUILD_DOUBLE==1) + int (*dsymm_incopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); + int (*dsymm_itcopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); + int (*dsymm_iutcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); + int (*dsymm_iltcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); + int (*dsymm_outcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); + int (*dsymm_oltcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); +#endif +} openblas_dsymm_dispatch_t; +extern const openblas_dsymm_dispatch_t *const openblas_dsymm_dispatch[OPENBLAS_NUM_CORES]; #endif -#if (BUILD_DOUBLE==1) - int (*dtrsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*dtrsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*dtrsm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*dtrsm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); - - int (*dtrsm_iunucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_iunncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_iutucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_iutncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_ilnucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_ilnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_iltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_iltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_ounucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_ounncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_outucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_outncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_olnucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_olnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_oltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_oltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { + int (*dtrmm_gemm_kernel)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); +#if (BUILD_DOUBLE==1) int (*dtrmm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); int (*dtrmm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); int (*dtrmm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); int (*dtrmm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*dtrmm_iunucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*dtrmm_iunncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*dtrmm_iutucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); @@ -471,8 +958,6 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*dtrmm_ilnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*dtrmm_iltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*dtrmm_iltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); - int (*dsymm_incopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); - int (*dsymm_itcopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*dtrmm_incopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*dtrmm_itcopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*dtrmm_ounucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); @@ -483,61 +968,218 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*dtrmm_olnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*dtrmm_oltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*dtrmm_oltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); +#endif +} openblas_dtrmm_dispatch_t; +extern const openblas_dtrmm_dispatch_t *const openblas_dtrmm_dispatch[OPENBLAS_NUM_CORES]; +#endif - int (*dsymm_iutcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); - int (*dsymm_iltcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); - int (*dsymm_outcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); - int (*dsymm_oltcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); +#if (BUILD_DOUBLE==1) +typedef struct { + int (*dtrsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*dtrsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*dtrsm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*dtrsm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*dtrsm_iunucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_iunncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_iutucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_iutncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_ilnucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_ilnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_iltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_iltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_ounucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_ounncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_outucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_outncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_olnucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_olnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_oltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_oltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); +} openblas_dtrsm_dispatch_t; +extern const openblas_dtrsm_dispatch_t *const openblas_dtrsm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_DOUBLE==1) +typedef struct { int (*dneg_tcopy) (BLASLONG, BLASLONG, double *, BLASLONG, double *); - int (*dlaswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, double *, BLASLONG, blasint *, double *); +} openblas_dneg_dispatch_t; +extern const openblas_dneg_dispatch_t *const openblas_dneg_dispatch[OPENBLAS_NUM_CORES]; #endif -#ifdef EXPRECISION - int qgemm_p, qgemm_q, qgemm_r; - int qgemm_unroll_m, qgemm_unroll_n, qgemm_unroll_mn; +#if (BUILD_DOUBLE==1) +typedef struct { + int (*dlaswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, double *, BLASLONG, blasint *, double *); +} openblas_dlaswp_dispatch_t; +extern const openblas_dlaswp_dispatch_t *const openblas_dlaswp_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { xdouble (*qamax_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_qamax_dispatch_t; +extern const openblas_qamax_dispatch_t *const openblas_qamax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { xdouble (*qamin_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_qamin_dispatch_t; +extern const openblas_qamin_dispatch_t *const openblas_qamin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { xdouble (*qmax_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_qmax_dispatch_t; +extern const openblas_qmax_dispatch_t *const openblas_qmax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { xdouble (*qmin_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_qmin_dispatch_t; +extern const openblas_qmin_dispatch_t *const openblas_qmin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { BLASLONG (*iqamax_k)(BLASLONG, xdouble *, BLASLONG); +} openblas_iqamax_dispatch_t; +extern const openblas_iqamax_dispatch_t *const openblas_iqamax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { BLASLONG (*iqamin_k)(BLASLONG, xdouble *, BLASLONG); +} openblas_iqamin_dispatch_t; +extern const openblas_iqamin_dispatch_t *const openblas_iqamin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { BLASLONG (*iqmax_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_iqmax_dispatch_t; +extern const openblas_iqmax_dispatch_t *const openblas_iqmax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { BLASLONG (*iqmin_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_iqmin_dispatch_t; +extern const openblas_iqmin_dispatch_t *const openblas_iqmin_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { xdouble (*qnrm2_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_qnrm2_dispatch_t; +extern const openblas_qnrm2_dispatch_t *const openblas_qnrm2_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { xdouble (*qasum_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_qasum_dispatch_t; +extern const openblas_qasum_dispatch_t *const openblas_qasum_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { xdouble (*qsum_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_qsum_dispatch_t; +extern const openblas_qsum_dispatch_t *const openblas_qsum_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*qcopy_k) (BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_qcopy_dispatch_t; +extern const openblas_qcopy_dispatch_t *const openblas_qcopy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { xdouble (*qdot_k) (BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_qdot_dispatch_t; +extern const openblas_qdot_dispatch_t *const openblas_qdot_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*qrot_k) (BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble, xdouble); +} openblas_qrot_dispatch_t; +extern const openblas_qrot_dispatch_t *const openblas_qrot_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*qrotm_k) (BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_qrotm_dispatch_t; +extern const openblas_qrotm_dispatch_t *const openblas_qrotm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*qaxpy_k) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_qaxpy_dispatch_t; +extern const openblas_qaxpy_dispatch_t *const openblas_qaxpy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*qscal_k) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_qscal_dispatch_t; +extern const openblas_qscal_dispatch_t *const openblas_qscal_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*qswap_k) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_qswap_dispatch_t; +extern const openblas_qswap_dispatch_t *const openblas_qswap_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*qgemv_n) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*qgemv_t) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_qgemv_dispatch_t; +extern const openblas_qgemv_dispatch_t *const openblas_qgemv_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*qger_k) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_qger_dispatch_t; +extern const openblas_qger_dispatch_t *const openblas_qger_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*qsymv_L) (BLASLONG, BLASLONG, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*qsymv_U) (BLASLONG, BLASLONG, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_qsymv_dispatch_t; +extern const openblas_qsymv_dispatch_t *const openblas_qsymv_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*qgemm_kernel )(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG); int (*qgemm_beta )(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); - int (*qgemm_incopy )(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*qgemm_itcopy )(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*qgemm_oncopy )(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*qgemm_otcopy )(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_qgemm_dispatch_t; +extern const openblas_qgemm_dispatch_t *const openblas_qgemm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*qtrsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*qtrsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*qtrsm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*qtrsm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); - int (*qtrsm_iunucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); int (*qtrsm_iunncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); int (*qtrsm_iutucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); @@ -554,12 +1196,16 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*qtrsm_olnncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); int (*qtrsm_oltucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); int (*qtrsm_oltncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); +} openblas_qtrsm_dispatch_t; +extern const openblas_qtrsm_dispatch_t *const openblas_qtrsm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*qtrmm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*qtrmm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*qtrmm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*qtrmm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); - int (*qtrmm_iunucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*qtrmm_iunncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*qtrmm_iutucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); @@ -576,41 +1222,141 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*qtrmm_olnncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*qtrmm_oltucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*qtrmm_oltncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); +} openblas_qtrmm_dispatch_t; +extern const openblas_qtrmm_dispatch_t *const openblas_qtrmm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*qsymm_iutcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*qsymm_iltcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*qsymm_outcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*qsymm_oltcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); +} openblas_qsymm_dispatch_t; +extern const openblas_qsymm_dispatch_t *const openblas_qsymm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*qneg_tcopy) (BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); - int (*qlaswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, xdouble *, BLASLONG, blasint *, xdouble *); +} openblas_qneg_dispatch_t; +extern const openblas_qneg_dispatch_t *const openblas_qneg_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { + int (*qlaswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, xdouble *, BLASLONG, blasint *, xdouble *); +} openblas_qlaswp_dispatch_t; +extern const openblas_qlaswp_dispatch_t *const openblas_qlaswp_dispatch[OPENBLAS_NUM_CORES]; #endif -#if (BUILD_COMPLEX==1) - int cgemm_p, cgemm_q, cgemm_r; - int cgemm_unroll_m, cgemm_unroll_n, cgemm_unroll_mn; - +#if (BUILD_COMPLEX==1) +typedef struct { float (*camax_k) (BLASLONG, float *, BLASLONG); +} openblas_camax_dispatch_t; +extern const openblas_camax_dispatch_t *const openblas_camax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { float (*camin_k) (BLASLONG, float *, BLASLONG); - +} openblas_camin_dispatch_t; +extern const openblas_camin_dispatch_t *const openblas_camin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { BLASLONG (*icamax_k)(BLASLONG, float *, BLASLONG); +} openblas_icamax_dispatch_t; +extern const openblas_icamax_dispatch_t *const openblas_icamax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { BLASLONG (*icamin_k)(BLASLONG, float *, BLASLONG); +} openblas_icamin_dispatch_t; +extern const openblas_icamin_dispatch_t *const openblas_icamin_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX==1) +typedef struct { float (*cnrm2_k) (BLASLONG, float *, BLASLONG); +} openblas_cnrm2_dispatch_t; +extern const openblas_cnrm2_dispatch_t *const openblas_cnrm2_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { float (*casum_k) (BLASLONG, float *, BLASLONG); +} openblas_casum_dispatch_t; +extern const openblas_casum_dispatch_t *const openblas_casum_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { float (*csum_k) (BLASLONG, float *, BLASLONG); +} openblas_csum_dispatch_t; +extern const openblas_csum_dispatch_t *const openblas_csum_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX==1) +typedef struct { int (*ccopy_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_ccopy_dispatch_t; +extern const openblas_ccopy_dispatch_t *const openblas_ccopy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { openblas_complex_float (*cdotu_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_cdotu_dispatch_t; +extern const openblas_cdotu_dispatch_t *const openblas_cdotu_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { openblas_complex_float (*cdotc_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_cdotc_dispatch_t; +extern const openblas_cdotc_dispatch_t *const openblas_cdotc_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX==1) +typedef struct { int (*csrot_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG, float, float); +} openblas_csrot_dispatch_t; +extern const openblas_csrot_dispatch_t *const openblas_csrot_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { int (*caxpy_k) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_caxpy_dispatch_t; +extern const openblas_caxpy_dispatch_t *const openblas_caxpy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { int (*caxpyc_k)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_caxpyc_dispatch_t; +extern const openblas_caxpyc_dispatch_t *const openblas_caxpyc_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { int (*cscal_k) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_cscal_dispatch_t; +extern const openblas_cscal_dispatch_t *const openblas_cscal_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { int (*cswap_k) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_cswap_dispatch_t; +extern const openblas_cswap_dispatch_t *const openblas_cswap_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX==1) +typedef struct { int (*cgemv_n) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*cgemv_t) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*cgemv_r) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); @@ -619,18 +1365,58 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*cgemv_u) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*cgemv_s) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*cgemv_d) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_cgemv_dispatch_t; +extern const openblas_cgemv_dispatch_t *const openblas_cgemv_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { int (*cgeru_k) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_cgeru_dispatch_t; +extern const openblas_cgeru_dispatch_t *const openblas_cgeru_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { int (*cgerc_k) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_cgerc_dispatch_t; +extern const openblas_cgerc_dispatch_t *const openblas_cgerc_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { int (*cgerv_k) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_cgerv_dispatch_t; +extern const openblas_cgerv_dispatch_t *const openblas_cgerv_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { int (*cgerd_k) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_cgerd_dispatch_t; +extern const openblas_cgerd_dispatch_t *const openblas_cgerd_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX==1) +typedef struct { int (*csymv_L) (BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*csymv_U) (BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_csymv_dispatch_t; +extern const openblas_csymv_dispatch_t *const openblas_csymv_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { int (*chemv_L) (BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*chemv_U) (BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*chemv_M) (BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*chemv_V) (BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_chemv_dispatch_t; +extern const openblas_chemv_dispatch_t *const openblas_chemv_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX==1) +typedef struct { #ifdef ARCH_ARM64 void (*sme_cgemm_kernel) (const char*, const char*, const BLASLONG, const BLASLONG, const BLASLONG, const float, const float, const float *, const BLASLONG , const float *, const BLASLONG, const float, const float, float *, const BLASLONG); #endif @@ -639,92 +1425,71 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*cgemm_kernel_r )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); int (*cgemm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); int (*cgemm_beta )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); - - int (*csymm_kernel_n )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); - int (*csymm_kernel_l )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); - int (*csymm_kernel_r )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); - int (*csymm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); - - int (*ctrmm_gemm_kernel_n)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); - int (*ctrmm_gemm_kernel_l)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); - int (*ctrmm_gemm_kernel_r)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); - int (*ctrmm_gemm_kernel_b)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); - int (*cgemm_incopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*cgemm_itcopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*cgemm_oncopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*cgemm_otcopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); - #ifdef SMALL_MATRIX_OPT int (*cgemm_small_matrix_permit)(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, float alpha0, float alpha1, float beta0, float beta1); - int (*cgemm_small_kernel_nn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_nt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_nr )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_nc )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); - int (*cgemm_small_kernel_tn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_tt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_tr )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_tc )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); - int (*cgemm_small_kernel_rn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_rt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_rr )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_rc )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); - int (*cgemm_small_kernel_cn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_ct )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_cr )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_cc )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); - int (*cgemm_small_kernel_b0_nn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_nt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_nr )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_nc )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); - int (*cgemm_small_kernel_b0_tn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_tt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_tr )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_tc )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); - int (*cgemm_small_kernel_b0_rn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_rt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_rr )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_rc )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); - int (*cgemm_small_kernel_b0_cn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_ct )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_cr )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_cc )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); #endif +} openblas_cgemm_dispatch_t; +extern const openblas_cgemm_dispatch_t *const openblas_cgemm_dispatch[OPENBLAS_NUM_CORES]; +#endif - int (*ctrsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*ctrsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*ctrsm_kernel_LR)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*ctrsm_kernel_LC)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*ctrsm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*ctrsm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*ctrsm_kernel_RR)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*ctrsm_kernel_RC)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); - - int (*ctrsm_iunucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_iunncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_iutucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_iutncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_ilnucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_ilnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_iltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_iltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_ounucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_ounncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_outucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_outncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_olnucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_olnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_oltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_oltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); +#if (BUILD_COMPLEX==1) +typedef struct { + int (*csymm_kernel_n )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*csymm_kernel_l )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*csymm_kernel_r )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*csymm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*csymm_incopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); + int (*csymm_itcopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); + int (*csymm_iutcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); + int (*csymm_iltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); + int (*csymm_outcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); + int (*csymm_oltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); +} openblas_csymm_dispatch_t; +extern const openblas_csymm_dispatch_t *const openblas_csymm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX==1) +typedef struct { + int (*ctrmm_gemm_kernel_n)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*ctrmm_gemm_kernel_l)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*ctrmm_gemm_kernel_r)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*ctrmm_gemm_kernel_b)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); int (*ctrmm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); int (*ctrmm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); int (*ctrmm_kernel_RR)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); @@ -733,7 +1498,6 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*ctrmm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); int (*ctrmm_kernel_LR)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); int (*ctrmm_kernel_LC)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*ctrmm_iunucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*ctrmm_iunncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*ctrmm_iutucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); @@ -742,8 +1506,6 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*ctrmm_ilnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*ctrmm_iltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*ctrmm_iltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); - int (*csymm_incopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); - int (*csymm_itcopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*ctrmm_incopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*ctrmm_itcopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*ctrmm_ounucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); @@ -754,90 +1516,226 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*ctrmm_olnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*ctrmm_oltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*ctrmm_oltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); +} openblas_ctrmm_dispatch_t; +extern const openblas_ctrmm_dispatch_t *const openblas_ctrmm_dispatch[OPENBLAS_NUM_CORES]; +#endif - int (*csymm_iutcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); - int (*csymm_iltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); - int (*csymm_outcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); - int (*csymm_oltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); +#if (BUILD_COMPLEX==1) +typedef struct { + int (*ctrsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*ctrsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*ctrsm_kernel_LR)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*ctrsm_kernel_LC)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*ctrsm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*ctrsm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*ctrsm_kernel_RR)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*ctrsm_kernel_RC)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*ctrsm_iunucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_iunncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_iutucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_iutncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_ilnucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_ilnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_iltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_iltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_ounucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_ounncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_outucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_outncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_olnucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_olnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_oltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_oltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); +} openblas_ctrsm_dispatch_t; +extern const openblas_ctrsm_dispatch_t *const openblas_ctrsm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX==1) +typedef struct { int (*chemm_iutcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*chemm_iltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*chemm_outcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*chemm_oltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); +} openblas_chemm_dispatch_t; +extern const openblas_chemm_dispatch_t *const openblas_chemm_dispatch[OPENBLAS_NUM_CORES]; +#endif - int cgemm3m_p, cgemm3m_q, cgemm3m_r; - int cgemm3m_unroll_m, cgemm3m_unroll_n, cgemm3m_unroll_mn; - +#if (BUILD_COMPLEX==1) +typedef struct { int (*cgemm3m_kernel)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); - int (*cgemm3m_incopyb)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*cgemm3m_incopyr)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*cgemm3m_incopyi)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*cgemm3m_itcopyb)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*cgemm3m_itcopyr)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*cgemm3m_itcopyi)(BLASLONG, BLASLONG, float *, BLASLONG, float *); - int (*cgemm3m_oncopyb)(BLASLONG, BLASLONG, float *, BLASLONG, float, float, float *); int (*cgemm3m_oncopyr)(BLASLONG, BLASLONG, float *, BLASLONG, float, float, float *); int (*cgemm3m_oncopyi)(BLASLONG, BLASLONG, float *, BLASLONG, float, float, float *); int (*cgemm3m_otcopyb)(BLASLONG, BLASLONG, float *, BLASLONG, float, float, float *); int (*cgemm3m_otcopyr)(BLASLONG, BLASLONG, float *, BLASLONG, float, float, float *); int (*cgemm3m_otcopyi)(BLASLONG, BLASLONG, float *, BLASLONG, float, float, float *); +} openblas_cgemm3m_dispatch_t; +extern const openblas_cgemm3m_dispatch_t *const openblas_cgemm3m_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX==1) +typedef struct { int (*csymm3m_iucopyb)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*csymm3m_ilcopyb)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*csymm3m_iucopyr)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*csymm3m_ilcopyr)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*csymm3m_iucopyi)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*csymm3m_ilcopyi)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); - int (*csymm3m_oucopyb)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); int (*csymm3m_olcopyb)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); int (*csymm3m_oucopyr)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); int (*csymm3m_olcopyr)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); int (*csymm3m_oucopyi)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); int (*csymm3m_olcopyi)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); +} openblas_csymm3m_dispatch_t; +extern const openblas_csymm3m_dispatch_t *const openblas_csymm3m_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX==1) +typedef struct { int (*chemm3m_iucopyb)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*chemm3m_ilcopyb)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*chemm3m_iucopyr)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*chemm3m_ilcopyr)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*chemm3m_iucopyi)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*chemm3m_ilcopyi)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); - int (*chemm3m_oucopyb)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); int (*chemm3m_olcopyb)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); int (*chemm3m_oucopyr)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); int (*chemm3m_olcopyr)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); int (*chemm3m_oucopyi)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); int (*chemm3m_olcopyi)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); +} openblas_chemm3m_dispatch_t; +extern const openblas_chemm3m_dispatch_t *const openblas_chemm3m_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX==1) +typedef struct { int (*cneg_tcopy) (BLASLONG, BLASLONG, float *, BLASLONG, float *); +} openblas_cneg_dispatch_t; +extern const openblas_cneg_dispatch_t *const openblas_cneg_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { int (*claswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, float *, BLASLONG, blasint *, float *); +} openblas_claswp_dispatch_t; +extern const openblas_claswp_dispatch_t *const openblas_claswp_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_COMPLEX16 == 1) - int zgemm_p, zgemm_q, zgemm_r; - int zgemm_unroll_m, zgemm_unroll_n, zgemm_unroll_mn; - +typedef struct { double (*zamax_k) (BLASLONG, double *, BLASLONG); +} openblas_zamax_dispatch_t; +extern const openblas_zamax_dispatch_t *const openblas_zamax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { double (*zamin_k) (BLASLONG, double *, BLASLONG); +} openblas_zamin_dispatch_t; +extern const openblas_zamin_dispatch_t *const openblas_zamin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { BLASLONG (*izamax_k)(BLASLONG, double *, BLASLONG); +} openblas_izamax_dispatch_t; +extern const openblas_izamax_dispatch_t *const openblas_izamax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { BLASLONG (*izamin_k)(BLASLONG, double *, BLASLONG); +} openblas_izamin_dispatch_t; +extern const openblas_izamin_dispatch_t *const openblas_izamin_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX16 == 1) +typedef struct { double (*znrm2_k) (BLASLONG, double *, BLASLONG); +} openblas_znrm2_dispatch_t; +extern const openblas_znrm2_dispatch_t *const openblas_znrm2_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { double (*zasum_k) (BLASLONG, double *, BLASLONG); +} openblas_zasum_dispatch_t; +extern const openblas_zasum_dispatch_t *const openblas_zasum_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { double (*zsum_k) (BLASLONG, double *, BLASLONG); +} openblas_zsum_dispatch_t; +extern const openblas_zsum_dispatch_t *const openblas_zsum_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zcopy_k) (BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_zcopy_dispatch_t; +extern const openblas_zcopy_dispatch_t *const openblas_zcopy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { openblas_complex_double (*zdotu_k) (BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_zdotu_dispatch_t; +extern const openblas_zdotu_dispatch_t *const openblas_zdotu_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { openblas_complex_double (*zdotc_k) (BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_zdotc_dispatch_t; +extern const openblas_zdotc_dispatch_t *const openblas_zdotc_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zdrot_k) (BLASLONG, double *, BLASLONG, double *, BLASLONG, double, double); +} openblas_zdrot_dispatch_t; +extern const openblas_zdrot_dispatch_t *const openblas_zdrot_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zaxpy_k) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_zaxpy_dispatch_t; +extern const openblas_zaxpy_dispatch_t *const openblas_zaxpy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zaxpyc_k)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_zaxpyc_dispatch_t; +extern const openblas_zaxpyc_dispatch_t *const openblas_zaxpyc_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zscal_k) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_zscal_dispatch_t; +extern const openblas_zscal_dispatch_t *const openblas_zscal_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zswap_k) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_zswap_dispatch_t; +extern const openblas_zswap_dispatch_t *const openblas_zswap_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zgemv_n) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); int (*zgemv_t) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); int (*zgemv_r) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); @@ -846,113 +1744,131 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*zgemv_u) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); int (*zgemv_s) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); int (*zgemv_d) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); +} openblas_zgemv_dispatch_t; +extern const openblas_zgemv_dispatch_t *const openblas_zgemv_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zgeru_k) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); +} openblas_zgeru_dispatch_t; +extern const openblas_zgeru_dispatch_t *const openblas_zgeru_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zgerc_k) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); +} openblas_zgerc_dispatch_t; +extern const openblas_zgerc_dispatch_t *const openblas_zgerc_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zgerv_k) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); +} openblas_zgerv_dispatch_t; +extern const openblas_zgerv_dispatch_t *const openblas_zgerv_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zgerd_k) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); +} openblas_zgerd_dispatch_t; +extern const openblas_zgerd_dispatch_t *const openblas_zgerd_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zsymv_L) (BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); int (*zsymv_U) (BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); +} openblas_zsymv_dispatch_t; +extern const openblas_zsymv_dispatch_t *const openblas_zsymv_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zhemv_L) (BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); int (*zhemv_U) (BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); int (*zhemv_M) (BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); int (*zhemv_V) (BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); +} openblas_zhemv_dispatch_t; +extern const openblas_zhemv_dispatch_t *const openblas_zhemv_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX16 == 1) +typedef struct { #ifdef ARCH_ARM64 void (*sme_zgemm_kernel) (const char*, const char*, const BLASLONG, const BLASLONG, const BLASLONG, const double, const double, const double *, const BLASLONG , const double *, const BLASLONG, const double, const double, double *, const BLASLONG); #endif - int (*zgemm_kernel_n )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); int (*zgemm_kernel_l )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); int (*zgemm_kernel_r )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); int (*zgemm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); int (*zgemm_beta )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG); - - int (*zsymm_kernel_n )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); - int (*zsymm_kernel_l )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); - int (*zsymm_kernel_r )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); - int (*zsymm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); - - int (*ztrmm_gemm_kernel_n)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); - int (*ztrmm_gemm_kernel_l)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); - int (*ztrmm_gemm_kernel_r)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); - int (*ztrmm_gemm_kernel_b)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); - int (*zgemm_incopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*zgemm_itcopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*zgemm_oncopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*zgemm_otcopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); - #ifdef SMALL_MATRIX_OPT int (*zgemm_small_matrix_permit)(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, double alpha0, double alpha1, double beta0, double beta1); - int (*zgemm_small_kernel_nn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_nt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_nr )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_nc )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); - int (*zgemm_small_kernel_tn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_tt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_tr )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_tc )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); - int (*zgemm_small_kernel_rn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_rt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_rr )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_rc )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); - int (*zgemm_small_kernel_cn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_ct )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_cr )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_cc )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); - int (*zgemm_small_kernel_b0_nn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_nt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_nr )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_nc )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); - int (*zgemm_small_kernel_b0_tn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_tt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_tr )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_tc )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); - int (*zgemm_small_kernel_b0_rn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_rt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_rr )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_rc )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); - int (*zgemm_small_kernel_b0_cn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_ct )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_cr )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_cc )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); #endif +} openblas_zgemm_dispatch_t; +extern const openblas_zgemm_dispatch_t *const openblas_zgemm_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { + int (*zsymm_kernel_n )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*zsymm_kernel_l )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*zsymm_kernel_r )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*zsymm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*zsymm_incopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); + int (*zsymm_itcopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); + int (*zsymm_iutcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); + int (*zsymm_iltcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); + int (*zsymm_outcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); + int (*zsymm_oltcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); +} openblas_zsymm_dispatch_t; +extern const openblas_zsymm_dispatch_t *const openblas_zsymm_dispatch[OPENBLAS_NUM_CORES]; +#endif - int (*ztrsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*ztrsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*ztrsm_kernel_LR)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*ztrsm_kernel_LC)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*ztrsm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*ztrsm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*ztrsm_kernel_RR)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*ztrsm_kernel_RC)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); - - int (*ztrsm_iunucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_iunncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_iutucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_iutncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_ilnucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_ilnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_iltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_iltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_ounucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_ounncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_outucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_outncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_olnucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_olnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_oltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_oltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - +#if (BUILD_COMPLEX16 == 1) +typedef struct { + int (*ztrmm_gemm_kernel_n)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*ztrmm_gemm_kernel_l)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*ztrmm_gemm_kernel_r)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*ztrmm_gemm_kernel_b)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); int (*ztrmm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); int (*ztrmm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); int (*ztrmm_kernel_RR)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); @@ -961,7 +1877,6 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*ztrmm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); int (*ztrmm_kernel_LR)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); int (*ztrmm_kernel_LC)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*ztrmm_iunucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*ztrmm_iunncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*ztrmm_iutucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); @@ -970,8 +1885,6 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*ztrmm_ilnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*ztrmm_iltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*ztrmm_iltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); - int (*zsymm_incopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); - int (*zsymm_itcopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*ztrmm_incopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*ztrmm_itcopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*ztrmm_ounucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); @@ -982,91 +1895,226 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*ztrmm_olnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*ztrmm_oltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*ztrmm_oltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); +} openblas_ztrmm_dispatch_t; +extern const openblas_ztrmm_dispatch_t *const openblas_ztrmm_dispatch[OPENBLAS_NUM_CORES]; +#endif - int (*zsymm_iutcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); - int (*zsymm_iltcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); - int (*zsymm_outcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); - int (*zsymm_oltcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); +#if (BUILD_COMPLEX16 == 1) +typedef struct { + int (*ztrsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*ztrsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*ztrsm_kernel_LR)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*ztrsm_kernel_LC)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*ztrsm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*ztrsm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*ztrsm_kernel_RR)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*ztrsm_kernel_RC)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*ztrsm_iunucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_iunncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_iutucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_iutncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_ilnucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_ilnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_iltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_iltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_ounucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_ounncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_outucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_outncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_olnucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_olnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_oltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_oltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); +} openblas_ztrsm_dispatch_t; +extern const openblas_ztrsm_dispatch_t *const openblas_ztrsm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zhemm_iutcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zhemm_iltcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zhemm_outcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zhemm_oltcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); +} openblas_zhemm_dispatch_t; +extern const openblas_zhemm_dispatch_t *const openblas_zhemm_dispatch[OPENBLAS_NUM_CORES]; +#endif - int zgemm3m_p, zgemm3m_q, zgemm3m_r; - int zgemm3m_unroll_m, zgemm3m_unroll_n, zgemm3m_unroll_mn; - +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zgemm3m_kernel)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); - int (*zgemm3m_incopyb)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*zgemm3m_incopyr)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*zgemm3m_incopyi)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*zgemm3m_itcopyb)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*zgemm3m_itcopyr)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*zgemm3m_itcopyi)(BLASLONG, BLASLONG, double *, BLASLONG, double *); - int (*zgemm3m_oncopyb)(BLASLONG, BLASLONG, double *, BLASLONG, double, double, double *); int (*zgemm3m_oncopyr)(BLASLONG, BLASLONG, double *, BLASLONG, double, double, double *); int (*zgemm3m_oncopyi)(BLASLONG, BLASLONG, double *, BLASLONG, double, double, double *); int (*zgemm3m_otcopyb)(BLASLONG, BLASLONG, double *, BLASLONG, double, double, double *); int (*zgemm3m_otcopyr)(BLASLONG, BLASLONG, double *, BLASLONG, double, double, double *); int (*zgemm3m_otcopyi)(BLASLONG, BLASLONG, double *, BLASLONG, double, double, double *); +} openblas_zgemm3m_dispatch_t; +extern const openblas_zgemm3m_dispatch_t *const openblas_zgemm3m_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zsymm3m_iucopyb)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zsymm3m_ilcopyb)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zsymm3m_iucopyr)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zsymm3m_ilcopyr)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zsymm3m_iucopyi)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zsymm3m_ilcopyi)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); - int (*zsymm3m_oucopyb)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); int (*zsymm3m_olcopyb)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); int (*zsymm3m_oucopyr)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); int (*zsymm3m_olcopyr)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); int (*zsymm3m_oucopyi)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); int (*zsymm3m_olcopyi)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); +} openblas_zsymm3m_dispatch_t; +extern const openblas_zsymm3m_dispatch_t *const openblas_zsymm3m_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zhemm3m_iucopyb)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zhemm3m_ilcopyb)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zhemm3m_iucopyr)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zhemm3m_ilcopyr)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zhemm3m_iucopyi)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zhemm3m_ilcopyi)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); - int (*zhemm3m_oucopyb)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); int (*zhemm3m_olcopyb)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); int (*zhemm3m_oucopyr)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); int (*zhemm3m_olcopyr)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); int (*zhemm3m_oucopyi)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); int (*zhemm3m_olcopyi)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); +} openblas_zhemm3m_dispatch_t; +extern const openblas_zhemm3m_dispatch_t *const openblas_zhemm3m_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zneg_tcopy) (BLASLONG, BLASLONG, double *, BLASLONG, double *); +} openblas_zneg_dispatch_t; +extern const openblas_zneg_dispatch_t *const openblas_zneg_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zlaswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, double *, BLASLONG, blasint *, double *); +} openblas_zlaswp_dispatch_t; +extern const openblas_zlaswp_dispatch_t *const openblas_zlaswp_dispatch[OPENBLAS_NUM_CORES]; #endif #ifdef EXPRECISION - - int xgemm_p, xgemm_q, xgemm_r; - int xgemm_unroll_m, xgemm_unroll_n, xgemm_unroll_mn; - +typedef struct { xdouble (*xamax_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_xamax_dispatch_t; +extern const openblas_xamax_dispatch_t *const openblas_xamax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { xdouble (*xamin_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_xamin_dispatch_t; +extern const openblas_xamin_dispatch_t *const openblas_xamin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { BLASLONG (*ixamax_k)(BLASLONG, xdouble *, BLASLONG); +} openblas_ixamax_dispatch_t; +extern const openblas_ixamax_dispatch_t *const openblas_ixamax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { BLASLONG (*ixamin_k)(BLASLONG, xdouble *, BLASLONG); +} openblas_ixamin_dispatch_t; +extern const openblas_ixamin_dispatch_t *const openblas_ixamin_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { xdouble (*xnrm2_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_xnrm2_dispatch_t; +extern const openblas_xnrm2_dispatch_t *const openblas_xnrm2_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { xdouble (*xasum_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_xasum_dispatch_t; +extern const openblas_xasum_dispatch_t *const openblas_xasum_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { xdouble (*xsum_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_xsum_dispatch_t; +extern const openblas_xsum_dispatch_t *const openblas_xsum_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*xcopy_k) (BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_xcopy_dispatch_t; +extern const openblas_xcopy_dispatch_t *const openblas_xcopy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { openblas_complex_xdouble (*xdotu_k) (BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_xdotu_dispatch_t; +extern const openblas_xdotu_dispatch_t *const openblas_xdotu_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { openblas_complex_xdouble (*xdotc_k) (BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_xdotc_dispatch_t; +extern const openblas_xdotc_dispatch_t *const openblas_xdotc_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*xqrot_k) (BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble, xdouble); +} openblas_xqrot_dispatch_t; +extern const openblas_xqrot_dispatch_t *const openblas_xqrot_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*xaxpy_k) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_xaxpy_dispatch_t; +extern const openblas_xaxpy_dispatch_t *const openblas_xaxpy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*xaxpyc_k)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_xaxpyc_dispatch_t; +extern const openblas_xaxpyc_dispatch_t *const openblas_xaxpyc_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*xscal_k) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_xscal_dispatch_t; +extern const openblas_xscal_dispatch_t *const openblas_xscal_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*xswap_k) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_xswap_dispatch_t; +extern const openblas_xswap_dispatch_t *const openblas_xswap_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*xgemv_n) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemv_t) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemv_r) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); @@ -1075,29 +2123,73 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*xgemv_u) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemv_s) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemv_d) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_xgemv_dispatch_t; +extern const openblas_xgemv_dispatch_t *const openblas_xgemv_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*xgeru_k) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_xgeru_dispatch_t; +extern const openblas_xgeru_dispatch_t *const openblas_xgeru_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*xgerc_k) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_xgerc_dispatch_t; +extern const openblas_xgerc_dispatch_t *const openblas_xgerc_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*xgerv_k) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_xgerv_dispatch_t; +extern const openblas_xgerv_dispatch_t *const openblas_xgerv_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*xgerd_k) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_xgerd_dispatch_t; +extern const openblas_xgerd_dispatch_t *const openblas_xgerd_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*xsymv_L) (BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xsymv_U) (BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_xsymv_dispatch_t; +extern const openblas_xsymv_dispatch_t *const openblas_xsymv_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*xhemv_L) (BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xhemv_U) (BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xhemv_M) (BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xhemv_V) (BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_xhemv_dispatch_t; +extern const openblas_xhemv_dispatch_t *const openblas_xhemv_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*xgemm_kernel_n )(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG); int (*xgemm_kernel_l )(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG); int (*xgemm_kernel_r )(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG); int (*xgemm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG); int (*xgemm_beta )(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); - int (*xgemm_incopy )(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemm_itcopy )(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemm_oncopy )(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemm_otcopy )(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_xgemm_dispatch_t; +extern const openblas_xgemm_dispatch_t *const openblas_xgemm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*xtrsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*xtrsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*xtrsm_kernel_LR)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); @@ -1106,7 +2198,6 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*xtrsm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*xtrsm_kernel_RR)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*xtrsm_kernel_RC)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); - int (*xtrsm_iunucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); int (*xtrsm_iunncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); int (*xtrsm_iutucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); @@ -1123,7 +2214,12 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*xtrsm_olnncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); int (*xtrsm_oltucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); int (*xtrsm_oltncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); +} openblas_xtrsm_dispatch_t; +extern const openblas_xtrsm_dispatch_t *const openblas_xtrsm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*xtrmm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*xtrmm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*xtrmm_kernel_RR)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); @@ -1132,7 +2228,6 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*xtrmm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*xtrmm_kernel_LR)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*xtrmm_kernel_LC)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); - int (*xtrmm_iunucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xtrmm_iunncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xtrmm_iutucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); @@ -1149,179 +2244,250 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*xtrmm_olnncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xtrmm_oltucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xtrmm_oltncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); +} openblas_xtrmm_dispatch_t; +extern const openblas_xtrmm_dispatch_t *const openblas_xtrmm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*xsymm_iutcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xsymm_iltcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xsymm_outcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xsymm_oltcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); +} openblas_xsymm_dispatch_t; +extern const openblas_xsymm_dispatch_t *const openblas_xsymm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*xhemm_iutcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xhemm_iltcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xhemm_outcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xhemm_oltcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); +} openblas_xhemm_dispatch_t; +extern const openblas_xhemm_dispatch_t *const openblas_xhemm_dispatch[OPENBLAS_NUM_CORES]; +#endif - int xgemm3m_p, xgemm3m_q, xgemm3m_r; - int xgemm3m_unroll_m, xgemm3m_unroll_n, xgemm3m_unroll_mn; - +#ifdef EXPRECISION +typedef struct { int (*xgemm3m_kernel)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG); - int (*xgemm3m_incopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemm3m_incopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemm3m_incopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemm3m_itcopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemm3m_itcopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemm3m_itcopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); - int (*xgemm3m_oncopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble, xdouble, xdouble *); int (*xgemm3m_oncopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble, xdouble, xdouble *); int (*xgemm3m_oncopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble, xdouble, xdouble *); int (*xgemm3m_otcopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble, xdouble, xdouble *); int (*xgemm3m_otcopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble, xdouble, xdouble *); int (*xgemm3m_otcopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble, xdouble, xdouble *); +} openblas_xgemm3m_dispatch_t; +extern const openblas_xgemm3m_dispatch_t *const openblas_xgemm3m_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*xsymm3m_iucopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xsymm3m_ilcopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xsymm3m_iucopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xsymm3m_ilcopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xsymm3m_iucopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xsymm3m_ilcopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); - int (*xsymm3m_oucopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); int (*xsymm3m_olcopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); int (*xsymm3m_oucopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); int (*xsymm3m_olcopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); int (*xsymm3m_oucopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); int (*xsymm3m_olcopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); +} openblas_xsymm3m_dispatch_t; +extern const openblas_xsymm3m_dispatch_t *const openblas_xsymm3m_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*xhemm3m_iucopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xhemm3m_ilcopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xhemm3m_iucopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xhemm3m_ilcopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xhemm3m_iucopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xhemm3m_ilcopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); - int (*xhemm3m_oucopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); int (*xhemm3m_olcopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); int (*xhemm3m_oucopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); int (*xhemm3m_olcopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); int (*xhemm3m_oucopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); int (*xhemm3m_olcopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); +} openblas_xhemm3m_dispatch_t; +extern const openblas_xhemm3m_dispatch_t *const openblas_xhemm3m_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*xneg_tcopy) (BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); - int (*xlaswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, xdouble *, BLASLONG, blasint *, xdouble *); - +} openblas_xneg_dispatch_t; +extern const openblas_xneg_dispatch_t *const openblas_xneg_dispatch[OPENBLAS_NUM_CORES]; #endif +#ifdef EXPRECISION +typedef struct { + int (*xlaswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, xdouble *, BLASLONG, blasint *, xdouble *); +} openblas_xlaswp_dispatch_t; +extern const openblas_xlaswp_dispatch_t *const openblas_xlaswp_dispatch[OPENBLAS_NUM_CORES]; +#endif - void (*init)(void); - - int snum_opt, dnum_opt, qnum_opt; #if (BUILD_SINGLE==1) +typedef struct { int (*saxpby_k) (BLASLONG, float, float*, BLASLONG,float, float*, BLASLONG); +} openblas_saxpby_dispatch_t; +extern const openblas_saxpby_dispatch_t *const openblas_saxpby_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_DOUBLE==1) +typedef struct { int (*daxpby_k) (BLASLONG, double, double*, BLASLONG,double, double*, BLASLONG); +} openblas_daxpby_dispatch_t; +extern const openblas_daxpby_dispatch_t *const openblas_daxpby_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_COMPLEX==1) +typedef struct { int (*caxpby_k) (BLASLONG, float, float, float*, BLASLONG,float,float, float*, BLASLONG); +} openblas_caxpby_dispatch_t; +extern const openblas_caxpby_dispatch_t *const openblas_caxpby_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_COMPLEX16==1) +typedef struct { int (*zaxpby_k) (BLASLONG, double, double, double*, BLASLONG,double,double, double*, BLASLONG); +} openblas_zaxpby_dispatch_t; +extern const openblas_zaxpby_dispatch_t *const openblas_zaxpby_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_SINGLE==1) +typedef struct { int (*somatcopy_k_cn) (BLASLONG, BLASLONG, float, float*, BLASLONG, float*, BLASLONG); int (*somatcopy_k_ct) (BLASLONG, BLASLONG, float, float*, BLASLONG, float*, BLASLONG); int (*somatcopy_k_rn) (BLASLONG, BLASLONG, float, float*, BLASLONG, float*, BLASLONG); int (*somatcopy_k_rt) (BLASLONG, BLASLONG, float, float*, BLASLONG, float*, BLASLONG); +} openblas_somatcopy_dispatch_t; +extern const openblas_somatcopy_dispatch_t *const openblas_somatcopy_dispatch[OPENBLAS_NUM_CORES]; #endif -#if (BUILD_DOUBLE==1) +#if (BUILD_DOUBLE==1) +typedef struct { int (*domatcopy_k_cn) (BLASLONG, BLASLONG, double, double*, BLASLONG, double*, BLASLONG); int (*domatcopy_k_ct) (BLASLONG, BLASLONG, double, double*, BLASLONG, double*, BLASLONG); int (*domatcopy_k_rn) (BLASLONG, BLASLONG, double, double*, BLASLONG, double*, BLASLONG); int (*domatcopy_k_rt) (BLASLONG, BLASLONG, double, double*, BLASLONG, double*, BLASLONG); +} openblas_domatcopy_dispatch_t; +extern const openblas_domatcopy_dispatch_t *const openblas_domatcopy_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_COMPLEX==1) +typedef struct { int (*comatcopy_k_cn) (BLASLONG, BLASLONG, float, float, float*, BLASLONG, float*, BLASLONG); int (*comatcopy_k_ct) (BLASLONG, BLASLONG, float, float, float*, BLASLONG, float*, BLASLONG); int (*comatcopy_k_rn) (BLASLONG, BLASLONG, float, float, float*, BLASLONG, float*, BLASLONG); int (*comatcopy_k_rt) (BLASLONG, BLASLONG, float, float, float*, BLASLONG, float*, BLASLONG); - int (*comatcopy_k_cnc) (BLASLONG, BLASLONG, float, float, float*, BLASLONG, float*, BLASLONG); int (*comatcopy_k_ctc) (BLASLONG, BLASLONG, float, float, float*, BLASLONG, float*, BLASLONG); int (*comatcopy_k_rnc) (BLASLONG, BLASLONG, float, float, float*, BLASLONG, float*, BLASLONG); int (*comatcopy_k_rtc) (BLASLONG, BLASLONG, float, float, float*, BLASLONG, float*, BLASLONG); +} openblas_comatcopy_dispatch_t; +extern const openblas_comatcopy_dispatch_t *const openblas_comatcopy_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_COMPLEX16==1) +typedef struct { int (*zomatcopy_k_cn) (BLASLONG, BLASLONG, double, double, double*, BLASLONG, double*, BLASLONG); int (*zomatcopy_k_ct) (BLASLONG, BLASLONG, double, double, double*, BLASLONG, double*, BLASLONG); int (*zomatcopy_k_rn) (BLASLONG, BLASLONG, double, double, double*, BLASLONG, double*, BLASLONG); int (*zomatcopy_k_rt) (BLASLONG, BLASLONG, double, double, double*, BLASLONG, double*, BLASLONG); - int (*zomatcopy_k_cnc) (BLASLONG, BLASLONG, double, double, double*, BLASLONG, double*, BLASLONG); int (*zomatcopy_k_ctc) (BLASLONG, BLASLONG, double, double, double*, BLASLONG, double*, BLASLONG); int (*zomatcopy_k_rnc) (BLASLONG, BLASLONG, double, double, double*, BLASLONG, double*, BLASLONG); int (*zomatcopy_k_rtc) (BLASLONG, BLASLONG, double, double, double*, BLASLONG, double*, BLASLONG); +} openblas_zomatcopy_dispatch_t; +extern const openblas_zomatcopy_dispatch_t *const openblas_zomatcopy_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_SINGLE==1) +typedef struct { int (*simatcopy_k_cn) (BLASLONG, BLASLONG, float, float*, BLASLONG); int (*simatcopy_k_ct) (BLASLONG, BLASLONG, float, float*, BLASLONG); int (*simatcopy_k_rn) (BLASLONG, BLASLONG, float, float*, BLASLONG); int (*simatcopy_k_rt) (BLASLONG, BLASLONG, float, float*, BLASLONG); +} openblas_simatcopy_dispatch_t; +extern const openblas_simatcopy_dispatch_t *const openblas_simatcopy_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_DOUBLE==1) +typedef struct { int (*dimatcopy_k_cn) (BLASLONG, BLASLONG, double, double*, BLASLONG); int (*dimatcopy_k_ct) (BLASLONG, BLASLONG, double, double*, BLASLONG); int (*dimatcopy_k_rn) (BLASLONG, BLASLONG, double, double*, BLASLONG); int (*dimatcopy_k_rt) (BLASLONG, BLASLONG, double, double*, BLASLONG); +} openblas_dimatcopy_dispatch_t; +extern const openblas_dimatcopy_dispatch_t *const openblas_dimatcopy_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_COMPLEX==1) +typedef struct { int (*cimatcopy_k_cn) (BLASLONG, BLASLONG, float, float, float*, BLASLONG); int (*cimatcopy_k_ct) (BLASLONG, BLASLONG, float, float, float*, BLASLONG); int (*cimatcopy_k_rn) (BLASLONG, BLASLONG, float, float, float*, BLASLONG); int (*cimatcopy_k_rt) (BLASLONG, BLASLONG, float, float, float*, BLASLONG); - int (*cimatcopy_k_cnc) (BLASLONG, BLASLONG, float, float, float*, BLASLONG); int (*cimatcopy_k_ctc) (BLASLONG, BLASLONG, float, float, float*, BLASLONG); int (*cimatcopy_k_rnc) (BLASLONG, BLASLONG, float, float, float*, BLASLONG); int (*cimatcopy_k_rtc) (BLASLONG, BLASLONG, float, float, float*, BLASLONG); +} openblas_cimatcopy_dispatch_t; +extern const openblas_cimatcopy_dispatch_t *const openblas_cimatcopy_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_COMPLEX16==1) +typedef struct { int (*zimatcopy_k_cn) (BLASLONG, BLASLONG, double, double, double*, BLASLONG); int (*zimatcopy_k_ct) (BLASLONG, BLASLONG, double, double, double*, BLASLONG); int (*zimatcopy_k_rn) (BLASLONG, BLASLONG, double, double, double*, BLASLONG); int (*zimatcopy_k_rt) (BLASLONG, BLASLONG, double, double, double*, BLASLONG); - int (*zimatcopy_k_cnc) (BLASLONG, BLASLONG, double, double, double*, BLASLONG); int (*zimatcopy_k_ctc) (BLASLONG, BLASLONG, double, double, double*, BLASLONG); int (*zimatcopy_k_rnc) (BLASLONG, BLASLONG, double, double, double*, BLASLONG); int (*zimatcopy_k_rtc) (BLASLONG, BLASLONG, double, double, double*, BLASLONG); +} openblas_zimatcopy_dispatch_t; +extern const openblas_zimatcopy_dispatch_t *const openblas_zimatcopy_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_SINGLE==1) +typedef struct { int (*sgeadd_k) (BLASLONG, BLASLONG, float, float *, BLASLONG, float, float *, BLASLONG, BLASLONG, BLASLONG); +} openblas_sgeadd_dispatch_t; +extern const openblas_sgeadd_dispatch_t *const openblas_sgeadd_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_DOUBLE==1) +typedef struct { int (*dgeadd_k) (BLASLONG, BLASLONG, double, double *, BLASLONG, double, double *, BLASLONG, BLASLONG, BLASLONG); +} openblas_dgeadd_dispatch_t; +extern const openblas_dgeadd_dispatch_t *const openblas_dgeadd_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_COMPLEX==1) +typedef struct { int (*cgeadd_k) (BLASLONG, BLASLONG, float, float, float *, BLASLONG, float, float, float *, BLASLONG, BLASLONG, BLASLONG); +} openblas_cgeadd_dispatch_t; +extern const openblas_cgeadd_dispatch_t *const openblas_cgeadd_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_COMPLEX16==1) +typedef struct { int (*zgeadd_k) (BLASLONG, BLASLONG, double, double, double *, BLASLONG, double, double, double *, BLASLONG, BLASLONG, BLASLONG); +} openblas_zgeadd_dispatch_t; +extern const openblas_zgeadd_dispatch_t *const openblas_zgeadd_dispatch[OPENBLAS_NUM_CORES]; #endif -} gotoblas_t; - -extern gotoblas_t *gotoblas; - -#define FUNC_OFFSET(func) (size_t)(&((gotoblas_t *)NULL)->func) #define DTB_ENTRIES gotoblas -> dtb_entries #define GEMM_OFFSET_A gotoblas -> offsetA @@ -1460,7 +2626,7 @@ extern gotoblas_t *gotoblas; #else -#define FUNC_OFFSET(func) (size_t)(func) +#define OPENBLAS_DISPATCH_OFFSET(group, func) (size_t)(func) #define DTB_ENTRIES DTB_DEFAULT_ENTRIES diff --git a/common_q.h b/common_q.h index 52da71c88c..9607fdebfa 100644 --- a/common_q.h +++ b/common_q.h @@ -147,116 +147,116 @@ #else -#define QAMAX_K gotoblas -> qamax_k -#define QAMIN_K gotoblas -> qamin_k -#define QMAX_K gotoblas -> qmax_k -#define QMIN_K gotoblas -> qmin_k -#define IQAMAX_K gotoblas -> iqamax_k -#define IQAMIN_K gotoblas -> iqamin_k -#define IQMAX_K gotoblas -> iqmax_k -#define IQMIN_K gotoblas -> iqmin_k -#define QASUM_K gotoblas -> qasum_k -#define QAXPYU_K gotoblas -> qaxpy_k -#define QAXPYC_K gotoblas -> qaxpy_k -#define QCOPY_K gotoblas -> qcopy_k -#define QDOTU_K gotoblas -> qdot_k -#define QDOTC_K gotoblas -> qdot_k -#define QNRM2_K gotoblas -> qnrm2_k -#define QSCAL_K gotoblas -> qscal_k -#define QSUM_K gotoblas -> qsum_k -#define QSWAP_K gotoblas -> qswap_k -#define QROT_K gotoblas -> qrot_k -#define QROTM_K gotoblas -> qrotm_k - -#define QGEMV_N gotoblas -> qgemv_n -#define QGEMV_T gotoblas -> qgemv_t -#define QGEMV_R gotoblas -> qgemv_n -#define QGEMV_C gotoblas -> qgemv_t -#define QGEMV_O gotoblas -> qgemv_n -#define QGEMV_U gotoblas -> qgemv_t -#define QGEMV_S gotoblas -> qgemv_n -#define QGEMV_D gotoblas -> qgemv_t - -#define QGERU_K gotoblas -> qger_k -#define QGERC_K gotoblas -> qger_k -#define QGERV_K gotoblas -> qger_k -#define QGERD_K gotoblas -> qger_k - -#define QSYMV_U gotoblas -> qsymv_U -#define QSYMV_L gotoblas -> qsymv_L +#define QAMAX_K OPENBLAS_DISPATCH(qamax) -> qamax_k +#define QAMIN_K OPENBLAS_DISPATCH(qamin) -> qamin_k +#define QMAX_K OPENBLAS_DISPATCH(qmax) -> qmax_k +#define QMIN_K OPENBLAS_DISPATCH(qmin) -> qmin_k +#define IQAMAX_K OPENBLAS_DISPATCH(iqamax) -> iqamax_k +#define IQAMIN_K OPENBLAS_DISPATCH(iqamin) -> iqamin_k +#define IQMAX_K OPENBLAS_DISPATCH(iqmax) -> iqmax_k +#define IQMIN_K OPENBLAS_DISPATCH(iqmin) -> iqmin_k +#define QASUM_K OPENBLAS_DISPATCH(qasum) -> qasum_k +#define QAXPYU_K OPENBLAS_DISPATCH(qaxpy) -> qaxpy_k +#define QAXPYC_K OPENBLAS_DISPATCH(qaxpy) -> qaxpy_k +#define QCOPY_K OPENBLAS_DISPATCH(qcopy) -> qcopy_k +#define QDOTU_K OPENBLAS_DISPATCH(qdot) -> qdot_k +#define QDOTC_K OPENBLAS_DISPATCH(qdot) -> qdot_k +#define QNRM2_K OPENBLAS_DISPATCH(qnrm2) -> qnrm2_k +#define QSCAL_K OPENBLAS_DISPATCH(qscal) -> qscal_k +#define QSUM_K OPENBLAS_DISPATCH(qsum) -> qsum_k +#define QSWAP_K OPENBLAS_DISPATCH(qswap) -> qswap_k +#define QROT_K OPENBLAS_DISPATCH(qrot) -> qrot_k +#define QROTM_K OPENBLAS_DISPATCH(qrotm) -> qrotm_k + +#define QGEMV_N OPENBLAS_DISPATCH(qgemv) -> qgemv_n +#define QGEMV_T OPENBLAS_DISPATCH(qgemv) -> qgemv_t +#define QGEMV_R OPENBLAS_DISPATCH(qgemv) -> qgemv_n +#define QGEMV_C OPENBLAS_DISPATCH(qgemv) -> qgemv_t +#define QGEMV_O OPENBLAS_DISPATCH(qgemv) -> qgemv_n +#define QGEMV_U OPENBLAS_DISPATCH(qgemv) -> qgemv_t +#define QGEMV_S OPENBLAS_DISPATCH(qgemv) -> qgemv_n +#define QGEMV_D OPENBLAS_DISPATCH(qgemv) -> qgemv_t + +#define QGERU_K OPENBLAS_DISPATCH(qger) -> qger_k +#define QGERC_K OPENBLAS_DISPATCH(qger) -> qger_k +#define QGERV_K OPENBLAS_DISPATCH(qger) -> qger_k +#define QGERD_K OPENBLAS_DISPATCH(qger) -> qger_k + +#define QSYMV_U OPENBLAS_DISPATCH(qsymv) -> qsymv_U +#define QSYMV_L OPENBLAS_DISPATCH(qsymv) -> qsymv_L #define QSYMV_THREAD_U qsymv_thread_U #define QSYMV_THREAD_L qsymv_thread_L -#define QGEMM_ONCOPY gotoblas -> qgemm_oncopy -#define QGEMM_OTCOPY gotoblas -> qgemm_otcopy -#define QGEMM_INCOPY gotoblas -> qgemm_incopy -#define QGEMM_ITCOPY gotoblas -> qgemm_itcopy - -#define QTRMM_OUNUCOPY gotoblas -> qtrmm_ounucopy -#define QTRMM_OUTUCOPY gotoblas -> qtrmm_outucopy -#define QTRMM_OLNUCOPY gotoblas -> qtrmm_olnucopy -#define QTRMM_OLTUCOPY gotoblas -> qtrmm_oltucopy -#define QTRSM_OUNUCOPY gotoblas -> qtrsm_ounucopy -#define QTRSM_OUTUCOPY gotoblas -> qtrsm_outucopy -#define QTRSM_OLNUCOPY gotoblas -> qtrsm_olnucopy -#define QTRSM_OLTUCOPY gotoblas -> qtrsm_oltucopy - -#define QTRMM_IUNUCOPY gotoblas -> qtrmm_iunucopy -#define QTRMM_IUTUCOPY gotoblas -> qtrmm_iutucopy -#define QTRMM_ILNUCOPY gotoblas -> qtrmm_ilnucopy -#define QTRMM_ILTUCOPY gotoblas -> qtrmm_iltucopy -#define QTRSM_IUNUCOPY gotoblas -> qtrsm_iunucopy -#define QTRSM_IUTUCOPY gotoblas -> qtrsm_iutucopy -#define QTRSM_ILNUCOPY gotoblas -> qtrsm_ilnucopy -#define QTRSM_ILTUCOPY gotoblas -> qtrsm_iltucopy - -#define QTRMM_OUNNCOPY gotoblas -> qtrmm_ounncopy -#define QTRMM_OUTNCOPY gotoblas -> qtrmm_outncopy -#define QTRMM_OLNNCOPY gotoblas -> qtrmm_olnncopy -#define QTRMM_OLTNCOPY gotoblas -> qtrmm_oltncopy -#define QTRSM_OUNNCOPY gotoblas -> qtrsm_ounncopy -#define QTRSM_OUTNCOPY gotoblas -> qtrsm_outncopy -#define QTRSM_OLNNCOPY gotoblas -> qtrsm_olnncopy -#define QTRSM_OLTNCOPY gotoblas -> qtrsm_oltncopy - -#define QTRMM_IUNNCOPY gotoblas -> qtrmm_iunncopy -#define QTRMM_IUTNCOPY gotoblas -> qtrmm_iutncopy -#define QTRMM_ILNNCOPY gotoblas -> qtrmm_ilnncopy -#define QTRMM_ILTNCOPY gotoblas -> qtrmm_iltncopy -#define QTRSM_IUNNCOPY gotoblas -> qtrsm_iunncopy -#define QTRSM_IUTNCOPY gotoblas -> qtrsm_iutncopy -#define QTRSM_ILNNCOPY gotoblas -> qtrsm_ilnncopy -#define QTRSM_ILTNCOPY gotoblas -> qtrsm_iltncopy - -#define QGEMM_BETA gotoblas -> qgemm_beta -#define QGEMM_KERNEL gotoblas -> qgemm_kernel - -#define QTRMM_KERNEL_LN gotoblas -> qtrmm_kernel_LN -#define QTRMM_KERNEL_LT gotoblas -> qtrmm_kernel_LT -#define QTRMM_KERNEL_LR gotoblas -> qtrmm_kernel_LN -#define QTRMM_KERNEL_LC gotoblas -> qtrmm_kernel_LT -#define QTRMM_KERNEL_RN gotoblas -> qtrmm_kernel_RN -#define QTRMM_KERNEL_RT gotoblas -> qtrmm_kernel_RT -#define QTRMM_KERNEL_RR gotoblas -> qtrmm_kernel_RN -#define QTRMM_KERNEL_RC gotoblas -> qtrmm_kernel_RT - -#define QTRSM_KERNEL_LN gotoblas -> qtrsm_kernel_LN -#define QTRSM_KERNEL_LT gotoblas -> qtrsm_kernel_LT -#define QTRSM_KERNEL_LR gotoblas -> qtrsm_kernel_LN -#define QTRSM_KERNEL_LC gotoblas -> qtrsm_kernel_LT -#define QTRSM_KERNEL_RN gotoblas -> qtrsm_kernel_RN -#define QTRSM_KERNEL_RT gotoblas -> qtrsm_kernel_RT -#define QTRSM_KERNEL_RR gotoblas -> qtrsm_kernel_RN -#define QTRSM_KERNEL_RC gotoblas -> qtrsm_kernel_RT - -#define QSYMM_IUTCOPY gotoblas -> qsymm_iutcopy -#define QSYMM_ILTCOPY gotoblas -> qsymm_iltcopy -#define QSYMM_OUTCOPY gotoblas -> qsymm_outcopy -#define QSYMM_OLTCOPY gotoblas -> qsymm_oltcopy - -#define QNEG_TCOPY gotoblas -> qneg_tcopy -#define QLASWP_NCOPY gotoblas -> qlaswp_ncopy +#define QGEMM_ONCOPY OPENBLAS_DISPATCH(qgemm) -> qgemm_oncopy +#define QGEMM_OTCOPY OPENBLAS_DISPATCH(qgemm) -> qgemm_otcopy +#define QGEMM_INCOPY OPENBLAS_DISPATCH(qgemm) -> qgemm_incopy +#define QGEMM_ITCOPY OPENBLAS_DISPATCH(qgemm) -> qgemm_itcopy + +#define QTRMM_OUNUCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_ounucopy +#define QTRMM_OUTUCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_outucopy +#define QTRMM_OLNUCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_olnucopy +#define QTRMM_OLTUCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_oltucopy +#define QTRSM_OUNUCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_ounucopy +#define QTRSM_OUTUCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_outucopy +#define QTRSM_OLNUCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_olnucopy +#define QTRSM_OLTUCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_oltucopy + +#define QTRMM_IUNUCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_iunucopy +#define QTRMM_IUTUCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_iutucopy +#define QTRMM_ILNUCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_ilnucopy +#define QTRMM_ILTUCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_iltucopy +#define QTRSM_IUNUCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_iunucopy +#define QTRSM_IUTUCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_iutucopy +#define QTRSM_ILNUCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_ilnucopy +#define QTRSM_ILTUCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_iltucopy + +#define QTRMM_OUNNCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_ounncopy +#define QTRMM_OUTNCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_outncopy +#define QTRMM_OLNNCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_olnncopy +#define QTRMM_OLTNCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_oltncopy +#define QTRSM_OUNNCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_ounncopy +#define QTRSM_OUTNCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_outncopy +#define QTRSM_OLNNCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_olnncopy +#define QTRSM_OLTNCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_oltncopy + +#define QTRMM_IUNNCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_iunncopy +#define QTRMM_IUTNCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_iutncopy +#define QTRMM_ILNNCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_ilnncopy +#define QTRMM_ILTNCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_iltncopy +#define QTRSM_IUNNCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_iunncopy +#define QTRSM_IUTNCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_iutncopy +#define QTRSM_ILNNCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_ilnncopy +#define QTRSM_ILTNCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_iltncopy + +#define QGEMM_BETA OPENBLAS_DISPATCH(qgemm) -> qgemm_beta +#define QGEMM_KERNEL OPENBLAS_DISPATCH(qgemm) -> qgemm_kernel + +#define QTRMM_KERNEL_LN OPENBLAS_DISPATCH(qtrmm) -> qtrmm_kernel_LN +#define QTRMM_KERNEL_LT OPENBLAS_DISPATCH(qtrmm) -> qtrmm_kernel_LT +#define QTRMM_KERNEL_LR OPENBLAS_DISPATCH(qtrmm) -> qtrmm_kernel_LN +#define QTRMM_KERNEL_LC OPENBLAS_DISPATCH(qtrmm) -> qtrmm_kernel_LT +#define QTRMM_KERNEL_RN OPENBLAS_DISPATCH(qtrmm) -> qtrmm_kernel_RN +#define QTRMM_KERNEL_RT OPENBLAS_DISPATCH(qtrmm) -> qtrmm_kernel_RT +#define QTRMM_KERNEL_RR OPENBLAS_DISPATCH(qtrmm) -> qtrmm_kernel_RN +#define QTRMM_KERNEL_RC OPENBLAS_DISPATCH(qtrmm) -> qtrmm_kernel_RT + +#define QTRSM_KERNEL_LN OPENBLAS_DISPATCH(qtrsm) -> qtrsm_kernel_LN +#define QTRSM_KERNEL_LT OPENBLAS_DISPATCH(qtrsm) -> qtrsm_kernel_LT +#define QTRSM_KERNEL_LR OPENBLAS_DISPATCH(qtrsm) -> qtrsm_kernel_LN +#define QTRSM_KERNEL_LC OPENBLAS_DISPATCH(qtrsm) -> qtrsm_kernel_LT +#define QTRSM_KERNEL_RN OPENBLAS_DISPATCH(qtrsm) -> qtrsm_kernel_RN +#define QTRSM_KERNEL_RT OPENBLAS_DISPATCH(qtrsm) -> qtrsm_kernel_RT +#define QTRSM_KERNEL_RR OPENBLAS_DISPATCH(qtrsm) -> qtrsm_kernel_RN +#define QTRSM_KERNEL_RC OPENBLAS_DISPATCH(qtrsm) -> qtrsm_kernel_RT + +#define QSYMM_IUTCOPY OPENBLAS_DISPATCH(qsymm) -> qsymm_iutcopy +#define QSYMM_ILTCOPY OPENBLAS_DISPATCH(qsymm) -> qsymm_iltcopy +#define QSYMM_OUTCOPY OPENBLAS_DISPATCH(qsymm) -> qsymm_outcopy +#define QSYMM_OLTCOPY OPENBLAS_DISPATCH(qsymm) -> qsymm_oltcopy + +#define QNEG_TCOPY OPENBLAS_DISPATCH(qneg) -> qneg_tcopy +#define QLASWP_NCOPY OPENBLAS_DISPATCH(qlaswp) -> qlaswp_ncopy #endif diff --git a/common_s.h b/common_s.h index 54b59595f3..abf8c7e738 100644 --- a/common_s.h +++ b/common_s.h @@ -231,176 +231,177 @@ #else -#define SAMAX_K gotoblas -> samax_k -#define SAMIN_K gotoblas -> samin_k -#define SMAX_K gotoblas -> smax_k -#define SMIN_K gotoblas -> smin_k -#define ISAMAX_K gotoblas -> isamax_k -#define ISAMIN_K gotoblas -> isamin_k -#define ISMAX_K gotoblas -> ismax_k -#define ISMIN_K gotoblas -> ismin_k -#define SASUM_K gotoblas -> sasum_k -#define SSUM_K gotoblas -> ssum_k -#define SAXPYU_K gotoblas -> saxpy_k -#define SAXPYC_K gotoblas -> saxpy_k -#define SCOPY_K gotoblas -> scopy_k -#define SDOTU_K gotoblas -> sdot_k -#define SDOTC_K gotoblas -> sdot_k -#define SDSDOT_K gotoblas -> dsdot_k -#define DSDOT_K gotoblas -> dsdot_k -#define SNRM2_K gotoblas -> snrm2_k -#define SSCAL_K gotoblas -> sscal_k -#define SSWAP_K gotoblas -> sswap_k -#define SROT_K gotoblas -> srot_k -#define SROTM_K gotoblas -> srotm_k - -#define SGEMV_N gotoblas -> sgemv_n -#define SGEMV_T gotoblas -> sgemv_t -#define SGEMV_R gotoblas -> sgemv_n -#define SGEMV_C gotoblas -> sgemv_t -#define SGEMV_O gotoblas -> sgemv_n -#define SGEMV_U gotoblas -> sgemv_t -#define SGEMV_S gotoblas -> sgemv_n -#define SGEMV_D gotoblas -> sgemv_t - -#define SGERU_K gotoblas -> sger_k -#define SGERC_K gotoblas -> sger_k -#define SGERV_K gotoblas -> sger_k -#define SGERD_K gotoblas -> sger_k - -#define SSYMV_U gotoblas -> ssymv_U -#define SSYMV_L gotoblas -> ssymv_L +#define SAMAX_K OPENBLAS_DISPATCH(samax) -> samax_k +#define SAMIN_K OPENBLAS_DISPATCH(samin) -> samin_k +#define SMAX_K OPENBLAS_DISPATCH(smax) -> smax_k +#define SMIN_K OPENBLAS_DISPATCH(smin) -> smin_k +#define ISAMAX_K OPENBLAS_DISPATCH(isamax) -> isamax_k +#define ISAMIN_K OPENBLAS_DISPATCH(isamin) -> isamin_k +#define ISMAX_K OPENBLAS_DISPATCH(ismax) -> ismax_k +#define ISMIN_K OPENBLAS_DISPATCH(ismin) -> ismin_k +#define SASUM_K OPENBLAS_DISPATCH(sasum) -> sasum_k +#define SSUM_K OPENBLAS_DISPATCH(ssum) -> ssum_k +#define SAXPYU_K OPENBLAS_DISPATCH(saxpy) -> saxpy_k +#define SAXPYC_K OPENBLAS_DISPATCH(saxpy) -> saxpy_k +#define SCOPY_K OPENBLAS_DISPATCH(scopy) -> scopy_k +#define SDOTU_K OPENBLAS_DISPATCH(sdot) -> sdot_k +#define SDOTC_K OPENBLAS_DISPATCH(sdot) -> sdot_k +#define SDSDOT_K OPENBLAS_DISPATCH(dsdot) -> dsdot_k +#define DSDOT_K OPENBLAS_DISPATCH(dsdot) -> dsdot_k +#define SNRM2_K OPENBLAS_DISPATCH(snrm2) -> snrm2_k +#define SSCAL_K OPENBLAS_DISPATCH(sscal) -> sscal_k +#define SSWAP_K OPENBLAS_DISPATCH(sswap) -> sswap_k +#define SROT_K OPENBLAS_DISPATCH(srot) -> srot_k +#define SROTM_K OPENBLAS_DISPATCH(srotm) -> srotm_k + +#define SGEMV_N OPENBLAS_DISPATCH(sgemv) -> sgemv_n +#define SGEMV_T OPENBLAS_DISPATCH(sgemv) -> sgemv_t +#define SGEMV_R OPENBLAS_DISPATCH(sgemv) -> sgemv_n +#define SGEMV_C OPENBLAS_DISPATCH(sgemv) -> sgemv_t +#define SGEMV_O OPENBLAS_DISPATCH(sgemv) -> sgemv_n +#define SGEMV_U OPENBLAS_DISPATCH(sgemv) -> sgemv_t +#define SGEMV_S OPENBLAS_DISPATCH(sgemv) -> sgemv_n +#define SGEMV_D OPENBLAS_DISPATCH(sgemv) -> sgemv_t + +#define SGERU_K OPENBLAS_DISPATCH(sger) -> sger_k +#define SGERC_K OPENBLAS_DISPATCH(sger) -> sger_k +#define SGERV_K OPENBLAS_DISPATCH(sger) -> sger_k +#define SGERD_K OPENBLAS_DISPATCH(sger) -> sger_k + +#define SSYMV_U OPENBLAS_DISPATCH(ssymv) -> ssymv_U +#define SSYMV_L OPENBLAS_DISPATCH(ssymv) -> ssymv_L #define SSYMV_THREAD_U ssymv_thread_U #define SSYMV_THREAD_L ssymv_thread_L #ifdef ARCH_X86_64 -#define SGEMM_DIRECT_PERFORMANT gotoblas -> sgemm_direct_performant -#define SGEMM_DIRECT gotoblas -> sgemm_direct +#define SGEMM_DIRECT_PERFORMANT OPENBLAS_DISPATCH(sgemm) -> sgemm_direct_performant +#define SGEMM_DIRECT OPENBLAS_DISPATCH(sgemm) -> sgemm_direct #elif ARCH_ARM64 -#define SGEMM_DIRECT_PERFORMANT gotoblas -> sgemm_direct_performant -#define SGEMM_DIRECT gotoblas -> sgemm_direct -#define SGEMM_DIRECT_ALPHA_BETA gotoblas -> sgemm_direct_alpha_beta -#define SSYMM_DIRECT_ALPHA_BETA_LU gotoblas -> ssymm_direct_alpha_betaLU -#define SSYMM_DIRECT_ALPHA_BETA_LL gotoblas -> ssymm_direct_alpha_betaLL -#define STRMM_DIRECT_LNUN gotoblas -> strmm_direct_LNUN -#define STRMM_DIRECT_LNLN gotoblas -> strmm_direct_LNLN -#define STRMM_DIRECT_LTUN gotoblas -> strmm_direct_LTUN -#define STRMM_DIRECT_LTLN gotoblas -> strmm_direct_LTLN -#define SSYRK_DIRECT_ALPHA_BETA_UN gotoblas -> ssyrk_direct_alpha_betaUN -#define SSYRK_DIRECT_ALPHA_BETA_UT gotoblas -> ssyrk_direct_alpha_betaUT -#define SSYRK_DIRECT_ALPHA_BETA_LN gotoblas -> ssyrk_direct_alpha_betaLN -#define SSYRK_DIRECT_ALPHA_BETA_LT gotoblas -> ssyrk_direct_alpha_betaLT -#define SSYR2K_DIRECT_ALPHA_BETA_UN gotoblas -> ssyr2k_direct_alpha_betaUN -#define SSYR2K_DIRECT_ALPHA_BETA_UT gotoblas -> ssyr2k_direct_alpha_betaUT -#define SSYR2K_DIRECT_ALPHA_BETA_LN gotoblas -> ssyr2k_direct_alpha_betaLN -#define SSYR2K_DIRECT_ALPHA_BETA_LT gotoblas -> ssyr2k_direct_alpha_betaLT -#define SME_SGEMM_KERNEL gotoblas -> sme_sgemm_kernel +#define SGEMM_DIRECT_PERFORMANT OPENBLAS_DISPATCH(sgemm) -> sgemm_direct_performant +#define SGEMM_DIRECT OPENBLAS_DISPATCH(sgemm) -> sgemm_direct +#define SGEMM_DIRECT_ALPHA_BETA OPENBLAS_DISPATCH(sgemm) -> sgemm_direct_alpha_beta +#define SSYMM_DIRECT_ALPHA_BETA_LU OPENBLAS_DISPATCH(ssymm) -> ssymm_direct_alpha_betaLU +#define SSYMM_DIRECT_ALPHA_BETA_LL OPENBLAS_DISPATCH(ssymm) -> ssymm_direct_alpha_betaLL +#define STRMM_DIRECT_LNUN OPENBLAS_DISPATCH(strmm) -> strmm_direct_LNUN +#define STRMM_DIRECT_LNLN OPENBLAS_DISPATCH(strmm) -> strmm_direct_LNLN +#define STRMM_DIRECT_LTUN OPENBLAS_DISPATCH(strmm) -> strmm_direct_LTUN +#define STRMM_DIRECT_LTLN OPENBLAS_DISPATCH(strmm) -> strmm_direct_LTLN +#define SSYRK_DIRECT_ALPHA_BETA_UN OPENBLAS_DISPATCH(ssyrk) -> ssyrk_direct_alpha_betaUN +#define SSYRK_DIRECT_ALPHA_BETA_UT OPENBLAS_DISPATCH(ssyrk) -> ssyrk_direct_alpha_betaUT +#define SSYRK_DIRECT_ALPHA_BETA_LN OPENBLAS_DISPATCH(ssyrk) -> ssyrk_direct_alpha_betaLN +#define SSYRK_DIRECT_ALPHA_BETA_LT OPENBLAS_DISPATCH(ssyrk) -> ssyrk_direct_alpha_betaLT +#define SSYR2K_DIRECT_ALPHA_BETA_UN OPENBLAS_DISPATCH(ssyr2k) -> ssyr2k_direct_alpha_betaUN +#define SSYR2K_DIRECT_ALPHA_BETA_UT OPENBLAS_DISPATCH(ssyr2k) -> ssyr2k_direct_alpha_betaUT +#define SSYR2K_DIRECT_ALPHA_BETA_LN OPENBLAS_DISPATCH(ssyr2k) -> ssyr2k_direct_alpha_betaLN +#define SSYR2K_DIRECT_ALPHA_BETA_LT OPENBLAS_DISPATCH(ssyr2k) -> ssyr2k_direct_alpha_betaLT +#define SME_SGEMM_KERNEL OPENBLAS_DISPATCH(sgemm) -> sme_sgemm_kernel #endif -#define SGEMM_ONCOPY gotoblas -> sgemm_oncopy -#define SGEMM_OTCOPY gotoblas -> sgemm_otcopy -#define SGEMM_INCOPY gotoblas -> sgemm_incopy -#define SGEMM_ITCOPY gotoblas -> sgemm_itcopy - -#define STRMM_OUNUCOPY gotoblas -> strmm_ounucopy -#define STRMM_OUTUCOPY gotoblas -> strmm_outucopy -#define STRMM_OLNUCOPY gotoblas -> strmm_olnucopy -#define STRMM_OLTUCOPY gotoblas -> strmm_oltucopy -#define STRSM_OUNUCOPY gotoblas -> strsm_ounucopy -#define STRSM_OUTUCOPY gotoblas -> strsm_outucopy -#define STRSM_OLNUCOPY gotoblas -> strsm_olnucopy -#define STRSM_OLTUCOPY gotoblas -> strsm_oltucopy - -#define STRMM_IUNUCOPY gotoblas -> strmm_iunucopy -#define STRMM_IUTUCOPY gotoblas -> strmm_iutucopy -#define STRMM_ILNUCOPY gotoblas -> strmm_ilnucopy -#define STRMM_ILTUCOPY gotoblas -> strmm_iltucopy -#define STRSM_IUNUCOPY gotoblas -> strsm_iunucopy -#define STRSM_IUTUCOPY gotoblas -> strsm_iutucopy -#define STRSM_ILNUCOPY gotoblas -> strsm_ilnucopy -#define STRSM_ILTUCOPY gotoblas -> strsm_iltucopy - -#define STRMM_OUNNCOPY gotoblas -> strmm_ounncopy -#define STRMM_OUTNCOPY gotoblas -> strmm_outncopy -#define STRMM_OLNNCOPY gotoblas -> strmm_olnncopy -#define STRMM_OLTNCOPY gotoblas -> strmm_oltncopy -#define STRSM_OUNNCOPY gotoblas -> strsm_ounncopy -#define STRSM_OUTNCOPY gotoblas -> strsm_outncopy -#define STRSM_OLNNCOPY gotoblas -> strsm_olnncopy -#define STRSM_OLTNCOPY gotoblas -> strsm_oltncopy - -#define STRMM_IUNNCOPY gotoblas -> strmm_iunncopy -#define STRMM_IUTNCOPY gotoblas -> strmm_iutncopy -#define STRMM_ILNNCOPY gotoblas -> strmm_ilnncopy -#define STRMM_ILTNCOPY gotoblas -> strmm_iltncopy -#define SSYMM_INCOPY gotoblas -> ssymm_incopy -#define SSYMM_ITCOPY gotoblas -> ssymm_itcopy -#define STRMM_INCOPY gotoblas -> strmm_incopy -#define STRMM_ITCOPY gotoblas -> strmm_itcopy - -#define STRSM_IUNNCOPY gotoblas -> strsm_iunncopy -#define STRSM_IUTNCOPY gotoblas -> strsm_iutncopy -#define STRSM_ILNNCOPY gotoblas -> strsm_ilnncopy -#define STRSM_ILTNCOPY gotoblas -> strsm_iltncopy - -#define SGEMM_BETA gotoblas -> sgemm_beta -#define SGEMM_KERNEL gotoblas -> sgemm_kernel -#define SSYMM_KERNEL gotoblas -> ssymm_kernel -#define STRMM_GEMM_KERNEL gotoblas -> strmm_gemm_kernel - -#define STRMM_KERNEL_LN gotoblas -> strmm_kernel_LN -#define STRMM_KERNEL_LT gotoblas -> strmm_kernel_LT -#define STRMM_KERNEL_LR gotoblas -> strmm_kernel_LN -#define STRMM_KERNEL_LC gotoblas -> strmm_kernel_LT -#define STRMM_KERNEL_RN gotoblas -> strmm_kernel_RN -#define STRMM_KERNEL_RT gotoblas -> strmm_kernel_RT -#define STRMM_KERNEL_RR gotoblas -> strmm_kernel_RN -#define STRMM_KERNEL_RC gotoblas -> strmm_kernel_RT - -#define STRSM_KERNEL_LN gotoblas -> strsm_kernel_LN -#define STRSM_KERNEL_LT gotoblas -> strsm_kernel_LT -#define STRSM_KERNEL_LR gotoblas -> strsm_kernel_LN -#define STRSM_KERNEL_LC gotoblas -> strsm_kernel_LT -#define STRSM_KERNEL_RN gotoblas -> strsm_kernel_RN -#define STRSM_KERNEL_RT gotoblas -> strsm_kernel_RT -#define STRSM_KERNEL_RR gotoblas -> strsm_kernel_RN -#define STRSM_KERNEL_RC gotoblas -> strsm_kernel_RT - -#define SSYMM_IUTCOPY gotoblas -> ssymm_iutcopy -#define SSYMM_ILTCOPY gotoblas -> ssymm_iltcopy -#define SSYMM_OUTCOPY gotoblas -> ssymm_outcopy -#define SSYMM_OLTCOPY gotoblas -> ssymm_oltcopy - -#define SNEG_TCOPY gotoblas -> sneg_tcopy -#define SLASWP_NCOPY gotoblas -> slaswp_ncopy - -#define SAXPBY_K gotoblas -> saxpby_k - -#define SOMATCOPY_K_CN gotoblas -> somatcopy_k_cn -#define SOMATCOPY_K_RN gotoblas -> somatcopy_k_rn -#define SOMATCOPY_K_CT gotoblas -> somatcopy_k_ct -#define SOMATCOPY_K_RT gotoblas -> somatcopy_k_rt -#define SIMATCOPY_K_CN gotoblas -> simatcopy_k_cn -#define SIMATCOPY_K_RN gotoblas -> simatcopy_k_rn -#define SIMATCOPY_K_CT gotoblas -> simatcopy_k_ct -#define SIMATCOPY_K_RT gotoblas -> simatcopy_k_rt - -#define SGEADD_K gotoblas -> sgeadd_k - -#define SGEMM_SMALL_MATRIX_PERMIT gotoblas -> sgemm_small_matrix_permit +#define SGEMM_ONCOPY OPENBLAS_DISPATCH(sgemm) -> sgemm_oncopy +#define SGEMM_OTCOPY OPENBLAS_DISPATCH(sgemm) -> sgemm_otcopy +#define SGEMM_INCOPY OPENBLAS_DISPATCH(sgemm) -> sgemm_incopy +#define SGEMM_ITCOPY OPENBLAS_DISPATCH(sgemm) -> sgemm_itcopy + +#define STRMM_OUNUCOPY OPENBLAS_DISPATCH(strmm) -> strmm_ounucopy +#define STRMM_OUTUCOPY OPENBLAS_DISPATCH(strmm) -> strmm_outucopy +#define STRMM_OLNUCOPY OPENBLAS_DISPATCH(strmm) -> strmm_olnucopy +#define STRMM_OLTUCOPY OPENBLAS_DISPATCH(strmm) -> strmm_oltucopy +#define STRSM_OUNUCOPY OPENBLAS_DISPATCH(strsm) -> strsm_ounucopy +#define STRSM_OUTUCOPY OPENBLAS_DISPATCH(strsm) -> strsm_outucopy +#define STRSM_OLNUCOPY OPENBLAS_DISPATCH(strsm) -> strsm_olnucopy +#define STRSM_OLTUCOPY OPENBLAS_DISPATCH(strsm) -> strsm_oltucopy + +#define STRMM_IUNUCOPY OPENBLAS_DISPATCH(strmm) -> strmm_iunucopy +#define STRMM_IUTUCOPY OPENBLAS_DISPATCH(strmm) -> strmm_iutucopy +#define STRMM_ILNUCOPY OPENBLAS_DISPATCH(strmm) -> strmm_ilnucopy +#define STRMM_ILTUCOPY OPENBLAS_DISPATCH(strmm) -> strmm_iltucopy +#define STRSM_IUNUCOPY OPENBLAS_DISPATCH(strsm) -> strsm_iunucopy +#define STRSM_IUTUCOPY OPENBLAS_DISPATCH(strsm) -> strsm_iutucopy +#define STRSM_ILNUCOPY OPENBLAS_DISPATCH(strsm) -> strsm_ilnucopy +#define STRSM_ILTUCOPY OPENBLAS_DISPATCH(strsm) -> strsm_iltucopy + +#define STRMM_OUNNCOPY OPENBLAS_DISPATCH(strmm) -> strmm_ounncopy +#define STRMM_OUTNCOPY OPENBLAS_DISPATCH(strmm) -> strmm_outncopy +#define STRMM_OLNNCOPY OPENBLAS_DISPATCH(strmm) -> strmm_olnncopy +#define STRMM_OLTNCOPY OPENBLAS_DISPATCH(strmm) -> strmm_oltncopy +#define STRSM_OUNNCOPY OPENBLAS_DISPATCH(strsm) -> strsm_ounncopy +#define STRSM_OUTNCOPY OPENBLAS_DISPATCH(strsm) -> strsm_outncopy +#define STRSM_OLNNCOPY OPENBLAS_DISPATCH(strsm) -> strsm_olnncopy +#define STRSM_OLTNCOPY OPENBLAS_DISPATCH(strsm) -> strsm_oltncopy + +#define STRMM_IUNNCOPY OPENBLAS_DISPATCH(strmm) -> strmm_iunncopy +#define STRMM_IUTNCOPY OPENBLAS_DISPATCH(strmm) -> strmm_iutncopy +#define STRMM_ILNNCOPY OPENBLAS_DISPATCH(strmm) -> strmm_ilnncopy +#define STRMM_ILTNCOPY OPENBLAS_DISPATCH(strmm) -> strmm_iltncopy +#define SSYMM_INCOPY OPENBLAS_DISPATCH(ssymm) -> ssymm_incopy +#define SSYMM_ITCOPY OPENBLAS_DISPATCH(ssymm) -> ssymm_itcopy +#define STRMM_INCOPY OPENBLAS_DISPATCH(strmm) -> strmm_incopy +#define STRMM_ITCOPY OPENBLAS_DISPATCH(strmm) -> strmm_itcopy + +#define STRSM_IUNNCOPY OPENBLAS_DISPATCH(strsm) -> strsm_iunncopy +#define STRSM_IUTNCOPY OPENBLAS_DISPATCH(strsm) -> strsm_iutncopy +#define STRSM_ILNNCOPY OPENBLAS_DISPATCH(strsm) -> strsm_ilnncopy +#define STRSM_ILTNCOPY OPENBLAS_DISPATCH(strsm) -> strsm_iltncopy + +#define SGEMM_BETA OPENBLAS_DISPATCH(sgemm) -> sgemm_beta +#define SGEMM_KERNEL OPENBLAS_DISPATCH(sgemm) -> sgemm_kernel +#define SSYMM_KERNEL OPENBLAS_DISPATCH(ssymm) -> ssymm_kernel +#define STRMM_GEMM_KERNEL OPENBLAS_DISPATCH(strmm) -> strmm_gemm_kernel + +#define STRMM_KERNEL_LN OPENBLAS_DISPATCH(strmm) -> strmm_kernel_LN +#define STRMM_KERNEL_LT OPENBLAS_DISPATCH(strmm) -> strmm_kernel_LT +#define STRMM_KERNEL_LR OPENBLAS_DISPATCH(strmm) -> strmm_kernel_LN +#define STRMM_KERNEL_LC OPENBLAS_DISPATCH(strmm) -> strmm_kernel_LT +#define STRMM_KERNEL_RN OPENBLAS_DISPATCH(strmm) -> strmm_kernel_RN +#define STRMM_KERNEL_RT OPENBLAS_DISPATCH(strmm) -> strmm_kernel_RT +#define STRMM_KERNEL_RR OPENBLAS_DISPATCH(strmm) -> strmm_kernel_RN +#define STRMM_KERNEL_RC OPENBLAS_DISPATCH(strmm) -> strmm_kernel_RT + +#define STRSM_KERNEL_LN OPENBLAS_DISPATCH(strsm) -> strsm_kernel_LN +#define STRSM_KERNEL_LT OPENBLAS_DISPATCH(strsm) -> strsm_kernel_LT +#define STRSM_KERNEL_LR OPENBLAS_DISPATCH(strsm) -> strsm_kernel_LN +#define STRSM_KERNEL_LC OPENBLAS_DISPATCH(strsm) -> strsm_kernel_LT +#define STRSM_KERNEL_RN OPENBLAS_DISPATCH(strsm) -> strsm_kernel_RN +#define STRSM_KERNEL_RT OPENBLAS_DISPATCH(strsm) -> strsm_kernel_RT +#define STRSM_KERNEL_RR OPENBLAS_DISPATCH(strsm) -> strsm_kernel_RN +#define STRSM_KERNEL_RC OPENBLAS_DISPATCH(strsm) -> strsm_kernel_RT + +#define SSYMM_IUTCOPY OPENBLAS_DISPATCH(ssymm) -> ssymm_iutcopy +#define SSYMM_ILTCOPY OPENBLAS_DISPATCH(ssymm) -> ssymm_iltcopy +#define SSYMM_OUTCOPY OPENBLAS_DISPATCH(ssymm) -> ssymm_outcopy +#define SSYMM_OLTCOPY OPENBLAS_DISPATCH(ssymm) -> ssymm_oltcopy + +#define SNEG_TCOPY OPENBLAS_DISPATCH(sneg) -> sneg_tcopy +#define SLASWP_NCOPY OPENBLAS_DISPATCH(slaswp) -> slaswp_ncopy + +#define SAXPBY_K OPENBLAS_DISPATCH(saxpby) -> saxpby_k + +#define SOMATCOPY_K_CN OPENBLAS_DISPATCH(somatcopy) -> somatcopy_k_cn +#define SOMATCOPY_K_RN OPENBLAS_DISPATCH(somatcopy) -> somatcopy_k_rn +#define SOMATCOPY_K_CT OPENBLAS_DISPATCH(somatcopy) -> somatcopy_k_ct +#define SOMATCOPY_K_RT OPENBLAS_DISPATCH(somatcopy) -> somatcopy_k_rt +#define SIMATCOPY_K_CN OPENBLAS_DISPATCH(simatcopy) -> simatcopy_k_cn +#define SIMATCOPY_K_RN OPENBLAS_DISPATCH(simatcopy) -> simatcopy_k_rn +#define SIMATCOPY_K_CT OPENBLAS_DISPATCH(simatcopy) -> simatcopy_k_ct +#define SIMATCOPY_K_RT OPENBLAS_DISPATCH(simatcopy) -> simatcopy_k_rt + +#define SGEADD_K OPENBLAS_DISPATCH(sgeadd) -> sgeadd_k + +#define SGEMM_SMALL_MATRIX_PERMIT OPENBLAS_DISPATCH(sgemm) -> sgemm_small_matrix_permit #endif -#define SGEMM_SMALL_KERNEL_NN FUNC_OFFSET(sgemm_small_kernel_nn) -#define SGEMM_SMALL_KERNEL_NT FUNC_OFFSET(sgemm_small_kernel_nt) -#define SGEMM_SMALL_KERNEL_TN FUNC_OFFSET(sgemm_small_kernel_tn) -#define SGEMM_SMALL_KERNEL_TT FUNC_OFFSET(sgemm_small_kernel_tt) +#define SGEMM_SMALL_KERNEL_BASE OPENBLAS_DISPATCH_BASE(sgemm) +#define SGEMM_SMALL_KERNEL_NN OPENBLAS_DISPATCH_OFFSET(sgemm, sgemm_small_kernel_nn) +#define SGEMM_SMALL_KERNEL_NT OPENBLAS_DISPATCH_OFFSET(sgemm, sgemm_small_kernel_nt) +#define SGEMM_SMALL_KERNEL_TN OPENBLAS_DISPATCH_OFFSET(sgemm, sgemm_small_kernel_tn) +#define SGEMM_SMALL_KERNEL_TT OPENBLAS_DISPATCH_OFFSET(sgemm, sgemm_small_kernel_tt) -#define SGEMM_SMALL_KERNEL_B0_NN FUNC_OFFSET(sgemm_small_kernel_b0_nn) -#define SGEMM_SMALL_KERNEL_B0_NT FUNC_OFFSET(sgemm_small_kernel_b0_nt) -#define SGEMM_SMALL_KERNEL_B0_TN FUNC_OFFSET(sgemm_small_kernel_b0_tn) -#define SGEMM_SMALL_KERNEL_B0_TT FUNC_OFFSET(sgemm_small_kernel_b0_tt) +#define SGEMM_SMALL_KERNEL_B0_NN OPENBLAS_DISPATCH_OFFSET(sgemm, sgemm_small_kernel_b0_nn) +#define SGEMM_SMALL_KERNEL_B0_NT OPENBLAS_DISPATCH_OFFSET(sgemm, sgemm_small_kernel_b0_nt) +#define SGEMM_SMALL_KERNEL_B0_TN OPENBLAS_DISPATCH_OFFSET(sgemm, sgemm_small_kernel_b0_tn) +#define SGEMM_SMALL_KERNEL_B0_TT OPENBLAS_DISPATCH_OFFSET(sgemm, sgemm_small_kernel_b0_tt) #define SGEMM_NN sgemm_nn diff --git a/common_sb.h b/common_sb.h index d21e7a563b..9e51a2d39b 100644 --- a/common_sb.h +++ b/common_sb.h @@ -27,33 +27,34 @@ #define SBGEMM_SMALL_MATRIX_PERMIT sbgemm_small_matrix_permit #else -#define SBDOT_K gotoblas -> sbdot_k -#define SBSTOBF16_K gotoblas -> sbstobf16_k -#define SBDTOBF16_K gotoblas -> sbdtobf16_k -#define SBF16TOS_K gotoblas -> sbf16tos_k -#define DBF16TOD_K gotoblas -> dbf16tod_k -#define SBGEMV_N_K gotoblas -> sbgemv_n -#define SBGEMV_T_K gotoblas -> sbgemv_t +#define SBDOT_K OPENBLAS_DISPATCH(sbdot) -> sbdot_k +#define SBSTOBF16_K OPENBLAS_DISPATCH(sbstobf16) -> sbstobf16_k +#define SBDTOBF16_K OPENBLAS_DISPATCH(sbdtobf16) -> sbdtobf16_k +#define SBF16TOS_K OPENBLAS_DISPATCH(sbf16tos) -> sbf16tos_k +#define DBF16TOD_K OPENBLAS_DISPATCH(dbf16tod) -> dbf16tod_k +#define SBGEMV_N_K OPENBLAS_DISPATCH(sbgemv) -> sbgemv_n +#define SBGEMV_T_K OPENBLAS_DISPATCH(sbgemv) -> sbgemv_t -#define SBGEMM_ONCOPY gotoblas -> sbgemm_oncopy -#define SBGEMM_OTCOPY gotoblas -> sbgemm_otcopy -#define SBGEMM_INCOPY gotoblas -> sbgemm_incopy -#define SBGEMM_ITCOPY gotoblas -> sbgemm_itcopy -#define SBGEMM_BETA gotoblas -> sbgemm_beta -#define SBGEMM_KERNEL gotoblas -> sbgemm_kernel +#define SBGEMM_ONCOPY OPENBLAS_DISPATCH(sbgemm) -> sbgemm_oncopy +#define SBGEMM_OTCOPY OPENBLAS_DISPATCH(sbgemm) -> sbgemm_otcopy +#define SBGEMM_INCOPY OPENBLAS_DISPATCH(sbgemm) -> sbgemm_incopy +#define SBGEMM_ITCOPY OPENBLAS_DISPATCH(sbgemm) -> sbgemm_itcopy +#define SBGEMM_BETA OPENBLAS_DISPATCH(sbgemm) -> sbgemm_beta +#define SBGEMM_KERNEL OPENBLAS_DISPATCH(sbgemm) -> sbgemm_kernel -#define SBGEMM_SMALL_MATRIX_PERMIT gotoblas -> sbgemm_small_matrix_permit +#define SBGEMM_SMALL_MATRIX_PERMIT OPENBLAS_DISPATCH(sbgemm) -> sbgemm_small_matrix_permit #endif -#define SBGEMM_SMALL_KERNEL_NN FUNC_OFFSET(sbgemm_small_kernel_nn) -#define SBGEMM_SMALL_KERNEL_NT FUNC_OFFSET(sbgemm_small_kernel_nt) -#define SBGEMM_SMALL_KERNEL_TN FUNC_OFFSET(sbgemm_small_kernel_tn) -#define SBGEMM_SMALL_KERNEL_TT FUNC_OFFSET(sbgemm_small_kernel_tt) +#define SBGEMM_SMALL_KERNEL_BASE OPENBLAS_DISPATCH_BASE(sbgemm) +#define SBGEMM_SMALL_KERNEL_NN OPENBLAS_DISPATCH_OFFSET(sbgemm, sbgemm_small_kernel_nn) +#define SBGEMM_SMALL_KERNEL_NT OPENBLAS_DISPATCH_OFFSET(sbgemm, sbgemm_small_kernel_nt) +#define SBGEMM_SMALL_KERNEL_TN OPENBLAS_DISPATCH_OFFSET(sbgemm, sbgemm_small_kernel_tn) +#define SBGEMM_SMALL_KERNEL_TT OPENBLAS_DISPATCH_OFFSET(sbgemm, sbgemm_small_kernel_tt) -#define SBGEMM_SMALL_KERNEL_B0_NN FUNC_OFFSET(sbgemm_small_kernel_b0_nn) -#define SBGEMM_SMALL_KERNEL_B0_NT FUNC_OFFSET(sbgemm_small_kernel_b0_nt) -#define SBGEMM_SMALL_KERNEL_B0_TN FUNC_OFFSET(sbgemm_small_kernel_b0_tn) -#define SBGEMM_SMALL_KERNEL_B0_TT FUNC_OFFSET(sbgemm_small_kernel_b0_tt) +#define SBGEMM_SMALL_KERNEL_B0_NN OPENBLAS_DISPATCH_OFFSET(sbgemm, sbgemm_small_kernel_b0_nn) +#define SBGEMM_SMALL_KERNEL_B0_NT OPENBLAS_DISPATCH_OFFSET(sbgemm, sbgemm_small_kernel_b0_nt) +#define SBGEMM_SMALL_KERNEL_B0_TN OPENBLAS_DISPATCH_OFFSET(sbgemm, sbgemm_small_kernel_b0_tn) +#define SBGEMM_SMALL_KERNEL_B0_TT OPENBLAS_DISPATCH_OFFSET(sbgemm, sbgemm_small_kernel_b0_tt) #define SBGEMM_NN sbgemm_nn #define SBGEMM_CN sbgemm_tn diff --git a/common_sh.h b/common_sh.h index 99dbb65180..7095b7d938 100644 --- a/common_sh.h +++ b/common_sh.h @@ -51,21 +51,21 @@ #else // #DYNAMIC_ARCH -#define SHGEMM_ONCOPY gotoblas -> shgemm_oncopy -#define SHGEMM_OTCOPY gotoblas -> shgemm_otcopy +#define SHGEMM_ONCOPY OPENBLAS_DISPATCH(shgemm) -> shgemm_oncopy +#define SHGEMM_OTCOPY OPENBLAS_DISPATCH(shgemm) -> shgemm_otcopy #if SGEMM_DEFAULT_UNROLL_M == SGEMM_DEFAULT_UNROLL_N -#define SHGEMM_INCOPY gotoblas -> shgemm_oncopy -#define SHGEMM_ITCOPY gotoblas -> shgemm_otcopy +#define SHGEMM_INCOPY OPENBLAS_DISPATCH(shgemm) -> shgemm_oncopy +#define SHGEMM_ITCOPY OPENBLAS_DISPATCH(shgemm) -> shgemm_otcopy #else -#define SHGEMM_INCOPY gotoblas -> shgemm_incopy -#define SHGEMM_ITCOPY gotoblas -> shgemm_itcopy +#define SHGEMM_INCOPY OPENBLAS_DISPATCH(shgemm) -> shgemm_incopy +#define SHGEMM_ITCOPY OPENBLAS_DISPATCH(shgemm) -> shgemm_itcopy #endif -#define SHGEMM_BETA gotoblas -> shgemm_beta -#define SHGEMM_KERNEL gotoblas -> shgemm_kernel +#define SHGEMM_BETA OPENBLAS_DISPATCH(shgemm) -> shgemm_beta +#define SHGEMM_KERNEL OPENBLAS_DISPATCH(shgemm) -> shgemm_kernel -#define SHGEMV_N_K gotoblas->shgemv_n -#define SHGEMV_T_K gotoblas->shgemv_t +#define SHGEMV_N_K OPENBLAS_DISPATCH(shgemv)->shgemv_n +#define SHGEMV_T_K OPENBLAS_DISPATCH(shgemv)->shgemv_t #endif // #DYNAMIC_ARCH diff --git a/common_x.h b/common_x.h index 12ed909c57..899d367b2a 100644 --- a/common_x.h +++ b/common_x.h @@ -212,46 +212,46 @@ #else -#define XAMAX_K gotoblas -> xamax_k -#define XAMIN_K gotoblas -> xamin_k +#define XAMAX_K OPENBLAS_DISPATCH(xamax) -> xamax_k +#define XAMIN_K OPENBLAS_DISPATCH(xamin) -> xamin_k #define XMAX_K gotoblas -> xmax_k #define XMIN_K gotoblas -> xmin_k -#define IXAMAX_K gotoblas -> ixamax_k -#define IXAMIN_K gotoblas -> ixamin_k +#define IXAMAX_K OPENBLAS_DISPATCH(ixamax) -> ixamax_k +#define IXAMIN_K OPENBLAS_DISPATCH(ixamin) -> ixamin_k #define IXMAX_K gotoblas -> ixmax_k #define IXMIN_K gotoblas -> ixmin_k -#define XASUM_K gotoblas -> xasum_k -#define XAXPYU_K gotoblas -> xaxpy_k -#define XAXPYC_K gotoblas -> xaxpyc_k -#define XCOPY_K gotoblas -> xcopy_k -#define XDOTU_K gotoblas -> xdotu_k -#define XDOTC_K gotoblas -> xdotc_k -#define XNRM2_K gotoblas -> xnrm2_k -#define XSCAL_K gotoblas -> xscal_k -#define XSUM_K gotoblas -> xsum_k -#define XSWAP_K gotoblas -> xswap_k -#define XROT_K gotoblas -> xqrot_k - -#define XGEMV_N gotoblas -> xgemv_n -#define XGEMV_T gotoblas -> xgemv_t -#define XGEMV_R gotoblas -> xgemv_r -#define XGEMV_C gotoblas -> xgemv_c -#define XGEMV_O gotoblas -> xgemv_o -#define XGEMV_U gotoblas -> xgemv_u -#define XGEMV_S gotoblas -> xgemv_s -#define XGEMV_D gotoblas -> xgemv_d - -#define XGERU_K gotoblas -> xgeru_k -#define XGERC_K gotoblas -> xgerc_k -#define XGERV_K gotoblas -> xgerv_k -#define XGERD_K gotoblas -> xgerd_k - -#define XSYMV_U gotoblas -> xsymv_U -#define XSYMV_L gotoblas -> xsymv_L -#define XHEMV_U gotoblas -> xhemv_U -#define XHEMV_L gotoblas -> xhemv_L -#define XHEMV_V gotoblas -> xhemv_V -#define XHEMV_M gotoblas -> xhemv_M +#define XASUM_K OPENBLAS_DISPATCH(xasum) -> xasum_k +#define XAXPYU_K OPENBLAS_DISPATCH(xaxpy) -> xaxpy_k +#define XAXPYC_K OPENBLAS_DISPATCH(xaxpyc) -> xaxpyc_k +#define XCOPY_K OPENBLAS_DISPATCH(xcopy) -> xcopy_k +#define XDOTU_K OPENBLAS_DISPATCH(xdotu) -> xdotu_k +#define XDOTC_K OPENBLAS_DISPATCH(xdotc) -> xdotc_k +#define XNRM2_K OPENBLAS_DISPATCH(xnrm2) -> xnrm2_k +#define XSCAL_K OPENBLAS_DISPATCH(xscal) -> xscal_k +#define XSUM_K OPENBLAS_DISPATCH(xsum) -> xsum_k +#define XSWAP_K OPENBLAS_DISPATCH(xswap) -> xswap_k +#define XROT_K OPENBLAS_DISPATCH(xqrot) -> xqrot_k + +#define XGEMV_N OPENBLAS_DISPATCH(xgemv) -> xgemv_n +#define XGEMV_T OPENBLAS_DISPATCH(xgemv) -> xgemv_t +#define XGEMV_R OPENBLAS_DISPATCH(xgemv) -> xgemv_r +#define XGEMV_C OPENBLAS_DISPATCH(xgemv) -> xgemv_c +#define XGEMV_O OPENBLAS_DISPATCH(xgemv) -> xgemv_o +#define XGEMV_U OPENBLAS_DISPATCH(xgemv) -> xgemv_u +#define XGEMV_S OPENBLAS_DISPATCH(xgemv) -> xgemv_s +#define XGEMV_D OPENBLAS_DISPATCH(xgemv) -> xgemv_d + +#define XGERU_K OPENBLAS_DISPATCH(xgeru) -> xgeru_k +#define XGERC_K OPENBLAS_DISPATCH(xgerc) -> xgerc_k +#define XGERV_K OPENBLAS_DISPATCH(xgerv) -> xgerv_k +#define XGERD_K OPENBLAS_DISPATCH(xgerd) -> xgerd_k + +#define XSYMV_U OPENBLAS_DISPATCH(xsymv) -> xsymv_U +#define XSYMV_L OPENBLAS_DISPATCH(xsymv) -> xsymv_L +#define XHEMV_U OPENBLAS_DISPATCH(xhemv) -> xhemv_U +#define XHEMV_L OPENBLAS_DISPATCH(xhemv) -> xhemv_L +#define XHEMV_V OPENBLAS_DISPATCH(xhemv) -> xhemv_V +#define XHEMV_M OPENBLAS_DISPATCH(xhemv) -> xhemv_M #define XSYMV_THREAD_U xsymv_thread_U #define XSYMV_THREAD_L xsymv_thread_L @@ -260,127 +260,127 @@ #define XHEMV_THREAD_V xhemv_thread_V #define XHEMV_THREAD_M xhemv_thread_M -#define XGEMM_ONCOPY gotoblas -> xgemm_oncopy -#define XGEMM_OTCOPY gotoblas -> xgemm_otcopy -#define XGEMM_INCOPY gotoblas -> xgemm_incopy -#define XGEMM_ITCOPY gotoblas -> xgemm_itcopy - -#define XTRMM_OUNUCOPY gotoblas -> xtrmm_ounucopy -#define XTRMM_OUTUCOPY gotoblas -> xtrmm_outucopy -#define XTRMM_OLNUCOPY gotoblas -> xtrmm_olnucopy -#define XTRMM_OLTUCOPY gotoblas -> xtrmm_oltucopy -#define XTRSM_OUNUCOPY gotoblas -> xtrsm_ounucopy -#define XTRSM_OUTUCOPY gotoblas -> xtrsm_outucopy -#define XTRSM_OLNUCOPY gotoblas -> xtrsm_olnucopy -#define XTRSM_OLTUCOPY gotoblas -> xtrsm_oltucopy - -#define XTRMM_IUNUCOPY gotoblas -> xtrmm_iunucopy -#define XTRMM_IUTUCOPY gotoblas -> xtrmm_iutucopy -#define XTRMM_ILNUCOPY gotoblas -> xtrmm_ilnucopy -#define XTRMM_ILTUCOPY gotoblas -> xtrmm_iltucopy -#define XTRSM_IUNUCOPY gotoblas -> xtrsm_iunucopy -#define XTRSM_IUTUCOPY gotoblas -> xtrsm_iutucopy -#define XTRSM_ILNUCOPY gotoblas -> xtrsm_ilnucopy -#define XTRSM_ILTUCOPY gotoblas -> xtrsm_iltucopy - -#define XTRMM_OUNNCOPY gotoblas -> xtrmm_ounncopy -#define XTRMM_OUTNCOPY gotoblas -> xtrmm_outncopy -#define XTRMM_OLNNCOPY gotoblas -> xtrmm_olnncopy -#define XTRMM_OLTNCOPY gotoblas -> xtrmm_oltncopy -#define XTRSM_OUNNCOPY gotoblas -> xtrsm_ounncopy -#define XTRSM_OUTNCOPY gotoblas -> xtrsm_outncopy -#define XTRSM_OLNNCOPY gotoblas -> xtrsm_olnncopy -#define XTRSM_OLTNCOPY gotoblas -> xtrsm_oltncopy - -#define XTRMM_IUNNCOPY gotoblas -> xtrmm_iunncopy -#define XTRMM_IUTNCOPY gotoblas -> xtrmm_iutncopy -#define XTRMM_ILNNCOPY gotoblas -> xtrmm_ilnncopy -#define XTRMM_ILTNCOPY gotoblas -> xtrmm_iltncopy -#define XTRSM_IUNNCOPY gotoblas -> xtrsm_iunncopy -#define XTRSM_IUTNCOPY gotoblas -> xtrsm_iutncopy -#define XTRSM_ILNNCOPY gotoblas -> xtrsm_ilnncopy -#define XTRSM_ILTNCOPY gotoblas -> xtrsm_iltncopy - -#define XGEMM_BETA gotoblas -> xgemm_beta -#define XGEMM_KERNEL_N gotoblas -> xgemm_kernel_n -#define XGEMM_KERNEL_L gotoblas -> xgemm_kernel_l -#define XGEMM_KERNEL_R gotoblas -> xgemm_kernel_r -#define XGEMM_KERNEL_B gotoblas -> xgemm_kernel_b - -#define XTRMM_KERNEL_LN gotoblas -> xtrmm_kernel_LN -#define XTRMM_KERNEL_LT gotoblas -> xtrmm_kernel_LT -#define XTRMM_KERNEL_LR gotoblas -> xtrmm_kernel_LR -#define XTRMM_KERNEL_LC gotoblas -> xtrmm_kernel_LC -#define XTRMM_KERNEL_RN gotoblas -> xtrmm_kernel_RN -#define XTRMM_KERNEL_RT gotoblas -> xtrmm_kernel_RT -#define XTRMM_KERNEL_RR gotoblas -> xtrmm_kernel_RR -#define XTRMM_KERNEL_RC gotoblas -> xtrmm_kernel_RC - -#define XTRSM_KERNEL_LN gotoblas -> xtrsm_kernel_LN -#define XTRSM_KERNEL_LT gotoblas -> xtrsm_kernel_LT -#define XTRSM_KERNEL_LR gotoblas -> xtrsm_kernel_LR -#define XTRSM_KERNEL_LC gotoblas -> xtrsm_kernel_LC -#define XTRSM_KERNEL_RN gotoblas -> xtrsm_kernel_RN -#define XTRSM_KERNEL_RT gotoblas -> xtrsm_kernel_RT -#define XTRSM_KERNEL_RR gotoblas -> xtrsm_kernel_RR -#define XTRSM_KERNEL_RC gotoblas -> xtrsm_kernel_RC - -#define XSYMM_IUTCOPY gotoblas -> xsymm_iutcopy -#define XSYMM_ILTCOPY gotoblas -> xsymm_iltcopy -#define XSYMM_OUTCOPY gotoblas -> xsymm_outcopy -#define XSYMM_OLTCOPY gotoblas -> xsymm_oltcopy - -#define XHEMM_OUTCOPY gotoblas -> xhemm_outcopy -#define XHEMM_OLTCOPY gotoblas -> xhemm_oltcopy -#define XHEMM_IUTCOPY gotoblas -> xhemm_iutcopy -#define XHEMM_ILTCOPY gotoblas -> xhemm_iltcopy - -#define XGEMM3M_ONCOPYB gotoblas -> xgemm3m_oncopyb -#define XGEMM3M_ONCOPYR gotoblas -> xgemm3m_oncopyr -#define XGEMM3M_ONCOPYI gotoblas -> xgemm3m_oncopyi -#define XGEMM3M_OTCOPYB gotoblas -> xgemm3m_otcopyb -#define XGEMM3M_OTCOPYR gotoblas -> xgemm3m_otcopyr -#define XGEMM3M_OTCOPYI gotoblas -> xgemm3m_otcopyi - -#define XGEMM3M_INCOPYB gotoblas -> xgemm3m_incopyb -#define XGEMM3M_INCOPYR gotoblas -> xgemm3m_incopyr -#define XGEMM3M_INCOPYI gotoblas -> xgemm3m_incopyi -#define XGEMM3M_ITCOPYB gotoblas -> xgemm3m_itcopyb -#define XGEMM3M_ITCOPYR gotoblas -> xgemm3m_itcopyr -#define XGEMM3M_ITCOPYI gotoblas -> xgemm3m_itcopyi - -#define XSYMM3M_ILCOPYB gotoblas -> xsymm3m_ilcopyb -#define XSYMM3M_IUCOPYB gotoblas -> xsymm3m_iucopyb -#define XSYMM3M_ILCOPYR gotoblas -> xsymm3m_ilcopyr -#define XSYMM3M_IUCOPYR gotoblas -> xsymm3m_iucopyr -#define XSYMM3M_ILCOPYI gotoblas -> xsymm3m_ilcopyi -#define XSYMM3M_IUCOPYI gotoblas -> xsymm3m_iucopyi - -#define XSYMM3M_OLCOPYB gotoblas -> xsymm3m_olcopyb -#define XSYMM3M_OUCOPYB gotoblas -> xsymm3m_oucopyb -#define XSYMM3M_OLCOPYR gotoblas -> xsymm3m_olcopyr -#define XSYMM3M_OUCOPYR gotoblas -> xsymm3m_oucopyr -#define XSYMM3M_OLCOPYI gotoblas -> xsymm3m_olcopyi -#define XSYMM3M_OUCOPYI gotoblas -> xsymm3m_oucopyi - -#define XHEMM3M_ILCOPYB gotoblas -> xhemm3m_ilcopyb -#define XHEMM3M_IUCOPYB gotoblas -> xhemm3m_iucopyb -#define XHEMM3M_ILCOPYR gotoblas -> xhemm3m_ilcopyr -#define XHEMM3M_IUCOPYR gotoblas -> xhemm3m_iucopyr -#define XHEMM3M_ILCOPYI gotoblas -> xhemm3m_ilcopyi -#define XHEMM3M_IUCOPYI gotoblas -> xhemm3m_iucopyi - -#define XHEMM3M_OLCOPYB gotoblas -> xhemm3m_olcopyb -#define XHEMM3M_OUCOPYB gotoblas -> xhemm3m_oucopyb -#define XHEMM3M_OLCOPYR gotoblas -> xhemm3m_olcopyr -#define XHEMM3M_OUCOPYR gotoblas -> xhemm3m_oucopyr -#define XHEMM3M_OLCOPYI gotoblas -> xhemm3m_olcopyi -#define XHEMM3M_OUCOPYI gotoblas -> xhemm3m_oucopyi - -#define XGEMM3M_KERNEL gotoblas -> xgemm3m_kernel - -#define XNEG_TCOPY gotoblas -> xneg_tcopy -#define XLASWP_NCOPY gotoblas -> xlaswp_ncopy +#define XGEMM_ONCOPY OPENBLAS_DISPATCH(xgemm) -> xgemm_oncopy +#define XGEMM_OTCOPY OPENBLAS_DISPATCH(xgemm) -> xgemm_otcopy +#define XGEMM_INCOPY OPENBLAS_DISPATCH(xgemm) -> xgemm_incopy +#define XGEMM_ITCOPY OPENBLAS_DISPATCH(xgemm) -> xgemm_itcopy + +#define XTRMM_OUNUCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_ounucopy +#define XTRMM_OUTUCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_outucopy +#define XTRMM_OLNUCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_olnucopy +#define XTRMM_OLTUCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_oltucopy +#define XTRSM_OUNUCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_ounucopy +#define XTRSM_OUTUCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_outucopy +#define XTRSM_OLNUCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_olnucopy +#define XTRSM_OLTUCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_oltucopy + +#define XTRMM_IUNUCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_iunucopy +#define XTRMM_IUTUCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_iutucopy +#define XTRMM_ILNUCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_ilnucopy +#define XTRMM_ILTUCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_iltucopy +#define XTRSM_IUNUCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_iunucopy +#define XTRSM_IUTUCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_iutucopy +#define XTRSM_ILNUCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_ilnucopy +#define XTRSM_ILTUCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_iltucopy + +#define XTRMM_OUNNCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_ounncopy +#define XTRMM_OUTNCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_outncopy +#define XTRMM_OLNNCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_olnncopy +#define XTRMM_OLTNCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_oltncopy +#define XTRSM_OUNNCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_ounncopy +#define XTRSM_OUTNCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_outncopy +#define XTRSM_OLNNCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_olnncopy +#define XTRSM_OLTNCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_oltncopy + +#define XTRMM_IUNNCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_iunncopy +#define XTRMM_IUTNCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_iutncopy +#define XTRMM_ILNNCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_ilnncopy +#define XTRMM_ILTNCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_iltncopy +#define XTRSM_IUNNCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_iunncopy +#define XTRSM_IUTNCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_iutncopy +#define XTRSM_ILNNCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_ilnncopy +#define XTRSM_ILTNCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_iltncopy + +#define XGEMM_BETA OPENBLAS_DISPATCH(xgemm) -> xgemm_beta +#define XGEMM_KERNEL_N OPENBLAS_DISPATCH(xgemm) -> xgemm_kernel_n +#define XGEMM_KERNEL_L OPENBLAS_DISPATCH(xgemm) -> xgemm_kernel_l +#define XGEMM_KERNEL_R OPENBLAS_DISPATCH(xgemm) -> xgemm_kernel_r +#define XGEMM_KERNEL_B OPENBLAS_DISPATCH(xgemm) -> xgemm_kernel_b + +#define XTRMM_KERNEL_LN OPENBLAS_DISPATCH(xtrmm) -> xtrmm_kernel_LN +#define XTRMM_KERNEL_LT OPENBLAS_DISPATCH(xtrmm) -> xtrmm_kernel_LT +#define XTRMM_KERNEL_LR OPENBLAS_DISPATCH(xtrmm) -> xtrmm_kernel_LR +#define XTRMM_KERNEL_LC OPENBLAS_DISPATCH(xtrmm) -> xtrmm_kernel_LC +#define XTRMM_KERNEL_RN OPENBLAS_DISPATCH(xtrmm) -> xtrmm_kernel_RN +#define XTRMM_KERNEL_RT OPENBLAS_DISPATCH(xtrmm) -> xtrmm_kernel_RT +#define XTRMM_KERNEL_RR OPENBLAS_DISPATCH(xtrmm) -> xtrmm_kernel_RR +#define XTRMM_KERNEL_RC OPENBLAS_DISPATCH(xtrmm) -> xtrmm_kernel_RC + +#define XTRSM_KERNEL_LN OPENBLAS_DISPATCH(xtrsm) -> xtrsm_kernel_LN +#define XTRSM_KERNEL_LT OPENBLAS_DISPATCH(xtrsm) -> xtrsm_kernel_LT +#define XTRSM_KERNEL_LR OPENBLAS_DISPATCH(xtrsm) -> xtrsm_kernel_LR +#define XTRSM_KERNEL_LC OPENBLAS_DISPATCH(xtrsm) -> xtrsm_kernel_LC +#define XTRSM_KERNEL_RN OPENBLAS_DISPATCH(xtrsm) -> xtrsm_kernel_RN +#define XTRSM_KERNEL_RT OPENBLAS_DISPATCH(xtrsm) -> xtrsm_kernel_RT +#define XTRSM_KERNEL_RR OPENBLAS_DISPATCH(xtrsm) -> xtrsm_kernel_RR +#define XTRSM_KERNEL_RC OPENBLAS_DISPATCH(xtrsm) -> xtrsm_kernel_RC + +#define XSYMM_IUTCOPY OPENBLAS_DISPATCH(xsymm) -> xsymm_iutcopy +#define XSYMM_ILTCOPY OPENBLAS_DISPATCH(xsymm) -> xsymm_iltcopy +#define XSYMM_OUTCOPY OPENBLAS_DISPATCH(xsymm) -> xsymm_outcopy +#define XSYMM_OLTCOPY OPENBLAS_DISPATCH(xsymm) -> xsymm_oltcopy + +#define XHEMM_OUTCOPY OPENBLAS_DISPATCH(xhemm) -> xhemm_outcopy +#define XHEMM_OLTCOPY OPENBLAS_DISPATCH(xhemm) -> xhemm_oltcopy +#define XHEMM_IUTCOPY OPENBLAS_DISPATCH(xhemm) -> xhemm_iutcopy +#define XHEMM_ILTCOPY OPENBLAS_DISPATCH(xhemm) -> xhemm_iltcopy + +#define XGEMM3M_ONCOPYB OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_oncopyb +#define XGEMM3M_ONCOPYR OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_oncopyr +#define XGEMM3M_ONCOPYI OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_oncopyi +#define XGEMM3M_OTCOPYB OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_otcopyb +#define XGEMM3M_OTCOPYR OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_otcopyr +#define XGEMM3M_OTCOPYI OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_otcopyi + +#define XGEMM3M_INCOPYB OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_incopyb +#define XGEMM3M_INCOPYR OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_incopyr +#define XGEMM3M_INCOPYI OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_incopyi +#define XGEMM3M_ITCOPYB OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_itcopyb +#define XGEMM3M_ITCOPYR OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_itcopyr +#define XGEMM3M_ITCOPYI OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_itcopyi + +#define XSYMM3M_ILCOPYB OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_ilcopyb +#define XSYMM3M_IUCOPYB OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_iucopyb +#define XSYMM3M_ILCOPYR OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_ilcopyr +#define XSYMM3M_IUCOPYR OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_iucopyr +#define XSYMM3M_ILCOPYI OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_ilcopyi +#define XSYMM3M_IUCOPYI OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_iucopyi + +#define XSYMM3M_OLCOPYB OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_olcopyb +#define XSYMM3M_OUCOPYB OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_oucopyb +#define XSYMM3M_OLCOPYR OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_olcopyr +#define XSYMM3M_OUCOPYR OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_oucopyr +#define XSYMM3M_OLCOPYI OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_olcopyi +#define XSYMM3M_OUCOPYI OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_oucopyi + +#define XHEMM3M_ILCOPYB OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_ilcopyb +#define XHEMM3M_IUCOPYB OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_iucopyb +#define XHEMM3M_ILCOPYR OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_ilcopyr +#define XHEMM3M_IUCOPYR OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_iucopyr +#define XHEMM3M_ILCOPYI OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_ilcopyi +#define XHEMM3M_IUCOPYI OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_iucopyi + +#define XHEMM3M_OLCOPYB OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_olcopyb +#define XHEMM3M_OUCOPYB OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_oucopyb +#define XHEMM3M_OLCOPYR OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_olcopyr +#define XHEMM3M_OUCOPYR OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_oucopyr +#define XHEMM3M_OLCOPYI OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_olcopyi +#define XHEMM3M_OUCOPYI OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_oucopyi + +#define XGEMM3M_KERNEL OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_kernel + +#define XNEG_TCOPY OPENBLAS_DISPATCH(xneg) -> xneg_tcopy +#define XLASWP_NCOPY OPENBLAS_DISPATCH(xlaswp) -> xlaswp_ncopy #endif diff --git a/common_z.h b/common_z.h index 24f88595df..40fb2dd39b 100644 --- a/common_z.h +++ b/common_z.h @@ -290,46 +290,46 @@ #else -#define ZAMAX_K gotoblas -> zamax_k -#define ZAMIN_K gotoblas -> zamin_k +#define ZAMAX_K OPENBLAS_DISPATCH(zamax) -> zamax_k +#define ZAMIN_K OPENBLAS_DISPATCH(zamin) -> zamin_k #define ZMAX_K gotoblas -> zmax_k #define ZMIN_K gotoblas -> zmin_k -#define IZAMAX_K gotoblas -> izamax_k -#define IZAMIN_K gotoblas -> izamin_k +#define IZAMAX_K OPENBLAS_DISPATCH(izamax) -> izamax_k +#define IZAMIN_K OPENBLAS_DISPATCH(izamin) -> izamin_k #define IZMAX_K gotoblas -> izmax_k #define IZMIN_K gotoblas -> izmin_k -#define ZASUM_K gotoblas -> zasum_k -#define ZAXPYU_K gotoblas -> zaxpy_k -#define ZAXPYC_K gotoblas -> zaxpyc_k -#define ZCOPY_K gotoblas -> zcopy_k -#define ZDOTU_K gotoblas -> zdotu_k -#define ZDOTC_K gotoblas -> zdotc_k -#define ZNRM2_K gotoblas -> znrm2_k -#define ZSCAL_K gotoblas -> zscal_k -#define ZSUM_K gotoblas -> zsum_k -#define ZSWAP_K gotoblas -> zswap_k -#define ZROT_K gotoblas -> zdrot_k - -#define ZGEMV_N gotoblas -> zgemv_n -#define ZGEMV_T gotoblas -> zgemv_t -#define ZGEMV_R gotoblas -> zgemv_r -#define ZGEMV_C gotoblas -> zgemv_c -#define ZGEMV_O gotoblas -> zgemv_o -#define ZGEMV_U gotoblas -> zgemv_u -#define ZGEMV_S gotoblas -> zgemv_s -#define ZGEMV_D gotoblas -> zgemv_d - -#define ZGERU_K gotoblas -> zgeru_k -#define ZGERC_K gotoblas -> zgerc_k -#define ZGERV_K gotoblas -> zgerv_k -#define ZGERD_K gotoblas -> zgerd_k - -#define ZSYMV_U gotoblas -> zsymv_U -#define ZSYMV_L gotoblas -> zsymv_L -#define ZHEMV_U gotoblas -> zhemv_U -#define ZHEMV_L gotoblas -> zhemv_L -#define ZHEMV_V gotoblas -> zhemv_V -#define ZHEMV_M gotoblas -> zhemv_M +#define ZASUM_K OPENBLAS_DISPATCH(zasum) -> zasum_k +#define ZAXPYU_K OPENBLAS_DISPATCH(zaxpy) -> zaxpy_k +#define ZAXPYC_K OPENBLAS_DISPATCH(zaxpyc) -> zaxpyc_k +#define ZCOPY_K OPENBLAS_DISPATCH(zcopy) -> zcopy_k +#define ZDOTU_K OPENBLAS_DISPATCH(zdotu) -> zdotu_k +#define ZDOTC_K OPENBLAS_DISPATCH(zdotc) -> zdotc_k +#define ZNRM2_K OPENBLAS_DISPATCH(znrm2) -> znrm2_k +#define ZSCAL_K OPENBLAS_DISPATCH(zscal) -> zscal_k +#define ZSUM_K OPENBLAS_DISPATCH(zsum) -> zsum_k +#define ZSWAP_K OPENBLAS_DISPATCH(zswap) -> zswap_k +#define ZROT_K OPENBLAS_DISPATCH(zdrot) -> zdrot_k + +#define ZGEMV_N OPENBLAS_DISPATCH(zgemv) -> zgemv_n +#define ZGEMV_T OPENBLAS_DISPATCH(zgemv) -> zgemv_t +#define ZGEMV_R OPENBLAS_DISPATCH(zgemv) -> zgemv_r +#define ZGEMV_C OPENBLAS_DISPATCH(zgemv) -> zgemv_c +#define ZGEMV_O OPENBLAS_DISPATCH(zgemv) -> zgemv_o +#define ZGEMV_U OPENBLAS_DISPATCH(zgemv) -> zgemv_u +#define ZGEMV_S OPENBLAS_DISPATCH(zgemv) -> zgemv_s +#define ZGEMV_D OPENBLAS_DISPATCH(zgemv) -> zgemv_d + +#define ZGERU_K OPENBLAS_DISPATCH(zgeru) -> zgeru_k +#define ZGERC_K OPENBLAS_DISPATCH(zgerc) -> zgerc_k +#define ZGERV_K OPENBLAS_DISPATCH(zgerv) -> zgerv_k +#define ZGERD_K OPENBLAS_DISPATCH(zgerd) -> zgerd_k + +#define ZSYMV_U OPENBLAS_DISPATCH(zsymv) -> zsymv_U +#define ZSYMV_L OPENBLAS_DISPATCH(zsymv) -> zsymv_L +#define ZHEMV_U OPENBLAS_DISPATCH(zhemv) -> zhemv_U +#define ZHEMV_L OPENBLAS_DISPATCH(zhemv) -> zhemv_L +#define ZHEMV_V OPENBLAS_DISPATCH(zhemv) -> zhemv_V +#define ZHEMV_M OPENBLAS_DISPATCH(zhemv) -> zhemv_M #define ZSYMV_THREAD_U zsymv_thread_U #define ZSYMV_THREAD_L zsymv_thread_L @@ -338,208 +338,209 @@ #define ZHEMV_THREAD_V zhemv_thread_V #define ZHEMV_THREAD_M zhemv_thread_M -#define ZGEMM_ONCOPY gotoblas -> zgemm_oncopy -#define ZGEMM_OTCOPY gotoblas -> zgemm_otcopy -#define ZGEMM_INCOPY gotoblas -> zgemm_incopy -#define ZGEMM_ITCOPY gotoblas -> zgemm_itcopy - -#define ZTRMM_OUNUCOPY gotoblas -> ztrmm_ounucopy -#define ZTRMM_OUTUCOPY gotoblas -> ztrmm_outucopy -#define ZTRMM_OLNUCOPY gotoblas -> ztrmm_olnucopy -#define ZTRMM_OLTUCOPY gotoblas -> ztrmm_oltucopy -#define ZTRSM_OUNUCOPY gotoblas -> ztrsm_ounucopy -#define ZTRSM_OUTUCOPY gotoblas -> ztrsm_outucopy -#define ZTRSM_OLNUCOPY gotoblas -> ztrsm_olnucopy -#define ZTRSM_OLTUCOPY gotoblas -> ztrsm_oltucopy - -#define ZTRMM_IUNUCOPY gotoblas -> ztrmm_iunucopy -#define ZTRMM_IUTUCOPY gotoblas -> ztrmm_iutucopy -#define ZTRMM_ILNUCOPY gotoblas -> ztrmm_ilnucopy -#define ZTRMM_ILTUCOPY gotoblas -> ztrmm_iltucopy -#define ZTRSM_IUNUCOPY gotoblas -> ztrsm_iunucopy -#define ZTRSM_IUTUCOPY gotoblas -> ztrsm_iutucopy -#define ZTRSM_ILNUCOPY gotoblas -> ztrsm_ilnucopy -#define ZTRSM_ILTUCOPY gotoblas -> ztrsm_iltucopy - -#define ZTRMM_OUNNCOPY gotoblas -> ztrmm_ounncopy -#define ZTRMM_OUTNCOPY gotoblas -> ztrmm_outncopy -#define ZTRMM_OLNNCOPY gotoblas -> ztrmm_olnncopy -#define ZTRMM_OLTNCOPY gotoblas -> ztrmm_oltncopy -#define ZTRSM_OUNNCOPY gotoblas -> ztrsm_ounncopy -#define ZTRSM_OUTNCOPY gotoblas -> ztrsm_outncopy -#define ZTRSM_OLNNCOPY gotoblas -> ztrsm_olnncopy -#define ZTRSM_OLTNCOPY gotoblas -> ztrsm_oltncopy - -#define ZTRMM_IUNNCOPY gotoblas -> ztrmm_iunncopy -#define ZTRMM_IUTNCOPY gotoblas -> ztrmm_iutncopy -#define ZTRMM_ILNNCOPY gotoblas -> ztrmm_ilnncopy -#define ZTRMM_ILTNCOPY gotoblas -> ztrmm_iltncopy -#define ZSYMM_INCOPY gotoblas -> zsymm_incopy -#define ZSYMM_ITCOPY gotoblas -> zsymm_itcopy -#define ZTRMM_INCOPY gotoblas -> ztrmm_incopy -#define ZTRMM_ITCOPY gotoblas -> ztrmm_itcopy - -#define ZTRSM_IUNNCOPY gotoblas -> ztrsm_iunncopy -#define ZTRSM_IUTNCOPY gotoblas -> ztrsm_iutncopy -#define ZTRSM_ILNNCOPY gotoblas -> ztrsm_ilnncopy -#define ZTRSM_ILTNCOPY gotoblas -> ztrsm_iltncopy - -#define ZGEMM_BETA gotoblas -> zgemm_beta -#define SME_ZGEMM_KERNEL gotoblas -> sme_zgemm_kernel -#define ZGEMM_KERNEL_N gotoblas -> zgemm_kernel_n -#define ZGEMM_KERNEL_L gotoblas -> zgemm_kernel_l -#define ZGEMM_KERNEL_R gotoblas -> zgemm_kernel_r -#define ZGEMM_KERNEL_B gotoblas -> zgemm_kernel_b - -#define ZSYMM_KERNEL_N gotoblas -> zsymm_kernel_n -#define ZSYMM_KERNEL_L gotoblas -> zsymm_kernel_l -#define ZSYMM_KERNEL_R gotoblas -> zsymm_kernel_r -#define ZSYMM_KERNEL_B gotoblas -> zsymm_kernel_b -#define ZTRMM_GEMM_KERNEL_N gotoblas -> ztrmm_gemm_kernel_n -#define ZTRMM_GEMM_KERNEL_L gotoblas -> ztrmm_gemm_kernel_l -#define ZTRMM_GEMM_KERNEL_R gotoblas -> ztrmm_gemm_kernel_r -#define ZTRMM_GEMM_KERNEL_B gotoblas -> ztrmm_gemm_kernel_b - -#define ZTRMM_KERNEL_LN gotoblas -> ztrmm_kernel_LN -#define ZTRMM_KERNEL_LT gotoblas -> ztrmm_kernel_LT -#define ZTRMM_KERNEL_LR gotoblas -> ztrmm_kernel_LR -#define ZTRMM_KERNEL_LC gotoblas -> ztrmm_kernel_LC -#define ZTRMM_KERNEL_RN gotoblas -> ztrmm_kernel_RN -#define ZTRMM_KERNEL_RT gotoblas -> ztrmm_kernel_RT -#define ZTRMM_KERNEL_RR gotoblas -> ztrmm_kernel_RR -#define ZTRMM_KERNEL_RC gotoblas -> ztrmm_kernel_RC - -#define ZTRSM_KERNEL_LN gotoblas -> ztrsm_kernel_LN -#define ZTRSM_KERNEL_LT gotoblas -> ztrsm_kernel_LT -#define ZTRSM_KERNEL_LR gotoblas -> ztrsm_kernel_LR -#define ZTRSM_KERNEL_LC gotoblas -> ztrsm_kernel_LC -#define ZTRSM_KERNEL_RN gotoblas -> ztrsm_kernel_RN -#define ZTRSM_KERNEL_RT gotoblas -> ztrsm_kernel_RT -#define ZTRSM_KERNEL_RR gotoblas -> ztrsm_kernel_RR -#define ZTRSM_KERNEL_RC gotoblas -> ztrsm_kernel_RC - -#define ZSYMM_IUTCOPY gotoblas -> zsymm_iutcopy -#define ZSYMM_ILTCOPY gotoblas -> zsymm_iltcopy -#define ZSYMM_OUTCOPY gotoblas -> zsymm_outcopy -#define ZSYMM_OLTCOPY gotoblas -> zsymm_oltcopy - -#define ZHEMM_OUTCOPY gotoblas -> zhemm_outcopy -#define ZHEMM_OLTCOPY gotoblas -> zhemm_oltcopy -#define ZHEMM_IUTCOPY gotoblas -> zhemm_iutcopy -#define ZHEMM_ILTCOPY gotoblas -> zhemm_iltcopy - -#define ZGEMM3M_ONCOPYB gotoblas -> zgemm3m_oncopyb -#define ZGEMM3M_ONCOPYR gotoblas -> zgemm3m_oncopyr -#define ZGEMM3M_ONCOPYI gotoblas -> zgemm3m_oncopyi -#define ZGEMM3M_OTCOPYB gotoblas -> zgemm3m_otcopyb -#define ZGEMM3M_OTCOPYR gotoblas -> zgemm3m_otcopyr -#define ZGEMM3M_OTCOPYI gotoblas -> zgemm3m_otcopyi - -#define ZGEMM3M_INCOPYB gotoblas -> zgemm3m_incopyb -#define ZGEMM3M_INCOPYR gotoblas -> zgemm3m_incopyr -#define ZGEMM3M_INCOPYI gotoblas -> zgemm3m_incopyi -#define ZGEMM3M_ITCOPYB gotoblas -> zgemm3m_itcopyb -#define ZGEMM3M_ITCOPYR gotoblas -> zgemm3m_itcopyr -#define ZGEMM3M_ITCOPYI gotoblas -> zgemm3m_itcopyi - -#define ZSYMM3M_ILCOPYB gotoblas -> zsymm3m_ilcopyb -#define ZSYMM3M_IUCOPYB gotoblas -> zsymm3m_iucopyb -#define ZSYMM3M_ILCOPYR gotoblas -> zsymm3m_ilcopyr -#define ZSYMM3M_IUCOPYR gotoblas -> zsymm3m_iucopyr -#define ZSYMM3M_ILCOPYI gotoblas -> zsymm3m_ilcopyi -#define ZSYMM3M_IUCOPYI gotoblas -> zsymm3m_iucopyi - -#define ZSYMM3M_OLCOPYB gotoblas -> zsymm3m_olcopyb -#define ZSYMM3M_OUCOPYB gotoblas -> zsymm3m_oucopyb -#define ZSYMM3M_OLCOPYR gotoblas -> zsymm3m_olcopyr -#define ZSYMM3M_OUCOPYR gotoblas -> zsymm3m_oucopyr -#define ZSYMM3M_OLCOPYI gotoblas -> zsymm3m_olcopyi -#define ZSYMM3M_OUCOPYI gotoblas -> zsymm3m_oucopyi - -#define ZHEMM3M_ILCOPYB gotoblas -> zhemm3m_ilcopyb -#define ZHEMM3M_IUCOPYB gotoblas -> zhemm3m_iucopyb -#define ZHEMM3M_ILCOPYR gotoblas -> zhemm3m_ilcopyr -#define ZHEMM3M_IUCOPYR gotoblas -> zhemm3m_iucopyr -#define ZHEMM3M_ILCOPYI gotoblas -> zhemm3m_ilcopyi -#define ZHEMM3M_IUCOPYI gotoblas -> zhemm3m_iucopyi - -#define ZHEMM3M_OLCOPYB gotoblas -> zhemm3m_olcopyb -#define ZHEMM3M_OUCOPYB gotoblas -> zhemm3m_oucopyb -#define ZHEMM3M_OLCOPYR gotoblas -> zhemm3m_olcopyr -#define ZHEMM3M_OUCOPYR gotoblas -> zhemm3m_oucopyr -#define ZHEMM3M_OLCOPYI gotoblas -> zhemm3m_olcopyi -#define ZHEMM3M_OUCOPYI gotoblas -> zhemm3m_oucopyi - -#define ZGEMM3M_KERNEL gotoblas -> zgemm3m_kernel - -#define ZNEG_TCOPY gotoblas -> zneg_tcopy -#define ZLASWP_NCOPY gotoblas -> zlaswp_ncopy - -#define ZAXPBY_K gotoblas -> zaxpby_k - -#define ZOMATCOPY_K_CN gotoblas -> zomatcopy_k_cn -#define ZOMATCOPY_K_RN gotoblas -> zomatcopy_k_rn -#define ZOMATCOPY_K_CT gotoblas -> zomatcopy_k_ct -#define ZOMATCOPY_K_RT gotoblas -> zomatcopy_k_rt -#define ZOMATCOPY_K_CNC gotoblas -> zomatcopy_k_cnc -#define ZOMATCOPY_K_RNC gotoblas -> zomatcopy_k_rnc -#define ZOMATCOPY_K_CTC gotoblas -> zomatcopy_k_ctc -#define ZOMATCOPY_K_RTC gotoblas -> zomatcopy_k_rtc - -#define ZIMATCOPY_K_CN gotoblas -> zimatcopy_k_cn -#define ZIMATCOPY_K_RN gotoblas -> zimatcopy_k_rn -#define ZIMATCOPY_K_CT gotoblas -> zimatcopy_k_ct -#define ZIMATCOPY_K_RT gotoblas -> zimatcopy_k_rt -#define ZIMATCOPY_K_CNC gotoblas -> zimatcopy_k_cnc -#define ZIMATCOPY_K_RNC gotoblas -> zimatcopy_k_rnc -#define ZIMATCOPY_K_CTC gotoblas -> zimatcopy_k_ctc -#define ZIMATCOPY_K_RTC gotoblas -> zimatcopy_k_rtc - -#define ZGEADD_K gotoblas -> zgeadd_k - -#define ZGEMM_SMALL_MATRIX_PERMIT gotoblas -> zgemm_small_matrix_permit +#define ZGEMM_ONCOPY OPENBLAS_DISPATCH(zgemm) -> zgemm_oncopy +#define ZGEMM_OTCOPY OPENBLAS_DISPATCH(zgemm) -> zgemm_otcopy +#define ZGEMM_INCOPY OPENBLAS_DISPATCH(zgemm) -> zgemm_incopy +#define ZGEMM_ITCOPY OPENBLAS_DISPATCH(zgemm) -> zgemm_itcopy + +#define ZTRMM_OUNUCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_ounucopy +#define ZTRMM_OUTUCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_outucopy +#define ZTRMM_OLNUCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_olnucopy +#define ZTRMM_OLTUCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_oltucopy +#define ZTRSM_OUNUCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_ounucopy +#define ZTRSM_OUTUCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_outucopy +#define ZTRSM_OLNUCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_olnucopy +#define ZTRSM_OLTUCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_oltucopy + +#define ZTRMM_IUNUCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_iunucopy +#define ZTRMM_IUTUCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_iutucopy +#define ZTRMM_ILNUCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_ilnucopy +#define ZTRMM_ILTUCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_iltucopy +#define ZTRSM_IUNUCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_iunucopy +#define ZTRSM_IUTUCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_iutucopy +#define ZTRSM_ILNUCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_ilnucopy +#define ZTRSM_ILTUCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_iltucopy + +#define ZTRMM_OUNNCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_ounncopy +#define ZTRMM_OUTNCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_outncopy +#define ZTRMM_OLNNCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_olnncopy +#define ZTRMM_OLTNCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_oltncopy +#define ZTRSM_OUNNCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_ounncopy +#define ZTRSM_OUTNCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_outncopy +#define ZTRSM_OLNNCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_olnncopy +#define ZTRSM_OLTNCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_oltncopy + +#define ZTRMM_IUNNCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_iunncopy +#define ZTRMM_IUTNCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_iutncopy +#define ZTRMM_ILNNCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_ilnncopy +#define ZTRMM_ILTNCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_iltncopy +#define ZSYMM_INCOPY OPENBLAS_DISPATCH(zsymm) -> zsymm_incopy +#define ZSYMM_ITCOPY OPENBLAS_DISPATCH(zsymm) -> zsymm_itcopy +#define ZTRMM_INCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_incopy +#define ZTRMM_ITCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_itcopy + +#define ZTRSM_IUNNCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_iunncopy +#define ZTRSM_IUTNCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_iutncopy +#define ZTRSM_ILNNCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_ilnncopy +#define ZTRSM_ILTNCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_iltncopy + +#define ZGEMM_BETA OPENBLAS_DISPATCH(zgemm) -> zgemm_beta +#define SME_ZGEMM_KERNEL OPENBLAS_DISPATCH(zgemm) -> sme_zgemm_kernel +#define ZGEMM_KERNEL_N OPENBLAS_DISPATCH(zgemm) -> zgemm_kernel_n +#define ZGEMM_KERNEL_L OPENBLAS_DISPATCH(zgemm) -> zgemm_kernel_l +#define ZGEMM_KERNEL_R OPENBLAS_DISPATCH(zgemm) -> zgemm_kernel_r +#define ZGEMM_KERNEL_B OPENBLAS_DISPATCH(zgemm) -> zgemm_kernel_b + +#define ZSYMM_KERNEL_N OPENBLAS_DISPATCH(zsymm) -> zsymm_kernel_n +#define ZSYMM_KERNEL_L OPENBLAS_DISPATCH(zsymm) -> zsymm_kernel_l +#define ZSYMM_KERNEL_R OPENBLAS_DISPATCH(zsymm) -> zsymm_kernel_r +#define ZSYMM_KERNEL_B OPENBLAS_DISPATCH(zsymm) -> zsymm_kernel_b +#define ZTRMM_GEMM_KERNEL_N OPENBLAS_DISPATCH(ztrmm) -> ztrmm_gemm_kernel_n +#define ZTRMM_GEMM_KERNEL_L OPENBLAS_DISPATCH(ztrmm) -> ztrmm_gemm_kernel_l +#define ZTRMM_GEMM_KERNEL_R OPENBLAS_DISPATCH(ztrmm) -> ztrmm_gemm_kernel_r +#define ZTRMM_GEMM_KERNEL_B OPENBLAS_DISPATCH(ztrmm) -> ztrmm_gemm_kernel_b + +#define ZTRMM_KERNEL_LN OPENBLAS_DISPATCH(ztrmm) -> ztrmm_kernel_LN +#define ZTRMM_KERNEL_LT OPENBLAS_DISPATCH(ztrmm) -> ztrmm_kernel_LT +#define ZTRMM_KERNEL_LR OPENBLAS_DISPATCH(ztrmm) -> ztrmm_kernel_LR +#define ZTRMM_KERNEL_LC OPENBLAS_DISPATCH(ztrmm) -> ztrmm_kernel_LC +#define ZTRMM_KERNEL_RN OPENBLAS_DISPATCH(ztrmm) -> ztrmm_kernel_RN +#define ZTRMM_KERNEL_RT OPENBLAS_DISPATCH(ztrmm) -> ztrmm_kernel_RT +#define ZTRMM_KERNEL_RR OPENBLAS_DISPATCH(ztrmm) -> ztrmm_kernel_RR +#define ZTRMM_KERNEL_RC OPENBLAS_DISPATCH(ztrmm) -> ztrmm_kernel_RC + +#define ZTRSM_KERNEL_LN OPENBLAS_DISPATCH(ztrsm) -> ztrsm_kernel_LN +#define ZTRSM_KERNEL_LT OPENBLAS_DISPATCH(ztrsm) -> ztrsm_kernel_LT +#define ZTRSM_KERNEL_LR OPENBLAS_DISPATCH(ztrsm) -> ztrsm_kernel_LR +#define ZTRSM_KERNEL_LC OPENBLAS_DISPATCH(ztrsm) -> ztrsm_kernel_LC +#define ZTRSM_KERNEL_RN OPENBLAS_DISPATCH(ztrsm) -> ztrsm_kernel_RN +#define ZTRSM_KERNEL_RT OPENBLAS_DISPATCH(ztrsm) -> ztrsm_kernel_RT +#define ZTRSM_KERNEL_RR OPENBLAS_DISPATCH(ztrsm) -> ztrsm_kernel_RR +#define ZTRSM_KERNEL_RC OPENBLAS_DISPATCH(ztrsm) -> ztrsm_kernel_RC + +#define ZSYMM_IUTCOPY OPENBLAS_DISPATCH(zsymm) -> zsymm_iutcopy +#define ZSYMM_ILTCOPY OPENBLAS_DISPATCH(zsymm) -> zsymm_iltcopy +#define ZSYMM_OUTCOPY OPENBLAS_DISPATCH(zsymm) -> zsymm_outcopy +#define ZSYMM_OLTCOPY OPENBLAS_DISPATCH(zsymm) -> zsymm_oltcopy + +#define ZHEMM_OUTCOPY OPENBLAS_DISPATCH(zhemm) -> zhemm_outcopy +#define ZHEMM_OLTCOPY OPENBLAS_DISPATCH(zhemm) -> zhemm_oltcopy +#define ZHEMM_IUTCOPY OPENBLAS_DISPATCH(zhemm) -> zhemm_iutcopy +#define ZHEMM_ILTCOPY OPENBLAS_DISPATCH(zhemm) -> zhemm_iltcopy + +#define ZGEMM3M_ONCOPYB OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_oncopyb +#define ZGEMM3M_ONCOPYR OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_oncopyr +#define ZGEMM3M_ONCOPYI OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_oncopyi +#define ZGEMM3M_OTCOPYB OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_otcopyb +#define ZGEMM3M_OTCOPYR OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_otcopyr +#define ZGEMM3M_OTCOPYI OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_otcopyi + +#define ZGEMM3M_INCOPYB OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_incopyb +#define ZGEMM3M_INCOPYR OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_incopyr +#define ZGEMM3M_INCOPYI OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_incopyi +#define ZGEMM3M_ITCOPYB OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_itcopyb +#define ZGEMM3M_ITCOPYR OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_itcopyr +#define ZGEMM3M_ITCOPYI OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_itcopyi + +#define ZSYMM3M_ILCOPYB OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_ilcopyb +#define ZSYMM3M_IUCOPYB OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_iucopyb +#define ZSYMM3M_ILCOPYR OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_ilcopyr +#define ZSYMM3M_IUCOPYR OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_iucopyr +#define ZSYMM3M_ILCOPYI OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_ilcopyi +#define ZSYMM3M_IUCOPYI OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_iucopyi + +#define ZSYMM3M_OLCOPYB OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_olcopyb +#define ZSYMM3M_OUCOPYB OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_oucopyb +#define ZSYMM3M_OLCOPYR OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_olcopyr +#define ZSYMM3M_OUCOPYR OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_oucopyr +#define ZSYMM3M_OLCOPYI OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_olcopyi +#define ZSYMM3M_OUCOPYI OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_oucopyi + +#define ZHEMM3M_ILCOPYB OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_ilcopyb +#define ZHEMM3M_IUCOPYB OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_iucopyb +#define ZHEMM3M_ILCOPYR OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_ilcopyr +#define ZHEMM3M_IUCOPYR OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_iucopyr +#define ZHEMM3M_ILCOPYI OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_ilcopyi +#define ZHEMM3M_IUCOPYI OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_iucopyi + +#define ZHEMM3M_OLCOPYB OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_olcopyb +#define ZHEMM3M_OUCOPYB OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_oucopyb +#define ZHEMM3M_OLCOPYR OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_olcopyr +#define ZHEMM3M_OUCOPYR OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_oucopyr +#define ZHEMM3M_OLCOPYI OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_olcopyi +#define ZHEMM3M_OUCOPYI OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_oucopyi + +#define ZGEMM3M_KERNEL OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_kernel + +#define ZNEG_TCOPY OPENBLAS_DISPATCH(zneg) -> zneg_tcopy +#define ZLASWP_NCOPY OPENBLAS_DISPATCH(zlaswp) -> zlaswp_ncopy + +#define ZAXPBY_K OPENBLAS_DISPATCH(zaxpby) -> zaxpby_k + +#define ZOMATCOPY_K_CN OPENBLAS_DISPATCH(zomatcopy) -> zomatcopy_k_cn +#define ZOMATCOPY_K_RN OPENBLAS_DISPATCH(zomatcopy) -> zomatcopy_k_rn +#define ZOMATCOPY_K_CT OPENBLAS_DISPATCH(zomatcopy) -> zomatcopy_k_ct +#define ZOMATCOPY_K_RT OPENBLAS_DISPATCH(zomatcopy) -> zomatcopy_k_rt +#define ZOMATCOPY_K_CNC OPENBLAS_DISPATCH(zomatcopy) -> zomatcopy_k_cnc +#define ZOMATCOPY_K_RNC OPENBLAS_DISPATCH(zomatcopy) -> zomatcopy_k_rnc +#define ZOMATCOPY_K_CTC OPENBLAS_DISPATCH(zomatcopy) -> zomatcopy_k_ctc +#define ZOMATCOPY_K_RTC OPENBLAS_DISPATCH(zomatcopy) -> zomatcopy_k_rtc + +#define ZIMATCOPY_K_CN OPENBLAS_DISPATCH(zimatcopy) -> zimatcopy_k_cn +#define ZIMATCOPY_K_RN OPENBLAS_DISPATCH(zimatcopy) -> zimatcopy_k_rn +#define ZIMATCOPY_K_CT OPENBLAS_DISPATCH(zimatcopy) -> zimatcopy_k_ct +#define ZIMATCOPY_K_RT OPENBLAS_DISPATCH(zimatcopy) -> zimatcopy_k_rt +#define ZIMATCOPY_K_CNC OPENBLAS_DISPATCH(zimatcopy) -> zimatcopy_k_cnc +#define ZIMATCOPY_K_RNC OPENBLAS_DISPATCH(zimatcopy) -> zimatcopy_k_rnc +#define ZIMATCOPY_K_CTC OPENBLAS_DISPATCH(zimatcopy) -> zimatcopy_k_ctc +#define ZIMATCOPY_K_RTC OPENBLAS_DISPATCH(zimatcopy) -> zimatcopy_k_rtc + +#define ZGEADD_K OPENBLAS_DISPATCH(zgeadd) -> zgeadd_k + +#define ZGEMM_SMALL_MATRIX_PERMIT OPENBLAS_DISPATCH(zgemm) -> zgemm_small_matrix_permit #endif -#define ZGEMM_SMALL_KERNEL_NN FUNC_OFFSET(zgemm_small_kernel_nn) -#define ZGEMM_SMALL_KERNEL_NT FUNC_OFFSET(zgemm_small_kernel_nt) -#define ZGEMM_SMALL_KERNEL_NR FUNC_OFFSET(zgemm_small_kernel_nr) -#define ZGEMM_SMALL_KERNEL_NC FUNC_OFFSET(zgemm_small_kernel_nc) - -#define ZGEMM_SMALL_KERNEL_TN FUNC_OFFSET(zgemm_small_kernel_tn) -#define ZGEMM_SMALL_KERNEL_TT FUNC_OFFSET(zgemm_small_kernel_tt) -#define ZGEMM_SMALL_KERNEL_TR FUNC_OFFSET(zgemm_small_kernel_tr) -#define ZGEMM_SMALL_KERNEL_TC FUNC_OFFSET(zgemm_small_kernel_tc) - -#define ZGEMM_SMALL_KERNEL_RN FUNC_OFFSET(zgemm_small_kernel_rn) -#define ZGEMM_SMALL_KERNEL_RT FUNC_OFFSET(zgemm_small_kernel_rt) -#define ZGEMM_SMALL_KERNEL_RR FUNC_OFFSET(zgemm_small_kernel_rr) -#define ZGEMM_SMALL_KERNEL_RC FUNC_OFFSET(zgemm_small_kernel_rc) - -#define ZGEMM_SMALL_KERNEL_CN FUNC_OFFSET(zgemm_small_kernel_cn) -#define ZGEMM_SMALL_KERNEL_CT FUNC_OFFSET(zgemm_small_kernel_ct) -#define ZGEMM_SMALL_KERNEL_CR FUNC_OFFSET(zgemm_small_kernel_cr) -#define ZGEMM_SMALL_KERNEL_CC FUNC_OFFSET(zgemm_small_kernel_cc) - -#define ZGEMM_SMALL_KERNEL_B0_NN FUNC_OFFSET(zgemm_small_kernel_b0_nn) -#define ZGEMM_SMALL_KERNEL_B0_NT FUNC_OFFSET(zgemm_small_kernel_b0_nt) -#define ZGEMM_SMALL_KERNEL_B0_NR FUNC_OFFSET(zgemm_small_kernel_b0_nr) -#define ZGEMM_SMALL_KERNEL_B0_NC FUNC_OFFSET(zgemm_small_kernel_b0_nc) - -#define ZGEMM_SMALL_KERNEL_B0_TN FUNC_OFFSET(zgemm_small_kernel_b0_tn) -#define ZGEMM_SMALL_KERNEL_B0_TT FUNC_OFFSET(zgemm_small_kernel_b0_tt) -#define ZGEMM_SMALL_KERNEL_B0_TR FUNC_OFFSET(zgemm_small_kernel_b0_tr) -#define ZGEMM_SMALL_KERNEL_B0_TC FUNC_OFFSET(zgemm_small_kernel_b0_tc) - -#define ZGEMM_SMALL_KERNEL_B0_RN FUNC_OFFSET(zgemm_small_kernel_b0_rn) -#define ZGEMM_SMALL_KERNEL_B0_RT FUNC_OFFSET(zgemm_small_kernel_b0_rt) -#define ZGEMM_SMALL_KERNEL_B0_RR FUNC_OFFSET(zgemm_small_kernel_b0_rr) -#define ZGEMM_SMALL_KERNEL_B0_RC FUNC_OFFSET(zgemm_small_kernel_b0_rc) - -#define ZGEMM_SMALL_KERNEL_B0_CN FUNC_OFFSET(zgemm_small_kernel_b0_cn) -#define ZGEMM_SMALL_KERNEL_B0_CT FUNC_OFFSET(zgemm_small_kernel_b0_ct) -#define ZGEMM_SMALL_KERNEL_B0_CR FUNC_OFFSET(zgemm_small_kernel_b0_cr) -#define ZGEMM_SMALL_KERNEL_B0_CC FUNC_OFFSET(zgemm_small_kernel_b0_cc) +#define ZGEMM_SMALL_KERNEL_BASE OPENBLAS_DISPATCH_BASE(zgemm) +#define ZGEMM_SMALL_KERNEL_NN OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_nn) +#define ZGEMM_SMALL_KERNEL_NT OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_nt) +#define ZGEMM_SMALL_KERNEL_NR OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_nr) +#define ZGEMM_SMALL_KERNEL_NC OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_nc) + +#define ZGEMM_SMALL_KERNEL_TN OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_tn) +#define ZGEMM_SMALL_KERNEL_TT OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_tt) +#define ZGEMM_SMALL_KERNEL_TR OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_tr) +#define ZGEMM_SMALL_KERNEL_TC OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_tc) + +#define ZGEMM_SMALL_KERNEL_RN OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_rn) +#define ZGEMM_SMALL_KERNEL_RT OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_rt) +#define ZGEMM_SMALL_KERNEL_RR OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_rr) +#define ZGEMM_SMALL_KERNEL_RC OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_rc) + +#define ZGEMM_SMALL_KERNEL_CN OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_cn) +#define ZGEMM_SMALL_KERNEL_CT OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_ct) +#define ZGEMM_SMALL_KERNEL_CR OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_cr) +#define ZGEMM_SMALL_KERNEL_CC OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_cc) + +#define ZGEMM_SMALL_KERNEL_B0_NN OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_nn) +#define ZGEMM_SMALL_KERNEL_B0_NT OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_nt) +#define ZGEMM_SMALL_KERNEL_B0_NR OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_nr) +#define ZGEMM_SMALL_KERNEL_B0_NC OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_nc) + +#define ZGEMM_SMALL_KERNEL_B0_TN OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_tn) +#define ZGEMM_SMALL_KERNEL_B0_TT OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_tt) +#define ZGEMM_SMALL_KERNEL_B0_TR OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_tr) +#define ZGEMM_SMALL_KERNEL_B0_TC OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_tc) + +#define ZGEMM_SMALL_KERNEL_B0_RN OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_rn) +#define ZGEMM_SMALL_KERNEL_B0_RT OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_rt) +#define ZGEMM_SMALL_KERNEL_B0_RR OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_rr) +#define ZGEMM_SMALL_KERNEL_B0_RC OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_rc) + +#define ZGEMM_SMALL_KERNEL_B0_CN OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_cn) +#define ZGEMM_SMALL_KERNEL_B0_CT OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_ct) +#define ZGEMM_SMALL_KERNEL_B0_CR OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_cr) +#define ZGEMM_SMALL_KERNEL_B0_CC OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_cc) #define ZGEMM_NN zgemm_nn diff --git a/driver/others/CMakeLists.txt b/driver/others/CMakeLists.txt index b25337db13..e06b2adafa 100644 --- a/driver/others/CMakeLists.txt +++ b/driver/others/CMakeLists.txt @@ -51,6 +51,7 @@ GenerateNamedObjects("abs.c" "DOUBLE" "z_abs" 0 "" "" 1) GenerateNamedObjects("openblas_get_config.c;openblas_get_parallel.c" "" "" 0 "" "" 1) if (DYNAMIC_ARCH) + list(APPEND COMMON_SOURCES dispatch.c) if (ARM64) list(APPEND COMMON_SOURCES dynamic_arm64.c) elseif (POWER) diff --git a/driver/others/Makefile b/driver/others/Makefile index 52a893e7bf..a8e1428506 100644 --- a/driver/others/Makefile +++ b/driver/others/Makefile @@ -15,6 +15,7 @@ endif # COMMONOBJS += info.$(SUFFIX) ifeq ($(DYNAMIC_ARCH), 1) +COMMONOBJS += dispatch.$(SUFFIX) ifeq ($(ARCH),arm64) COMMONOBJS += dynamic_arm64.$(SUFFIX) else @@ -133,6 +134,9 @@ dynamic%$(SUFFIX) : dynamic%c dynamic%$(PSUFFIX) : dynamic%c $(CC) $(PFLAGS) -c $< -o $(@F) +dispatch.$(SUFFIX) : dispatch.c + $(CC) $(CFLAGS) -c $< -o $(@F) + parameter.$(SUFFIX) : parameter.c ../../param.h $(CC) $(CFLAGS) -c $< -o $(@F) diff --git a/driver/others/dispatch.c b/driver/others/dispatch.c new file mode 100644 index 0000000000..f2241298af --- /dev/null +++ b/driver/others/dispatch.c @@ -0,0 +1,985 @@ +/***************************************************************************** +Copyright (c) 2026, The OpenBLAS Project +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. Neither the name of the OpenBLAS project nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +/* The arrays of per-core kernel tables that OPENBLAS_DISPATCH(group) + indexes (see common_param.h), one per kernel group. */ + +#include "common.h" + +#define DISPATCH_EXTERN(core, group) \ + extern const openblas_##group##_dispatch_t openblas_##group##_dispatch_##core; +#define DISPATCH_ENTRY(core, group) \ + [OPENBLAS_CORE_##core] = &openblas_##group##_dispatch_##core, + +/* openblas__dispatch[OPENBLAS_CORE_] points to + openblas__dispatch_, which kernel/setparam-ref.c defines for + that core. */ +#define DEFINE_DISPATCH_TABLE(group) \ + OPENBLAS_CORE_LIST(DISPATCH_EXTERN, group) \ + const openblas_##group##_dispatch_t *const openblas_##group##_dispatch[OPENBLAS_NUM_CORES] = { \ + OPENBLAS_CORE_LIST(DISPATCH_ENTRY, group) \ + }; + +#if BUILD_HFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(shgemm) +#endif + +#if BUILD_HFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(shgemv) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbstobf16) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbdtobf16) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbf16tos) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(dbf16tod) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbamax) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbamin) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbmax) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbmin) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(isbamax) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(isbamin) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(isbmax) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(isbmin) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbnrm2) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbasum) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbsum) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbcopy) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbdot) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(dsbdot) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbrot) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbrotm) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(bscal) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbaxpy) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbscal) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbswap) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(bgemv) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbgemv) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbger) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbsymv) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(bgemm) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbgemm) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbtrsm) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbtrmm) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbsymm) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbneg) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sblaswp) +#endif + +#if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +DEFINE_DISPATCH_TABLE(samax) +#endif + +#if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +DEFINE_DISPATCH_TABLE(samin) +#endif + +#if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +DEFINE_DISPATCH_TABLE(smax) +#endif + +#if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +DEFINE_DISPATCH_TABLE(smin) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE ==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(isamax) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(isamin) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(ismax) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(ismin) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(snrm2) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(sasum) +#endif + +#if (BUILD_SINGLE==1) +DEFINE_DISPATCH_TABLE(ssum) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(scopy) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(sdot) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(srot) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(srotm) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(saxpy) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(sscal) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(sswap) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(sgemv) +#endif + +#if (BUILD_SINGLE==1) +DEFINE_DISPATCH_TABLE(sger) +#endif + +#if (BUILD_SINGLE==1) +DEFINE_DISPATCH_TABLE(ssymv) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(sgemm) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(ssymm) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(strmm) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +#ifdef ARCH_ARM64 +DEFINE_DISPATCH_TABLE(ssyrk) +#endif +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +#ifdef ARCH_ARM64 +DEFINE_DISPATCH_TABLE(ssyr2k) +#endif +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(strsm) +#endif + +#if (BUILD_SINGLE==1) +DEFINE_DISPATCH_TABLE(sneg) +#endif + +#if (BUILD_SINGLE==1) +DEFINE_DISPATCH_TABLE(slaswp) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(damax) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(damin) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(dmax) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(dmin) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(idamax) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(idamin) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(idmax) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(idmin) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(dnrm2) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(dasum) +#endif + +#if (BUILD_DOUBLE==1) +DEFINE_DISPATCH_TABLE(dsum) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(dcopy) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(ddot) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) +DEFINE_DISPATCH_TABLE(dsdot) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(drot) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(drotm) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(daxpy) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(dscal) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(dswap) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(dgemv) +#endif + +#if (BUILD_DOUBLE==1) +DEFINE_DISPATCH_TABLE(dger) +#endif + +#if (BUILD_DOUBLE==1) +DEFINE_DISPATCH_TABLE(dsymv) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(dgemm) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(dsymm) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(dtrmm) +#endif + +#if (BUILD_DOUBLE==1) +DEFINE_DISPATCH_TABLE(dtrsm) +#endif + +#if (BUILD_DOUBLE==1) +DEFINE_DISPATCH_TABLE(dneg) +#endif + +#if (BUILD_DOUBLE==1) +DEFINE_DISPATCH_TABLE(dlaswp) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qamax) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qamin) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qmax) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qmin) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(iqamax) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(iqamin) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(iqmax) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(iqmin) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qnrm2) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qasum) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qsum) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qcopy) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qdot) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qrot) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qrotm) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qaxpy) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qscal) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qswap) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qgemv) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qger) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qsymv) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qgemm) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qtrsm) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qtrmm) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qsymm) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qneg) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qlaswp) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(camax) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(camin) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(icamax) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(icamin) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cnrm2) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(casum) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(csum) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(ccopy) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cdotu) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cdotc) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(csrot) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(caxpy) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(caxpyc) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cscal) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cswap) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cgemv) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cgeru) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cgerc) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cgerv) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cgerd) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(csymv) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(chemv) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cgemm) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(csymm) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(ctrmm) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(ctrsm) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(chemm) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cgemm3m) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(csymm3m) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(chemm3m) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cneg) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(claswp) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zamax) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zamin) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(izamax) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(izamin) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(znrm2) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zasum) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zsum) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zcopy) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zdotu) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zdotc) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zdrot) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zaxpy) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zaxpyc) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zscal) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zswap) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zgemv) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zgeru) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zgerc) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zgerv) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zgerd) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zsymv) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zhemv) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zgemm) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zsymm) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(ztrmm) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(ztrsm) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zhemm) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zgemm3m) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zsymm3m) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zhemm3m) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zneg) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zlaswp) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xamax) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xamin) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(ixamax) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(ixamin) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xnrm2) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xasum) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xsum) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xcopy) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xdotu) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xdotc) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xqrot) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xaxpy) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xaxpyc) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xscal) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xswap) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xgemv) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xgeru) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xgerc) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xgerv) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xgerd) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xsymv) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xhemv) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xgemm) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xtrsm) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xtrmm) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xsymm) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xhemm) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xgemm3m) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xsymm3m) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xhemm3m) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xneg) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xlaswp) +#endif + +#if (BUILD_SINGLE==1) +DEFINE_DISPATCH_TABLE(saxpby) +#endif + +#if (BUILD_DOUBLE==1) +DEFINE_DISPATCH_TABLE(daxpby) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(caxpby) +#endif + +#if (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(zaxpby) +#endif + +#if (BUILD_SINGLE==1) +DEFINE_DISPATCH_TABLE(somatcopy) +#endif + +#if (BUILD_DOUBLE==1) +DEFINE_DISPATCH_TABLE(domatcopy) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(comatcopy) +#endif + +#if (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(zomatcopy) +#endif + +#if (BUILD_SINGLE==1) +DEFINE_DISPATCH_TABLE(simatcopy) +#endif + +#if (BUILD_DOUBLE==1) +DEFINE_DISPATCH_TABLE(dimatcopy) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cimatcopy) +#endif + +#if (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(zimatcopy) +#endif + +#if (BUILD_SINGLE==1) +DEFINE_DISPATCH_TABLE(sgeadd) +#endif + +#if (BUILD_DOUBLE==1) +DEFINE_DISPATCH_TABLE(dgeadd) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cgeadd) +#endif + +#if (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(zgeadd) +#endif diff --git a/interface/gemm.c b/interface/gemm.c index f1fd49cb15..8572b052bf 100644 --- a/interface/gemm.c +++ b/interface/gemm.c @@ -128,7 +128,7 @@ static int (*gemm[])(blas_arg_t *, BLASLONG *, BLASLONG *, IFLOAT *, IFLOAT *, B #ifndef DYNAMIC_ARCH #define SMALL_KERNEL_ADDR(table, idx) ((void *)(table[idx])) #else -#define SMALL_KERNEL_ADDR(table, idx) ((void *)(*(uintptr_t *)((char *)gotoblas + (size_t)(table[idx])))) +#define SMALL_KERNEL_ADDR(table, idx) ((void *)(*(uintptr_t *)(GEMM_SMALL_KERNEL_BASE + (size_t)(table[idx])))) #endif diff --git a/interface/gemm_batch.c b/interface/gemm_batch.c index 7a8789e820..b0f2f5adc7 100644 --- a/interface/gemm_batch.c +++ b/interface/gemm_batch.c @@ -75,7 +75,7 @@ static int (*gemm[])(blas_arg_t *, BLASLONG *, BLASLONG *, IFLOAT *, IFLOAT *, B #ifndef DYNAMIC_ARCH #define SMALL_KERNEL_ADDR(table, idx) ((void *)(table[idx])) #else -#define SMALL_KERNEL_ADDR(table, idx) ((void *)(*(uintptr_t *)((char *)gotoblas + (size_t)(table[idx])))) +#define SMALL_KERNEL_ADDR(table, idx) ((void *)(*(uintptr_t *)(GEMM_SMALL_KERNEL_BASE + (size_t)(table[idx])))) #endif diff --git a/interface/gemm_batch_strided.c b/interface/gemm_batch_strided.c index 026eeb50e2..92d4326506 100644 --- a/interface/gemm_batch_strided.c +++ b/interface/gemm_batch_strided.c @@ -75,7 +75,7 @@ static int (*gemm[])(blas_arg_t *, BLASLONG *, BLASLONG *, IFLOAT *, IFLOAT *, B #ifndef DYNAMIC_ARCH #define SMALL_KERNEL_ADDR(table, idx) ((void *)(table[idx])) #else -#define SMALL_KERNEL_ADDR(table, idx) ((void *)(*(uintptr_t *)((char *)gotoblas + (size_t)(table[idx])))) +#define SMALL_KERNEL_ADDR(table, idx) ((void *)(*(uintptr_t *)(GEMM_SMALL_KERNEL_BASE + (size_t)(table[idx])))) #endif diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 6ae74322dd..0987456210 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -1601,7 +1601,7 @@ endif () add_library(kernel${TSUFFIX} OBJECT ${OPENBLAS_SRC}) set_target_properties(kernel${TSUFFIX} PROPERTIES COMPILE_FLAGS "${KERNEL_DEFINITIONS}") get_target_property(KERNEL_INCLUDE_DIRECTORIES kernel${TSUFFIX} INCLUDE_DIRECTORIES) - set_target_properties(kernel${TSUFFIX} PROPERTIES INCLUDE_DIRECTORIES "${KERNEL_INCLUDE_DIRECTORIES};${TARGET_CONF_DIR}") + set_target_properties(kernel${TSUFFIX} PROPERTIES INCLUDE_DIRECTORIES "${KERNEL_INCLUDE_DIRECTORIES};${TARGET_CONF_DIR};${PROJECT_BINARY_DIR}") if (USE_GEMM3M) target_compile_definitions(kernel${TSUFFIX} PRIVATE USE_GEMM3M) endif() diff --git a/kernel/setparam-ref.c b/kernel/setparam-ref.c index 6993062ed2..5478ab0c61 100644 --- a/kernel/setparam-ref.c +++ b/kernel/setparam-ref.c @@ -51,941 +51,2778 @@ extern char* gotoblas_corename(void); static void init_parameter(void); gotoblas_t TABLE_NAME = { - DTB_DEFAULT_ENTRIES, + .core = OPENBLAS_CORETS, + .dtb_entries = DTB_DEFAULT_ENTRIES, + .switch_ratio = SWITCH_RATIO, + .divide_rate = GEMM_DIVIDE_RATE, + .divide_limit = GEMM_DIVIDE_LIMIT, + .preferred_size = GEMM_PREFERRED_SIZE, + .offsetA = GEMM_DEFAULT_OFFSET_A, + .offsetB = GEMM_DEFAULT_OFFSET_B, + .align = GEMM_DEFAULT_ALIGN, +#ifdef BUILD_HFLOAT16 + .shgemm_p = 0, + .shgemm_q = 0, + .shgemm_r = 0, + .shgemm_unroll_m = SHGEMM_DEFAULT_UNROLL_M, + .shgemm_unroll_n = SHGEMM_DEFAULT_UNROLL_N, +#ifdef SHGEMM_DEFAULT_UNROLL_MN + .shgemm_unroll_mn = SHGEMM_DEFAULT_UNROLL_MN, +#else + .shgemm_unroll_mn = MAX(SHGEMM_DEFAULT_UNROLL_M, SHGEMM_DEFAULT_UNROLL_N), +#endif +#endif +#ifdef BUILD_BFLOAT16 + .bgemm_p = 0, + .bgemm_q = 0, + .bgemm_r = 0, + .bgemm_unroll_m = BGEMM_DEFAULT_UNROLL_M, + .bgemm_unroll_n = BGEMM_DEFAULT_UNROLL_N, +#ifdef BGEMM_DEFAULT_UNROLL_MN + .bgemm_unroll_mn = BGEMM_DEFAULT_UNROLL_MN, +#else + .bgemm_unroll_mn = MAX(BGEMM_DEFAULT_UNROLL_M, BGEMM_DEFAULT_UNROLL_N), +#endif + .bgemm_align_k = BGEMM_ALIGN_K, + .sbgemm_p = 0, + .sbgemm_q = 0, + .sbgemm_r = 0, + .sbgemm_unroll_m = SBGEMM_DEFAULT_UNROLL_M, + .sbgemm_unroll_n = SBGEMM_DEFAULT_UNROLL_N, +#ifdef SBGEMM_DEFAULT_UNROLL_MN + .sbgemm_unroll_mn = SBGEMM_DEFAULT_UNROLL_MN, +#else + .sbgemm_unroll_mn = MAX(SBGEMM_DEFAULT_UNROLL_M, SBGEMM_DEFAULT_UNROLL_N), +#endif + .sbgemm_align_k = SBGEMM_ALIGN_K, + .need_amxtile_permission = 0, // need_amxtile_permission +#endif +#if ( BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) || (BUILD_COMPLEX16==1) + .sgemm_p = 0, + .sgemm_q = 0, + .sgemm_r = 0, + .sgemm_unroll_m = SGEMM_DEFAULT_UNROLL_M, + .sgemm_unroll_n = SGEMM_DEFAULT_UNROLL_N, +#ifdef SGEMM_DEFAULT_UNROLL_MN + .sgemm_unroll_mn = SGEMM_DEFAULT_UNROLL_MN, +#else + .sgemm_unroll_mn = MAX(SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N), +#endif +#endif +#ifdef HAVE_EXCLUSIVE_CACHE + .exclusive_cache = 1, +#else + .exclusive_cache = 0, +#endif +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) + .dgemm_p = 0, + .dgemm_q = 0, + .dgemm_r = 0, + .dgemm_unroll_m = DGEMM_DEFAULT_UNROLL_M, + .dgemm_unroll_n = DGEMM_DEFAULT_UNROLL_N, +#ifdef DGEMM_DEFAULT_UNROLL_MN + .dgemm_unroll_mn = DGEMM_DEFAULT_UNROLL_MN, +#else + .dgemm_unroll_mn = MAX(DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N), +#endif +#endif +#ifdef EXPRECISION + .qgemm_p = 0, + .qgemm_q = 0, + .qgemm_r = 0, + .qgemm_unroll_m = QGEMM_DEFAULT_UNROLL_M, + .qgemm_unroll_n = QGEMM_DEFAULT_UNROLL_N, + .qgemm_unroll_mn = MAX(QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N), +#endif +#if (BUILD_COMPLEX) + .cgemm_p = 0, + .cgemm_q = 0, + .cgemm_r = 0, + .cgemm_unroll_m = CGEMM_DEFAULT_UNROLL_M, + .cgemm_unroll_n = CGEMM_DEFAULT_UNROLL_N, +#ifdef CGEMM_DEFAULT_UNROLL_MN + .cgemm_unroll_mn = CGEMM_DEFAULT_UNROLL_MN, +#else + .cgemm_unroll_mn = MAX(CGEMM_DEFAULT_UNROLL_M, CGEMM_DEFAULT_UNROLL_N), +#endif +#endif +#if (BUILD_COMPLEX) + .cgemm3m_p = 0, + .cgemm3m_q = 0, + .cgemm3m_r = 0, +#if (USE_GEMM3M) +#ifdef CGEMM3M_DEFAULT_UNROLL_M + .cgemm3m_unroll_m = CGEMM3M_DEFAULT_UNROLL_M, + .cgemm3m_unroll_n = CGEMM3M_DEFAULT_UNROLL_N, + .cgemm3m_unroll_mn = MAX(CGEMM3M_DEFAULT_UNROLL_M, CGEMM3M_DEFAULT_UNROLL_N), +#else + .cgemm3m_unroll_m = SGEMM_DEFAULT_UNROLL_M, + .cgemm3m_unroll_n = SGEMM_DEFAULT_UNROLL_N, + .cgemm3m_unroll_mn = MAX(SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N), +#endif +#else + .cgemm3m_unroll_m = 0, + .cgemm3m_unroll_n = 0, + .cgemm3m_unroll_mn = 0, +#endif +#endif +#if BUILD_COMPLEX16 == 1 + .zgemm_p = 0, + .zgemm_q = 0, + .zgemm_r = 0, + .zgemm_unroll_m = ZGEMM_DEFAULT_UNROLL_M, + .zgemm_unroll_n = ZGEMM_DEFAULT_UNROLL_N, +#ifdef ZGEMM_DEFAULT_UNROLL_MN + .zgemm_unroll_mn = ZGEMM_DEFAULT_UNROLL_MN, +#else + .zgemm_unroll_mn = MAX(ZGEMM_DEFAULT_UNROLL_M, ZGEMM_DEFAULT_UNROLL_N), +#endif + .zgemm3m_p = 0, + .zgemm3m_q = 0, + .zgemm3m_r = 0, +#if (USE_GEMM3M) +#ifdef ZGEMM3M_DEFAULT_UNROLL_M + .zgemm3m_unroll_m = ZGEMM3M_DEFAULT_UNROLL_M, + .zgemm3m_unroll_n = ZGEMM3M_DEFAULT_UNROLL_N, + .zgemm3m_unroll_mn = MAX(ZGEMM3M_DEFAULT_UNROLL_M, ZGEMM3M_DEFAULT_UNROLL_N), +#else + .zgemm3m_unroll_m = DGEMM_DEFAULT_UNROLL_M, + .zgemm3m_unroll_n = DGEMM_DEFAULT_UNROLL_N, + .zgemm3m_unroll_mn = MAX(DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N), +#endif +#else + .zgemm3m_unroll_m = 0, + .zgemm3m_unroll_n = 0, + .zgemm3m_unroll_mn = 0, +#endif +#endif +#ifdef EXPRECISION + .xgemm_p = 0, + .xgemm_q = 0, + .xgemm_r = 0, + .xgemm_unroll_m = XGEMM_DEFAULT_UNROLL_M, + .xgemm_unroll_n = XGEMM_DEFAULT_UNROLL_N, + .xgemm_unroll_mn = MAX(XGEMM_DEFAULT_UNROLL_M, XGEMM_DEFAULT_UNROLL_N), + .xgemm3m_p = 0, + .xgemm3m_q = 0, + .xgemm3m_r = 0, +#if (USE_GEMM3M) + .xgemm3m_unroll_m = QGEMM_DEFAULT_UNROLL_M, + .xgemm3m_unroll_n = QGEMM_DEFAULT_UNROLL_N, + .xgemm3m_unroll_mn = MAX(QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N), +#else + .xgemm3m_unroll_m = 0, + .xgemm3m_unroll_n = 0, + .xgemm3m_unroll_mn = 0, +#endif +#endif + .init = init_parameter, + .snum_opt = SNUMOPT, + .dnum_opt = DNUMOPT, + .qnum_opt = QNUMOPT, +}; + +#if BUILD_HFLOAT16 == 1 +const openblas_shgemm_dispatch_t openblas_shgemm_dispatchTS = { + .shgemm_kernel = shgemm_kernelTS, + .shgemm_beta = shgemm_betaTS, +#if SHGEMM_DEFAULT_UNROLL_M != SHGEMM_DEFAULT_UNROLL_N + .shgemm_incopy = shgemm_incopyTS, + .shgemm_itcopy = shgemm_itcopyTS, +#else + .shgemm_incopy = shgemm_oncopyTS, + .shgemm_itcopy = shgemm_otcopyTS, +#endif + .shgemm_oncopy = shgemm_oncopyTS, + .shgemm_otcopy = shgemm_otcopyTS, +}; +#endif + +#if BUILD_HFLOAT16 == 1 +const openblas_shgemv_dispatch_t openblas_shgemv_dispatchTS = { + .shgemv_n = shgemv_nTS, + .shgemv_t = shgemv_tTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbstobf16_dispatch_t openblas_sbstobf16_dispatchTS = { + .sbstobf16_k = sbstobf16_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbdtobf16_dispatch_t openblas_sbdtobf16_dispatchTS = { + .sbdtobf16_k = sbdtobf16_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbf16tos_dispatch_t openblas_sbf16tos_dispatchTS = { + .sbf16tos_k = sbf16tos_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_dbf16tod_dispatch_t openblas_dbf16tod_dispatchTS = { + .dbf16tod_k = dbf16tod_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbamax_dispatch_t openblas_sbamax_dispatchTS = { + .sbamax_k = samax_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbamin_dispatch_t openblas_sbamin_dispatchTS = { + .sbamin_k = samin_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbmax_dispatch_t openblas_sbmax_dispatchTS = { + .sbmax_k = smax_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbmin_dispatch_t openblas_sbmin_dispatchTS = { + .sbmin_k = smin_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_isbamax_dispatch_t openblas_isbamax_dispatchTS = { + .isbamax_k = isamax_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_isbamin_dispatch_t openblas_isbamin_dispatchTS = { + .isbamin_k = isamin_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_isbmax_dispatch_t openblas_isbmax_dispatchTS = { + .isbmax_k = ismax_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_isbmin_dispatch_t openblas_isbmin_dispatchTS = { + .isbmin_k = ismin_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbnrm2_dispatch_t openblas_sbnrm2_dispatchTS = { + .sbnrm2_k = snrm2_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbasum_dispatch_t openblas_sbasum_dispatchTS = { + .sbasum_k = sasum_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbsum_dispatch_t openblas_sbsum_dispatchTS = { + .sbsum_k = ssum_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbcopy_dispatch_t openblas_sbcopy_dispatchTS = { + .sbcopy_k = scopy_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbdot_dispatch_t openblas_sbdot_dispatchTS = { + .sbdot_k = sbdot_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_dsbdot_dispatch_t openblas_dsbdot_dispatchTS = { + .dsbdot_k = dsdot_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbrot_dispatch_t openblas_sbrot_dispatchTS = { + .sbrot_k = srot_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbrotm_dispatch_t openblas_sbrotm_dispatchTS = { + .sbrotm_k = srotm_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_bscal_dispatch_t openblas_bscal_dispatchTS = { + .bscal_k = bscal_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbaxpy_dispatch_t openblas_sbaxpy_dispatchTS = { + .sbaxpy_k = saxpy_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbscal_dispatch_t openblas_sbscal_dispatchTS = { + .sbscal_k = sscal_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbswap_dispatch_t openblas_sbswap_dispatchTS = { + .sbswap_k = sswap_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_bgemv_dispatch_t openblas_bgemv_dispatchTS = { + .bgemv_n = bgemv_nTS, + .bgemv_t = bgemv_tTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbgemv_dispatch_t openblas_sbgemv_dispatchTS = { + .sbgemv_n = sbgemv_nTS, + .sbgemv_t = sbgemv_tTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbger_dispatch_t openblas_sbger_dispatchTS = { + .sbger_k = sger_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbsymv_dispatch_t openblas_sbsymv_dispatchTS = { + .sbsymv_L = ssymv_LTS, + .sbsymv_U = ssymv_UTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_bgemm_dispatch_t openblas_bgemm_dispatchTS = { + .bgemm_kernel = bgemm_kernelTS, + .bgemm_beta = bgemm_betaTS, +#if BGEMM_DEFAULT_UNROLL_M != BGEMM_DEFAULT_UNROLL_N + .bgemm_incopy = bgemm_incopyTS, + .bgemm_itcopy = bgemm_itcopyTS, +#else + .bgemm_incopy = bgemm_oncopyTS, + .bgemm_itcopy = bgemm_otcopyTS, +#endif + .bgemm_oncopy = bgemm_oncopyTS, + .bgemm_otcopy = bgemm_otcopyTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbgemm_dispatch_t openblas_sbgemm_dispatchTS = { + .sbgemm_kernel = sbgemm_kernelTS, + .sbgemm_beta = sbgemm_betaTS, +#if SBGEMM_DEFAULT_UNROLL_M != SBGEMM_DEFAULT_UNROLL_N + .sbgemm_incopy = sbgemm_incopyTS, + .sbgemm_itcopy = sbgemm_itcopyTS, +#else + .sbgemm_incopy = sbgemm_oncopyTS, + .sbgemm_itcopy = sbgemm_otcopyTS, +#endif + .sbgemm_oncopy = sbgemm_oncopyTS, + .sbgemm_otcopy = sbgemm_otcopyTS, +#ifdef SMALL_MATRIX_OPT + .sbgemm_small_matrix_permit = sbgemm_small_matrix_permitTS, + .sbgemm_small_kernel_nn = sbgemm_small_kernel_nnTS, + .sbgemm_small_kernel_nt = sbgemm_small_kernel_ntTS, + .sbgemm_small_kernel_tn = sbgemm_small_kernel_tnTS, + .sbgemm_small_kernel_tt = sbgemm_small_kernel_ttTS, + .sbgemm_small_kernel_b0_nn = sbgemm_small_kernel_b0_nnTS, + .sbgemm_small_kernel_b0_nt = sbgemm_small_kernel_b0_ntTS, + .sbgemm_small_kernel_b0_tn = sbgemm_small_kernel_b0_tnTS, + .sbgemm_small_kernel_b0_tt = sbgemm_small_kernel_b0_ttTS, +#endif +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbtrsm_dispatch_t openblas_sbtrsm_dispatchTS = { + .sbtrsm_kernel_LN = strsm_kernel_LNTS, + .sbtrsm_kernel_LT = strsm_kernel_LTTS, + .sbtrsm_kernel_RN = strsm_kernel_RNTS, + .sbtrsm_kernel_RT = strsm_kernel_RTTS, +#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N + .sbtrsm_iunucopy = strsm_iunucopyTS, + .sbtrsm_iunncopy = strsm_iunncopyTS, + .sbtrsm_iutucopy = strsm_iutucopyTS, + .sbtrsm_iutncopy = strsm_iutncopyTS, + .sbtrsm_ilnucopy = strsm_ilnucopyTS, + .sbtrsm_ilnncopy = strsm_ilnncopyTS, + .sbtrsm_iltucopy = strsm_iltucopyTS, + .sbtrsm_iltncopy = strsm_iltncopyTS, +#else + .sbtrsm_iunucopy = strsm_ounucopyTS, + .sbtrsm_iunncopy = strsm_ounncopyTS, + .sbtrsm_iutucopy = strsm_outucopyTS, + .sbtrsm_iutncopy = strsm_outncopyTS, + .sbtrsm_ilnucopy = strsm_olnucopyTS, + .sbtrsm_ilnncopy = strsm_olnncopyTS, + .sbtrsm_iltucopy = strsm_oltucopyTS, + .sbtrsm_iltncopy = strsm_oltncopyTS, +#endif + .sbtrsm_ounucopy = strsm_ounucopyTS, + .sbtrsm_ounncopy = strsm_ounncopyTS, + .sbtrsm_outucopy = strsm_outucopyTS, + .sbtrsm_outncopy = strsm_outncopyTS, + .sbtrsm_olnucopy = strsm_olnucopyTS, + .sbtrsm_olnncopy = strsm_olnncopyTS, + .sbtrsm_oltucopy = strsm_oltucopyTS, + .sbtrsm_oltncopy = strsm_oltncopyTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbtrmm_dispatch_t openblas_sbtrmm_dispatchTS = { + .sbtrmm_kernel_RN = strmm_kernel_RNTS, + .sbtrmm_kernel_RT = strmm_kernel_RTTS, + .sbtrmm_kernel_LN = strmm_kernel_LNTS, + .sbtrmm_kernel_LT = strmm_kernel_LTTS, +#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N + .sbtrmm_iunucopy = strmm_iunucopyTS, + .sbtrmm_iunncopy = strmm_iunncopyTS, + .sbtrmm_iutucopy = strmm_iutucopyTS, + .sbtrmm_iutncopy = strmm_iutncopyTS, + .sbtrmm_ilnucopy = strmm_ilnucopyTS, + .sbtrmm_ilnncopy = strmm_ilnncopyTS, + .sbtrmm_iltucopy = strmm_iltucopyTS, + .sbtrmm_iltncopy = strmm_iltncopyTS, +#else + .sbtrmm_iunucopy = strmm_ounucopyTS, + .sbtrmm_iunncopy = strmm_ounncopyTS, + .sbtrmm_iutucopy = strmm_outucopyTS, + .sbtrmm_iutncopy = strmm_outncopyTS, + .sbtrmm_ilnucopy = strmm_olnucopyTS, + .sbtrmm_ilnncopy = strmm_olnncopyTS, + .sbtrmm_iltucopy = strmm_oltucopyTS, + .sbtrmm_iltncopy = strmm_oltncopyTS, +#endif + .sbtrmm_ounucopy = strmm_ounucopyTS, + .sbtrmm_ounncopy = strmm_ounncopyTS, + .sbtrmm_outucopy = strmm_outucopyTS, + .sbtrmm_outncopy = strmm_outncopyTS, + .sbtrmm_olnucopy = strmm_olnucopyTS, + .sbtrmm_olnncopy = strmm_olnncopyTS, + .sbtrmm_oltucopy = strmm_oltucopyTS, + .sbtrmm_oltncopy = strmm_oltncopyTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbsymm_dispatch_t openblas_sbsymm_dispatchTS = { +#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N + .sbsymm_iutcopy = ssymm_iutcopyTS, + .sbsymm_iltcopy = ssymm_iltcopyTS, +#else + .sbsymm_iutcopy = ssymm_outcopyTS, + .sbsymm_iltcopy = ssymm_oltcopyTS, +#endif + .sbsymm_outcopy = ssymm_outcopyTS, + .sbsymm_oltcopy = ssymm_oltcopyTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbneg_dispatch_t openblas_sbneg_dispatchTS = { +#ifndef NO_LAPACK + .sbneg_tcopy = sneg_tcopyTS, +#else + .sbneg_tcopy = NULL, +#endif +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sblaswp_dispatch_t openblas_sblaswp_dispatchTS = { +#ifndef NO_LAPACK + .sblaswp_ncopy = slaswp_ncopyTS, +#else + .sblaswp_ncopy = NULL, +#endif +}; +#endif + +#if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +const openblas_samax_dispatch_t openblas_samax_dispatchTS = { + .samax_k = samax_kTS, +}; +#endif + +#if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +const openblas_samin_dispatch_t openblas_samin_dispatchTS = { + .samin_k = samin_kTS, +}; +#endif + +#if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +const openblas_smax_dispatch_t openblas_smax_dispatchTS = { + .smax_k = smax_kTS, +}; +#endif + +#if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +const openblas_smin_dispatch_t openblas_smin_dispatchTS = { + .smin_k = smin_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE ==1) || (BUILD_COMPLEX==1) +const openblas_isamax_dispatch_t openblas_isamax_dispatchTS = { + .isamax_k = isamax_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +const openblas_isamin_dispatch_t openblas_isamin_dispatchTS = { + .isamin_k = isamin_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +const openblas_ismax_dispatch_t openblas_ismax_dispatchTS = { + .ismax_k = ismax_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +const openblas_ismin_dispatch_t openblas_ismin_dispatchTS = { + .ismin_k = ismin_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +const openblas_snrm2_dispatch_t openblas_snrm2_dispatchTS = { + .snrm2_k = snrm2_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +const openblas_sasum_dispatch_t openblas_sasum_dispatchTS = { + .sasum_k = sasum_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) +const openblas_ssum_dispatch_t openblas_ssum_dispatchTS = { + .ssum_k = ssum_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +const openblas_scopy_dispatch_t openblas_scopy_dispatchTS = { + .scopy_k = scopy_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +const openblas_sdot_dispatch_t openblas_sdot_dispatchTS = { + .sdot_k = sdot_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +const openblas_srot_dispatch_t openblas_srot_dispatchTS = { + .srot_k = srot_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +const openblas_srotm_dispatch_t openblas_srotm_dispatchTS = { + .srotm_k = srotm_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +const openblas_saxpy_dispatch_t openblas_saxpy_dispatchTS = { + .saxpy_k = saxpy_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) || (BUILD_COMPLEX16==1) +const openblas_sscal_dispatch_t openblas_sscal_dispatchTS = { + .sscal_k = sscal_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +const openblas_sswap_dispatch_t openblas_sswap_dispatchTS = { + .sswap_k = sswap_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +const openblas_sgemv_dispatch_t openblas_sgemv_dispatchTS = { + .sgemv_n = sgemv_nTS, + .sgemv_t = sgemv_tTS, +}; +#endif + +#if (BUILD_SINGLE==1) +const openblas_sger_dispatch_t openblas_sger_dispatchTS = { + .sger_k = sger_kTS, +}; +#endif - SWITCH_RATIO, +#if (BUILD_SINGLE==1) +const openblas_ssymv_dispatch_t openblas_ssymv_dispatchTS = { + .ssymv_L = ssymv_LTS, + .ssymv_U = ssymv_UTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +const openblas_sgemm_dispatch_t openblas_sgemm_dispatchTS = { +#ifdef ARCH_X86_64 + .sgemm_direct = sgemm_directTS, + .sgemm_direct_performant = sgemm_direct_performantTS, +#endif +#ifdef ARCH_ARM64 + .sgemm_direct = sgemm_directTS, + .sgemm_direct_performant = sgemm_direct_performantTS, + .sgemm_direct_alpha_beta = sgemm_direct_alpha_betaTS, +#ifdef HAVE_SME + .sme_sgemm_kernel = sme_sgemm_kernelTS, +#else + .sme_sgemm_kernel = NULL, +#endif +#endif + .sgemm_kernel = sgemm_kernelTS, + .sgemm_beta = sgemm_betaTS, +#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N + .sgemm_incopy = sgemm_incopyTS, + .sgemm_itcopy = sgemm_itcopyTS, +#else + .sgemm_incopy = sgemm_oncopyTS, + .sgemm_itcopy = sgemm_otcopyTS, +#endif + .sgemm_oncopy = sgemm_oncopyTS, + .sgemm_otcopy = sgemm_otcopyTS, +#ifdef SMALL_MATRIX_OPT + .sgemm_small_matrix_permit = sgemm_small_matrix_permitTS, + .sgemm_small_kernel_nn = sgemm_small_kernel_nnTS, + .sgemm_small_kernel_nt = sgemm_small_kernel_ntTS, + .sgemm_small_kernel_tn = sgemm_small_kernel_tnTS, + .sgemm_small_kernel_tt = sgemm_small_kernel_ttTS, + .sgemm_small_kernel_b0_nn = sgemm_small_kernel_b0_nnTS, + .sgemm_small_kernel_b0_nt = sgemm_small_kernel_b0_ntTS, + .sgemm_small_kernel_b0_tn = sgemm_small_kernel_b0_tnTS, + .sgemm_small_kernel_b0_tt = sgemm_small_kernel_b0_ttTS, +#endif +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +const openblas_ssymm_dispatch_t openblas_ssymm_dispatchTS = { +#ifdef ARCH_ARM64 + .ssymm_direct_alpha_betaLU = ssymm_direct_alpha_betaLUTS, + .ssymm_direct_alpha_betaLL = ssymm_direct_alpha_betaLLTS, +#endif + .ssymm_kernel = ssymm_kernelTS, +#if (BUILD_SINGLE==1) + .ssymm_incopy = ssymm_incopyTS, + .ssymm_itcopy = ssymm_itcopyTS, +#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N + .ssymm_iutcopy = ssymm_iutcopyTS, + .ssymm_iltcopy = ssymm_iltcopyTS, +#else + .ssymm_iutcopy = ssymm_outcopyTS, + .ssymm_iltcopy = ssymm_oltcopyTS, +#endif + .ssymm_outcopy = ssymm_outcopyTS, + .ssymm_oltcopy = ssymm_oltcopyTS, +#endif +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +const openblas_strmm_dispatch_t openblas_strmm_dispatchTS = { +#ifdef ARCH_ARM64 + .strmm_direct_LNUN = strmm_direct_LNUNTS, + .strmm_direct_LNLN = strmm_direct_LNLNTS, + .strmm_direct_LTUN = strmm_direct_LTUNTS, + .strmm_direct_LTLN = strmm_direct_LTLNTS, +#endif + .strmm_gemm_kernel = strmm_gemm_kernelTS, +#if (BUILD_SINGLE==1) + .strmm_kernel_RN = strmm_kernel_RNTS, + .strmm_kernel_RT = strmm_kernel_RTTS, + .strmm_kernel_LN = strmm_kernel_LNTS, + .strmm_kernel_LT = strmm_kernel_LTTS, +#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N + .strmm_iunucopy = strmm_iunucopyTS, + .strmm_iunncopy = strmm_iunncopyTS, + .strmm_iutucopy = strmm_iutucopyTS, + .strmm_iutncopy = strmm_iutncopyTS, + .strmm_ilnucopy = strmm_ilnucopyTS, + .strmm_ilnncopy = strmm_ilnncopyTS, + .strmm_iltucopy = strmm_iltucopyTS, + .strmm_iltncopy = strmm_iltncopyTS, +#else + .strmm_iunucopy = strmm_ounucopyTS, + .strmm_iunncopy = strmm_ounncopyTS, + .strmm_iutucopy = strmm_outucopyTS, + .strmm_iutncopy = strmm_outncopyTS, + .strmm_ilnucopy = strmm_olnucopyTS, + .strmm_ilnncopy = strmm_olnncopyTS, + .strmm_iltucopy = strmm_oltucopyTS, + .strmm_iltncopy = strmm_oltncopyTS, +#endif + .strmm_incopy = strmm_incopyTS, + .strmm_itcopy = strmm_itcopyTS, + .strmm_ounucopy = strmm_ounucopyTS, + .strmm_ounncopy = strmm_ounncopyTS, + .strmm_outucopy = strmm_outucopyTS, + .strmm_outncopy = strmm_outncopyTS, + .strmm_olnucopy = strmm_olnucopyTS, + .strmm_olnncopy = strmm_olnncopyTS, + .strmm_oltucopy = strmm_oltucopyTS, + .strmm_oltncopy = strmm_oltncopyTS, +#endif +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +#ifdef ARCH_ARM64 +const openblas_ssyrk_dispatch_t openblas_ssyrk_dispatchTS = { + .ssyrk_direct_alpha_betaUN = ssyrk_direct_alpha_betaUNTS, + .ssyrk_direct_alpha_betaUT = ssyrk_direct_alpha_betaUTTS, + .ssyrk_direct_alpha_betaLN = ssyrk_direct_alpha_betaLNTS, + .ssyrk_direct_alpha_betaLT = ssyrk_direct_alpha_betaLTTS, +}; +#endif +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +#ifdef ARCH_ARM64 +const openblas_ssyr2k_dispatch_t openblas_ssyr2k_dispatchTS = { + .ssyr2k_direct_alpha_betaUN = ssyr2k_direct_alpha_betaUNTS, + .ssyr2k_direct_alpha_betaUT = ssyr2k_direct_alpha_betaUTTS, + .ssyr2k_direct_alpha_betaLN = ssyr2k_direct_alpha_betaLNTS, + .ssyr2k_direct_alpha_betaLT = ssyr2k_direct_alpha_betaLTTS, +}; +#endif +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +const openblas_strsm_dispatch_t openblas_strsm_dispatchTS = { + .strsm_kernel_LN = strsm_kernel_LNTS, + .strsm_kernel_LT = strsm_kernel_LTTS, + .strsm_kernel_RN = strsm_kernel_RNTS, + .strsm_kernel_RT = strsm_kernel_RTTS, +#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N + .strsm_iunucopy = strsm_iunucopyTS, + .strsm_iunncopy = strsm_iunncopyTS, + .strsm_iutucopy = strsm_iutucopyTS, + .strsm_iutncopy = strsm_iutncopyTS, + .strsm_ilnucopy = strsm_ilnucopyTS, + .strsm_ilnncopy = strsm_ilnncopyTS, + .strsm_iltucopy = strsm_iltucopyTS, + .strsm_iltncopy = strsm_iltncopyTS, +#else + .strsm_iunucopy = strsm_ounucopyTS, + .strsm_iunncopy = strsm_ounncopyTS, + .strsm_iutucopy = strsm_outucopyTS, + .strsm_iutncopy = strsm_outncopyTS, + .strsm_ilnucopy = strsm_olnucopyTS, + .strsm_ilnncopy = strsm_olnncopyTS, + .strsm_iltucopy = strsm_oltucopyTS, + .strsm_iltncopy = strsm_oltncopyTS, +#endif + .strsm_ounucopy = strsm_ounucopyTS, + .strsm_ounncopy = strsm_ounncopyTS, + .strsm_outucopy = strsm_outucopyTS, + .strsm_outncopy = strsm_outncopyTS, + .strsm_olnucopy = strsm_olnucopyTS, + .strsm_olnncopy = strsm_olnncopyTS, + .strsm_oltucopy = strsm_oltucopyTS, + .strsm_oltncopy = strsm_oltncopyTS, +}; +#endif + +#if (BUILD_SINGLE==1) +const openblas_sneg_dispatch_t openblas_sneg_dispatchTS = { +#ifndef NO_LAPACK + .sneg_tcopy = sneg_tcopyTS, +#else + .sneg_tcopy = NULL, +#endif +}; +#endif + +#if (BUILD_SINGLE==1) +const openblas_slaswp_dispatch_t openblas_slaswp_dispatchTS = { +#ifndef NO_LAPACK + .slaswp_ncopy = slaswp_ncopyTS, +#else + .slaswp_ncopy = NULL, +#endif +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_damax_dispatch_t openblas_damax_dispatchTS = { + .damax_k = damax_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_damin_dispatch_t openblas_damin_dispatchTS = { + .damin_k = damin_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_dmax_dispatch_t openblas_dmax_dispatchTS = { + .dmax_k = dmax_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_dmin_dispatch_t openblas_dmin_dispatchTS = { + .dmin_k = dmin_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_idamax_dispatch_t openblas_idamax_dispatchTS = { + .idamax_k = idamax_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_idamin_dispatch_t openblas_idamin_dispatchTS = { + .idamin_k = idamin_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_idmax_dispatch_t openblas_idmax_dispatchTS = { + .idmax_k = idmax_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_idmin_dispatch_t openblas_idmin_dispatchTS = { + .idmin_k = idmin_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_dnrm2_dispatch_t openblas_dnrm2_dispatchTS = { + .dnrm2_k = dnrm2_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_dasum_dispatch_t openblas_dasum_dispatchTS = { + .dasum_k = dasum_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) +const openblas_dsum_dispatch_t openblas_dsum_dispatchTS = { + .dsum_k = dsum_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_dcopy_dispatch_t openblas_dcopy_dispatchTS = { + .dcopy_k = dcopy_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_ddot_dispatch_t openblas_ddot_dispatchTS = { + .ddot_k = ddot_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) +const openblas_dsdot_dispatch_t openblas_dsdot_dispatchTS = { + .dsdot_k = dsdot_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_drot_dispatch_t openblas_drot_dispatchTS = { + .drot_k = drot_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_drotm_dispatch_t openblas_drotm_dispatchTS = { + .drotm_k = drotm_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_daxpy_dispatch_t openblas_daxpy_dispatchTS = { + .daxpy_k = daxpy_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_dscal_dispatch_t openblas_dscal_dispatchTS = { + .dscal_k = dscal_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_dswap_dispatch_t openblas_dswap_dispatchTS = { + .dswap_k = dswap_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_dgemv_dispatch_t openblas_dgemv_dispatchTS = { + .dgemv_n = dgemv_nTS, + .dgemv_t = dgemv_tTS, +}; +#endif + +#if (BUILD_DOUBLE==1) +const openblas_dger_dispatch_t openblas_dger_dispatchTS = { + .dger_k = dger_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) +const openblas_dsymv_dispatch_t openblas_dsymv_dispatchTS = { + .dsymv_L = dsymv_LTS, + .dsymv_U = dsymv_UTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_dgemm_dispatch_t openblas_dgemm_dispatchTS = { +#ifdef ARCH_ARM64 +#ifdef HAVE_SME + .sme_dgemm_kernel = sme_dgemm_kernelTS, +#else + .sme_dgemm_kernel = NULL, +#endif +#endif + .dgemm_kernel = dgemm_kernelTS, + .dgemm_beta = dgemm_betaTS, +#if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N + .dgemm_incopy = dgemm_incopyTS, + .dgemm_itcopy = dgemm_itcopyTS, +#else + .dgemm_incopy = dgemm_oncopyTS, + .dgemm_itcopy = dgemm_otcopyTS, +#endif + .dgemm_oncopy = dgemm_oncopyTS, + .dgemm_otcopy = dgemm_otcopyTS, +#ifdef SMALL_MATRIX_OPT + .dgemm_small_matrix_permit = dgemm_small_matrix_permitTS, + .dgemm_small_kernel_nn = dgemm_small_kernel_nnTS, + .dgemm_small_kernel_nt = dgemm_small_kernel_ntTS, + .dgemm_small_kernel_tn = dgemm_small_kernel_tnTS, + .dgemm_small_kernel_tt = dgemm_small_kernel_ttTS, + .dgemm_small_kernel_b0_nn = dgemm_small_kernel_b0_nnTS, + .dgemm_small_kernel_b0_nt = dgemm_small_kernel_b0_ntTS, + .dgemm_small_kernel_b0_tn = dgemm_small_kernel_b0_tnTS, + .dgemm_small_kernel_b0_tt = dgemm_small_kernel_b0_ttTS, +#endif +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_dsymm_dispatch_t openblas_dsymm_dispatchTS = { + .dsymm_kernel = dsymm_kernelTS, +#if (BUILD_DOUBLE==1) + .dsymm_incopy = dsymm_incopyTS, + .dsymm_itcopy = dsymm_itcopyTS, +#if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N + .dsymm_iutcopy = dsymm_iutcopyTS, + .dsymm_iltcopy = dsymm_iltcopyTS, +#else + .dsymm_iutcopy = dsymm_outcopyTS, + .dsymm_iltcopy = dsymm_oltcopyTS, +#endif + .dsymm_outcopy = dsymm_outcopyTS, + .dsymm_oltcopy = dsymm_oltcopyTS, +#endif +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_dtrmm_dispatch_t openblas_dtrmm_dispatchTS = { + .dtrmm_gemm_kernel = dtrmm_gemm_kernelTS, +#if (BUILD_DOUBLE==1) + .dtrmm_kernel_RN = dtrmm_kernel_RNTS, + .dtrmm_kernel_RT = dtrmm_kernel_RTTS, + .dtrmm_kernel_LN = dtrmm_kernel_LNTS, + .dtrmm_kernel_LT = dtrmm_kernel_LTTS, +#if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N + .dtrmm_iunucopy = dtrmm_iunucopyTS, + .dtrmm_iunncopy = dtrmm_iunncopyTS, + .dtrmm_iutucopy = dtrmm_iutucopyTS, + .dtrmm_iutncopy = dtrmm_iutncopyTS, + .dtrmm_ilnucopy = dtrmm_ilnucopyTS, + .dtrmm_ilnncopy = dtrmm_ilnncopyTS, + .dtrmm_iltucopy = dtrmm_iltucopyTS, + .dtrmm_iltncopy = dtrmm_iltncopyTS, +#else + .dtrmm_iunucopy = dtrmm_ounucopyTS, + .dtrmm_iunncopy = dtrmm_ounncopyTS, + .dtrmm_iutucopy = dtrmm_outucopyTS, + .dtrmm_iutncopy = dtrmm_outncopyTS, + .dtrmm_ilnucopy = dtrmm_olnucopyTS, + .dtrmm_ilnncopy = dtrmm_olnncopyTS, + .dtrmm_iltucopy = dtrmm_oltucopyTS, + .dtrmm_iltncopy = dtrmm_oltncopyTS, +#endif + .dtrmm_incopy = dtrmm_incopyTS, + .dtrmm_itcopy = dtrmm_itcopyTS, + .dtrmm_ounucopy = dtrmm_ounucopyTS, + .dtrmm_ounncopy = dtrmm_ounncopyTS, + .dtrmm_outucopy = dtrmm_outucopyTS, + .dtrmm_outncopy = dtrmm_outncopyTS, + .dtrmm_olnucopy = dtrmm_olnucopyTS, + .dtrmm_olnncopy = dtrmm_olnncopyTS, + .dtrmm_oltucopy = dtrmm_oltucopyTS, + .dtrmm_oltncopy = dtrmm_oltncopyTS, +#endif +}; +#endif + +#if (BUILD_DOUBLE==1) +const openblas_dtrsm_dispatch_t openblas_dtrsm_dispatchTS = { + .dtrsm_kernel_LN = dtrsm_kernel_LNTS, + .dtrsm_kernel_LT = dtrsm_kernel_LTTS, + .dtrsm_kernel_RN = dtrsm_kernel_RNTS, + .dtrsm_kernel_RT = dtrsm_kernel_RTTS, +#if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N + .dtrsm_iunucopy = dtrsm_iunucopyTS, + .dtrsm_iunncopy = dtrsm_iunncopyTS, + .dtrsm_iutucopy = dtrsm_iutucopyTS, + .dtrsm_iutncopy = dtrsm_iutncopyTS, + .dtrsm_ilnucopy = dtrsm_ilnucopyTS, + .dtrsm_ilnncopy = dtrsm_ilnncopyTS, + .dtrsm_iltucopy = dtrsm_iltucopyTS, + .dtrsm_iltncopy = dtrsm_iltncopyTS, +#else + .dtrsm_iunucopy = dtrsm_ounucopyTS, + .dtrsm_iunncopy = dtrsm_ounncopyTS, + .dtrsm_iutucopy = dtrsm_outucopyTS, + .dtrsm_iutncopy = dtrsm_outncopyTS, + .dtrsm_ilnucopy = dtrsm_olnucopyTS, + .dtrsm_ilnncopy = dtrsm_olnncopyTS, + .dtrsm_iltucopy = dtrsm_oltucopyTS, + .dtrsm_iltncopy = dtrsm_oltncopyTS, +#endif + .dtrsm_ounucopy = dtrsm_ounucopyTS, + .dtrsm_ounncopy = dtrsm_ounncopyTS, + .dtrsm_outucopy = dtrsm_outucopyTS, + .dtrsm_outncopy = dtrsm_outncopyTS, + .dtrsm_olnucopy = dtrsm_olnucopyTS, + .dtrsm_olnncopy = dtrsm_olnncopyTS, + .dtrsm_oltucopy = dtrsm_oltucopyTS, + .dtrsm_oltncopy = dtrsm_oltncopyTS, +}; +#endif + +#if (BUILD_DOUBLE==1) +const openblas_dneg_dispatch_t openblas_dneg_dispatchTS = { +#ifndef NO_LAPACK + .dneg_tcopy = dneg_tcopyTS, +#else + .dneg_tcopy = NULL, +#endif +}; +#endif + +#if (BUILD_DOUBLE==1) +const openblas_dlaswp_dispatch_t openblas_dlaswp_dispatchTS = { +#ifndef NO_LAPACK + .dlaswp_ncopy = dlaswp_ncopyTS, +#else + .dlaswp_ncopy = NULL, +#endif +}; +#endif + +#ifdef EXPRECISION +const openblas_qamax_dispatch_t openblas_qamax_dispatchTS = { + .qamax_k = qamax_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qamin_dispatch_t openblas_qamin_dispatchTS = { + .qamin_k = qamin_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qmax_dispatch_t openblas_qmax_dispatchTS = { + .qmax_k = qmax_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qmin_dispatch_t openblas_qmin_dispatchTS = { + .qmin_k = qmin_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_iqamax_dispatch_t openblas_iqamax_dispatchTS = { + .iqamax_k = iqamax_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_iqamin_dispatch_t openblas_iqamin_dispatchTS = { + .iqamin_k = iqamin_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_iqmax_dispatch_t openblas_iqmax_dispatchTS = { + .iqmax_k = iqmax_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_iqmin_dispatch_t openblas_iqmin_dispatchTS = { + .iqmin_k = iqmin_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qnrm2_dispatch_t openblas_qnrm2_dispatchTS = { + .qnrm2_k = qnrm2_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qasum_dispatch_t openblas_qasum_dispatchTS = { + .qasum_k = qasum_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qsum_dispatch_t openblas_qsum_dispatchTS = { + .qsum_k = qsum_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qcopy_dispatch_t openblas_qcopy_dispatchTS = { + .qcopy_k = qcopy_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qdot_dispatch_t openblas_qdot_dispatchTS = { + .qdot_k = qdot_kTS, +}; +#endif - GEMM_DIVIDE_RATE, +#ifdef EXPRECISION +const openblas_qrot_dispatch_t openblas_qrot_dispatchTS = { + .qrot_k = qrot_kTS, +}; +#endif - GEMM_DIVIDE_LIMIT, +#ifdef EXPRECISION +const openblas_qrotm_dispatch_t openblas_qrotm_dispatchTS = { + .qrotm_k = qrotm_kTS, +}; +#endif - GEMM_PREFERRED_SIZE, +#ifdef EXPRECISION +const openblas_qaxpy_dispatch_t openblas_qaxpy_dispatchTS = { + .qaxpy_k = qaxpy_kTS, +}; +#endif - GEMM_DEFAULT_OFFSET_A, GEMM_DEFAULT_OFFSET_B, GEMM_DEFAULT_ALIGN, +#ifdef EXPRECISION +const openblas_qscal_dispatch_t openblas_qscal_dispatchTS = { + .qscal_k = qscal_kTS, +}; +#endif -#ifdef BUILD_HFLOAT16 - 0, 0, 0, - SHGEMM_DEFAULT_UNROLL_M, SHGEMM_DEFAULT_UNROLL_N, -#ifdef SHGEMM_DEFAULT_UNROLL_MN - SHGEMM_DEFAULT_UNROLL_MN, -#else - MAX(SHGEMM_DEFAULT_UNROLL_M, SHGEMM_DEFAULT_UNROLL_N), +#ifdef EXPRECISION +const openblas_qswap_dispatch_t openblas_qswap_dispatchTS = { + .qswap_k = qswap_kTS, +}; #endif - shgemm_kernelTS, shgemm_betaTS, -#if SHGEMM_DEFAULT_UNROLL_M != SHGEMM_DEFAULT_UNROLL_N - shgemm_incopyTS, shgemm_itcopyTS, -#else - shgemm_oncopyTS, shgemm_otcopyTS, + +#ifdef EXPRECISION +const openblas_qgemv_dispatch_t openblas_qgemv_dispatchTS = { + .qgemv_n = qgemv_nTS, + .qgemv_t = qgemv_tTS, +}; #endif - shgemm_oncopyTS, shgemm_otcopyTS, - shgemv_nTS, shgemv_tTS, + +#ifdef EXPRECISION +const openblas_qger_dispatch_t openblas_qger_dispatchTS = { + .qger_k = qger_kTS, +}; #endif -#ifdef BUILD_BFLOAT16 - 0, 0, 0, - BGEMM_DEFAULT_UNROLL_M, BGEMM_DEFAULT_UNROLL_N, -#ifdef BGEMM_DEFAULT_UNROLL_MN - BGEMM_DEFAULT_UNROLL_MN, -#else - MAX(BGEMM_DEFAULT_UNROLL_M, BGEMM_DEFAULT_UNROLL_N), +#ifdef EXPRECISION +const openblas_qsymv_dispatch_t openblas_qsymv_dispatchTS = { + .qsymv_L = qsymv_LTS, + .qsymv_U = qsymv_UTS, +}; #endif - BGEMM_ALIGN_K, - 0, 0, 0, - SBGEMM_DEFAULT_UNROLL_M, SBGEMM_DEFAULT_UNROLL_N, -#ifdef SBGEMM_DEFAULT_UNROLL_MN - SBGEMM_DEFAULT_UNROLL_MN, +#ifdef EXPRECISION +const openblas_qgemm_dispatch_t openblas_qgemm_dispatchTS = { + .qgemm_kernel = qgemm_kernelTS, + .qgemm_beta = qgemm_betaTS, +#if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N + .qgemm_incopy = qgemm_incopyTS, + .qgemm_itcopy = qgemm_itcopyTS, #else - MAX(SBGEMM_DEFAULT_UNROLL_M, SBGEMM_DEFAULT_UNROLL_N), + .qgemm_incopy = qgemm_oncopyTS, + .qgemm_itcopy = qgemm_otcopyTS, +#endif + .qgemm_oncopy = qgemm_oncopyTS, + .qgemm_otcopy = qgemm_otcopyTS, +}; #endif - SBGEMM_ALIGN_K, - 0, // need_amxtile_permission - - sbstobf16_kTS, sbdtobf16_kTS, sbf16tos_kTS, dbf16tod_kTS, - - samax_kTS, samin_kTS, smax_kTS, smin_kTS, - isamax_kTS, isamin_kTS, ismax_kTS, ismin_kTS, - snrm2_kTS, sasum_kTS, ssum_kTS, scopy_kTS, sbdot_kTS, - dsdot_kTS, - srot_kTS, srotm_kTS, bscal_kTS, saxpy_kTS, sscal_kTS, sswap_kTS, - bgemv_nTS, bgemv_tTS, sbgemv_nTS, sbgemv_tTS, sger_kTS, - ssymv_LTS, ssymv_UTS, - - bgemm_kernelTS, bgemm_betaTS, -#if BGEMM_DEFAULT_UNROLL_M != BGEMM_DEFAULT_UNROLL_N - bgemm_incopyTS, bgemm_itcopyTS, +#ifdef EXPRECISION +const openblas_qtrsm_dispatch_t openblas_qtrsm_dispatchTS = { + .qtrsm_kernel_LN = qtrsm_kernel_LNTS, + .qtrsm_kernel_LT = qtrsm_kernel_LTTS, + .qtrsm_kernel_RN = qtrsm_kernel_RNTS, + .qtrsm_kernel_RT = qtrsm_kernel_RTTS, +#if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N + .qtrsm_iunucopy = qtrsm_iunucopyTS, + .qtrsm_iunncopy = qtrsm_iunncopyTS, + .qtrsm_iutucopy = qtrsm_iutucopyTS, + .qtrsm_iutncopy = qtrsm_iutncopyTS, + .qtrsm_ilnucopy = qtrsm_ilnucopyTS, + .qtrsm_ilnncopy = qtrsm_ilnncopyTS, + .qtrsm_iltucopy = qtrsm_iltucopyTS, + .qtrsm_iltncopy = qtrsm_iltncopyTS, #else - bgemm_oncopyTS, bgemm_otcopyTS, + .qtrsm_iunucopy = qtrsm_ounucopyTS, + .qtrsm_iunncopy = qtrsm_ounncopyTS, + .qtrsm_iutucopy = qtrsm_outucopyTS, + .qtrsm_iutncopy = qtrsm_outncopyTS, + .qtrsm_ilnucopy = qtrsm_olnucopyTS, + .qtrsm_ilnncopy = qtrsm_olnncopyTS, + .qtrsm_iltucopy = qtrsm_oltucopyTS, + .qtrsm_iltncopy = qtrsm_oltncopyTS, +#endif + .qtrsm_ounucopy = qtrsm_ounucopyTS, + .qtrsm_ounncopy = qtrsm_ounncopyTS, + .qtrsm_outucopy = qtrsm_outucopyTS, + .qtrsm_outncopy = qtrsm_outncopyTS, + .qtrsm_olnucopy = qtrsm_olnucopyTS, + .qtrsm_olnncopy = qtrsm_olnncopyTS, + .qtrsm_oltucopy = qtrsm_oltucopyTS, + .qtrsm_oltncopy = qtrsm_oltncopyTS, +}; #endif - bgemm_oncopyTS, bgemm_otcopyTS, - sbgemm_kernelTS, sbgemm_betaTS, -#if SBGEMM_DEFAULT_UNROLL_M != SBGEMM_DEFAULT_UNROLL_N - sbgemm_incopyTS, sbgemm_itcopyTS, +#ifdef EXPRECISION +const openblas_qtrmm_dispatch_t openblas_qtrmm_dispatchTS = { + .qtrmm_kernel_RN = qtrmm_kernel_RNTS, + .qtrmm_kernel_RT = qtrmm_kernel_RTTS, + .qtrmm_kernel_LN = qtrmm_kernel_LNTS, + .qtrmm_kernel_LT = qtrmm_kernel_LTTS, +#if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N + .qtrmm_iunucopy = qtrmm_iunucopyTS, + .qtrmm_iunncopy = qtrmm_iunncopyTS, + .qtrmm_iutucopy = qtrmm_iutucopyTS, + .qtrmm_iutncopy = qtrmm_iutncopyTS, + .qtrmm_ilnucopy = qtrmm_ilnucopyTS, + .qtrmm_ilnncopy = qtrmm_ilnncopyTS, + .qtrmm_iltucopy = qtrmm_iltucopyTS, + .qtrmm_iltncopy = qtrmm_iltncopyTS, #else - sbgemm_oncopyTS, sbgemm_otcopyTS, + .qtrmm_iunucopy = qtrmm_ounucopyTS, + .qtrmm_iunncopy = qtrmm_ounncopyTS, + .qtrmm_iutucopy = qtrmm_outucopyTS, + .qtrmm_iutncopy = qtrmm_outncopyTS, + .qtrmm_ilnucopy = qtrmm_olnucopyTS, + .qtrmm_ilnncopy = qtrmm_olnncopyTS, + .qtrmm_iltucopy = qtrmm_oltucopyTS, + .qtrmm_iltncopy = qtrmm_oltncopyTS, +#endif + .qtrmm_ounucopy = qtrmm_ounucopyTS, + .qtrmm_ounncopy = qtrmm_ounncopyTS, + .qtrmm_outucopy = qtrmm_outucopyTS, + .qtrmm_outncopy = qtrmm_outncopyTS, + .qtrmm_olnucopy = qtrmm_olnucopyTS, + .qtrmm_olnncopy = qtrmm_olnncopyTS, + .qtrmm_oltucopy = qtrmm_oltucopyTS, + .qtrmm_oltncopy = qtrmm_oltncopyTS, +}; #endif - sbgemm_oncopyTS, sbgemm_otcopyTS, - strsm_kernel_LNTS, strsm_kernel_LTTS, strsm_kernel_RNTS, strsm_kernel_RTTS, -#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N - strsm_iunucopyTS, strsm_iunncopyTS, strsm_iutucopyTS, strsm_iutncopyTS, - strsm_ilnucopyTS, strsm_ilnncopyTS, strsm_iltucopyTS, strsm_iltncopyTS, +#ifdef EXPRECISION +const openblas_qsymm_dispatch_t openblas_qsymm_dispatchTS = { +#if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N + .qsymm_iutcopy = qsymm_iutcopyTS, + .qsymm_iltcopy = qsymm_iltcopyTS, #else - strsm_ounucopyTS, strsm_ounncopyTS, strsm_outucopyTS, strsm_outncopyTS, - strsm_olnucopyTS, strsm_olnncopyTS, strsm_oltucopyTS, strsm_oltncopyTS, + .qsymm_iutcopy = qsymm_outcopyTS, + .qsymm_iltcopy = qsymm_oltcopyTS, #endif - strsm_ounucopyTS, strsm_ounncopyTS, strsm_outucopyTS, strsm_outncopyTS, - strsm_olnucopyTS, strsm_olnncopyTS, strsm_oltucopyTS, strsm_oltncopyTS, - strmm_kernel_RNTS, strmm_kernel_RTTS, strmm_kernel_LNTS, strmm_kernel_LTTS, -#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N - strmm_iunucopyTS, strmm_iunncopyTS, strmm_iutucopyTS, strmm_iutncopyTS, - strmm_ilnucopyTS, strmm_ilnncopyTS, strmm_iltucopyTS, strmm_iltncopyTS, -#else - strmm_ounucopyTS, strmm_ounncopyTS, strmm_outucopyTS, strmm_outncopyTS, - strmm_olnucopyTS, strmm_olnncopyTS, strmm_oltucopyTS, strmm_oltncopyTS, + .qsymm_outcopy = qsymm_outcopyTS, + .qsymm_oltcopy = qsymm_oltcopyTS, +}; #endif - strmm_ounucopyTS, strmm_ounncopyTS, strmm_outucopyTS, strmm_outncopyTS, - strmm_olnucopyTS, strmm_olnncopyTS, strmm_oltucopyTS, strmm_oltncopyTS, -#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N - ssymm_iutcopyTS, ssymm_iltcopyTS, + +#ifdef EXPRECISION +const openblas_qneg_dispatch_t openblas_qneg_dispatchTS = { +#ifndef NO_LAPACK + .qneg_tcopy = qneg_tcopyTS, #else - ssymm_outcopyTS, ssymm_oltcopyTS, + .qneg_tcopy = NULL, +#endif +}; #endif - ssymm_outcopyTS, ssymm_oltcopyTS, +#ifdef EXPRECISION +const openblas_qlaswp_dispatch_t openblas_qlaswp_dispatchTS = { #ifndef NO_LAPACK - sneg_tcopyTS, slaswp_ncopyTS, + .qlaswp_ncopy = qlaswp_ncopyTS, #else - NULL,NULL, + .qlaswp_ncopy = NULL, #endif -#ifdef SMALL_MATRIX_OPT - sbgemm_small_matrix_permitTS, - sbgemm_small_kernel_nnTS, sbgemm_small_kernel_ntTS, sbgemm_small_kernel_tnTS, sbgemm_small_kernel_ttTS, - sbgemm_small_kernel_b0_nnTS, sbgemm_small_kernel_b0_ntTS, sbgemm_small_kernel_b0_tnTS, sbgemm_small_kernel_b0_ttTS, +}; +#endif + +#if (BUILD_COMPLEX==1) +const openblas_camax_dispatch_t openblas_camax_dispatchTS = { + .camax_k = camax_kTS, +}; #endif + +#if (BUILD_COMPLEX==1) +const openblas_camin_dispatch_t openblas_camin_dispatchTS = { + .camin_k = camin_kTS, +}; #endif -#if ( BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) || (BUILD_COMPLEX16==1) - 0, 0, 0, - SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N, -#ifdef SGEMM_DEFAULT_UNROLL_MN - SGEMM_DEFAULT_UNROLL_MN, -#else - MAX(SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N), +#if (BUILD_COMPLEX==1) +const openblas_icamax_dispatch_t openblas_icamax_dispatchTS = { + .icamax_k = icamax_kTS, +}; #endif + +#if (BUILD_COMPLEX==1) +const openblas_icamin_dispatch_t openblas_icamin_dispatchTS = { + .icamin_k = icamin_kTS, +}; #endif -#ifdef HAVE_EXCLUSIVE_CACHE - 1, -#else - 0, +#if (BUILD_COMPLEX==1) +const openblas_cnrm2_dispatch_t openblas_cnrm2_dispatchTS = { + .cnrm2_k = cnrm2_kTS, +}; #endif -#if (BUILD_SINGLE==1 ) || (BUILD_COMPLEX==1) - samax_kTS, samin_kTS, smax_kTS, smin_kTS, +#if (BUILD_COMPLEX==1) +const openblas_casum_dispatch_t openblas_casum_dispatchTS = { + .casum_k = casum_kTS, +}; #endif -#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) - isamax_kTS, + +#if (BUILD_COMPLEX==1) +const openblas_csum_dispatch_t openblas_csum_dispatchTS = { + .csum_k = csum_kTS, +}; #endif -#if (BUILD_SINGLE==1 ) || (BUILD_COMPLEX==1) - isamin_kTS, ismax_kTS, ismin_kTS, - snrm2_kTS, sasum_kTS, + +#if (BUILD_COMPLEX==1) +const openblas_ccopy_dispatch_t openblas_ccopy_dispatchTS = { + .ccopy_k = ccopy_kTS, +}; #endif -#if BUILD_SINGLE == 1 - ssum_kTS, + +#if (BUILD_COMPLEX==1) +const openblas_cdotu_dispatch_t openblas_cdotu_dispatchTS = { + .cdotu_k = cdotu_kTS, +}; #endif -#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) - scopy_kTS, sdot_kTS, -// dsdot_kTS, - srot_kTS, srotm_kTS, saxpy_kTS, +#if (BUILD_COMPLEX==1) +const openblas_cdotc_dispatch_t openblas_cdotc_dispatchTS = { + .cdotc_k = cdotc_kTS, +}; #endif -#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) || (BUILD_COMPLEX16==1) - sscal_kTS, + +#if (BUILD_COMPLEX==1) +const openblas_csrot_dispatch_t openblas_csrot_dispatchTS = { + .csrot_k = csrot_kTS, +}; #endif -#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) - sswap_kTS, - sgemv_nTS, sgemv_tTS, + +#if (BUILD_COMPLEX==1) +const openblas_caxpy_dispatch_t openblas_caxpy_dispatchTS = { + .caxpy_k = caxpy_kTS, +}; #endif -#if BUILD_SINGLE == 1 - sger_kTS, + +#if (BUILD_COMPLEX==1) +const openblas_caxpyc_dispatch_t openblas_caxpyc_dispatchTS = { + .caxpyc_k = caxpyc_kTS, +}; #endif -#if BUILD_SINGLE == 1 - ssymv_LTS, ssymv_UTS, + +#if (BUILD_COMPLEX==1) +const openblas_cscal_dispatch_t openblas_cscal_dispatchTS = { + .cscal_k = cscal_kTS, +}; #endif -#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) -#ifdef ARCH_X86_64 - sgemm_directTS, - sgemm_direct_performantTS, +#if (BUILD_COMPLEX==1) +const openblas_cswap_dispatch_t openblas_cswap_dispatchTS = { + .cswap_k = cswap_kTS, +}; #endif -#ifdef ARCH_ARM64 - sgemm_directTS, - sgemm_direct_performantTS, - sgemm_direct_alpha_betaTS, - ssymm_direct_alpha_betaLUTS, - ssymm_direct_alpha_betaLLTS, - strmm_direct_LNUNTS, - strmm_direct_LNLNTS, - strmm_direct_LTUNTS, - strmm_direct_LTLNTS, - ssyrk_direct_alpha_betaUNTS, - ssyrk_direct_alpha_betaUTTS, - ssyrk_direct_alpha_betaLNTS, - ssyrk_direct_alpha_betaLTTS, - ssyr2k_direct_alpha_betaUNTS, - ssyr2k_direct_alpha_betaUTTS, - ssyr2k_direct_alpha_betaLNTS, - ssyr2k_direct_alpha_betaLTTS, -#ifdef HAVE_SME - sme_sgemm_kernelTS, -#else - NULL, + +#if (BUILD_COMPLEX==1) +const openblas_cgemv_dispatch_t openblas_cgemv_dispatchTS = { + .cgemv_n = cgemv_nTS, + .cgemv_t = cgemv_tTS, + .cgemv_r = cgemv_rTS, + .cgemv_c = cgemv_cTS, + .cgemv_o = cgemv_oTS, + .cgemv_u = cgemv_uTS, + .cgemv_s = cgemv_sTS, + .cgemv_d = cgemv_dTS, +}; #endif + +#if (BUILD_COMPLEX==1) +const openblas_cgeru_dispatch_t openblas_cgeru_dispatchTS = { + .cgeru_k = cgeru_kTS, +}; #endif - sgemm_kernelTS, sgemm_betaTS, ssymm_kernelTS, strmm_gemm_kernelTS, -#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N - sgemm_incopyTS, sgemm_itcopyTS, -#else - sgemm_oncopyTS, sgemm_otcopyTS, +#if (BUILD_COMPLEX==1) +const openblas_cgerc_dispatch_t openblas_cgerc_dispatchTS = { + .cgerc_k = cgerc_kTS, +}; #endif - sgemm_oncopyTS, sgemm_otcopyTS, + +#if (BUILD_COMPLEX==1) +const openblas_cgerv_dispatch_t openblas_cgerv_dispatchTS = { + .cgerv_k = cgerv_kTS, +}; #endif -#if BUILD_SINGLE == 1 || BUILD_DOUBLE == 1 || BUILD_COMPLEX == 1 -#ifdef SMALL_MATRIX_OPT - sgemm_small_matrix_permitTS, - sgemm_small_kernel_nnTS, sgemm_small_kernel_ntTS, sgemm_small_kernel_tnTS, sgemm_small_kernel_ttTS, - sgemm_small_kernel_b0_nnTS, sgemm_small_kernel_b0_ntTS, sgemm_small_kernel_b0_tnTS, sgemm_small_kernel_b0_ttTS, +#if (BUILD_COMPLEX==1) +const openblas_cgerd_dispatch_t openblas_cgerd_dispatchTS = { + .cgerd_k = cgerd_kTS, +}; +#endif + +#if (BUILD_COMPLEX==1) +const openblas_csymv_dispatch_t openblas_csymv_dispatchTS = { + .csymv_L = csymv_LTS, + .csymv_U = csymv_UTS, +}; #endif + +#if (BUILD_COMPLEX==1) +const openblas_chemv_dispatch_t openblas_chemv_dispatchTS = { + .chemv_L = chemv_LTS, + .chemv_U = chemv_UTS, + .chemv_M = chemv_MTS, + .chemv_V = chemv_VTS, +}; #endif -#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX == 1) - strsm_kernel_LNTS, strsm_kernel_LTTS, strsm_kernel_RNTS, strsm_kernel_RTTS, -#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N - strsm_iunucopyTS, strsm_iunncopyTS, strsm_iutucopyTS, strsm_iutncopyTS, - strsm_ilnucopyTS, strsm_ilnncopyTS, strsm_iltucopyTS, strsm_iltncopyTS, +#if (BUILD_COMPLEX==1) +const openblas_cgemm_dispatch_t openblas_cgemm_dispatchTS = { +#ifdef ARCH_ARM64 +#ifdef HAVE_SME + .sme_cgemm_kernel = sme_cgemm_kernelTS, #else - strsm_ounucopyTS, strsm_ounncopyTS, strsm_outucopyTS, strsm_outncopyTS, - strsm_olnucopyTS, strsm_olnncopyTS, strsm_oltucopyTS, strsm_oltncopyTS, + .sme_cgemm_kernel = NULL, #endif - strsm_ounucopyTS, strsm_ounncopyTS, strsm_outucopyTS, strsm_outncopyTS, - strsm_olnucopyTS, strsm_olnncopyTS, strsm_oltucopyTS, strsm_oltncopyTS, #endif -#if (BUILD_SINGLE==1) - strmm_kernel_RNTS, strmm_kernel_RTTS, strmm_kernel_LNTS, strmm_kernel_LTTS, -#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N - strmm_iunucopyTS, strmm_iunncopyTS, strmm_iutucopyTS, strmm_iutncopyTS, - strmm_ilnucopyTS, strmm_ilnncopyTS, strmm_iltucopyTS, strmm_iltncopyTS, + .cgemm_kernel_n = cgemm_kernel_nTS, + .cgemm_kernel_l = cgemm_kernel_lTS, + .cgemm_kernel_r = cgemm_kernel_rTS, + .cgemm_kernel_b = cgemm_kernel_bTS, + .cgemm_beta = cgemm_betaTS, +#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N + .cgemm_incopy = cgemm_incopyTS, + .cgemm_itcopy = cgemm_itcopyTS, #else - strmm_ounucopyTS, strmm_ounncopyTS, strmm_outucopyTS, strmm_outncopyTS, - strmm_olnucopyTS, strmm_olnncopyTS, strmm_oltucopyTS, strmm_oltncopyTS, + .cgemm_incopy = cgemm_oncopyTS, + .cgemm_itcopy = cgemm_otcopyTS, #endif - ssymm_incopyTS, ssymm_itcopyTS, - strmm_incopyTS, strmm_itcopyTS, - strmm_ounucopyTS, strmm_ounncopyTS, strmm_outucopyTS, strmm_outncopyTS, - strmm_olnucopyTS, strmm_olnncopyTS, strmm_oltucopyTS, strmm_oltncopyTS, -#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N - ssymm_iutcopyTS, ssymm_iltcopyTS, -#else - ssymm_outcopyTS, ssymm_oltcopyTS, + .cgemm_oncopy = cgemm_oncopyTS, + .cgemm_otcopy = cgemm_otcopyTS, +#ifdef SMALL_MATRIX_OPT + .cgemm_small_matrix_permit = cgemm_small_matrix_permitTS, + .cgemm_small_kernel_nn = cgemm_small_kernel_nnTS, + .cgemm_small_kernel_nt = cgemm_small_kernel_ntTS, + .cgemm_small_kernel_nr = cgemm_small_kernel_nrTS, + .cgemm_small_kernel_nc = cgemm_small_kernel_ncTS, + .cgemm_small_kernel_tn = cgemm_small_kernel_tnTS, + .cgemm_small_kernel_tt = cgemm_small_kernel_ttTS, + .cgemm_small_kernel_tr = cgemm_small_kernel_trTS, + .cgemm_small_kernel_tc = cgemm_small_kernel_tcTS, + .cgemm_small_kernel_rn = cgemm_small_kernel_rnTS, + .cgemm_small_kernel_rt = cgemm_small_kernel_rtTS, + .cgemm_small_kernel_rr = cgemm_small_kernel_rrTS, + .cgemm_small_kernel_rc = cgemm_small_kernel_rcTS, + .cgemm_small_kernel_cn = cgemm_small_kernel_cnTS, + .cgemm_small_kernel_ct = cgemm_small_kernel_ctTS, + .cgemm_small_kernel_cr = cgemm_small_kernel_crTS, + .cgemm_small_kernel_cc = cgemm_small_kernel_ccTS, + .cgemm_small_kernel_b0_nn = cgemm_small_kernel_b0_nnTS, + .cgemm_small_kernel_b0_nt = cgemm_small_kernel_b0_ntTS, + .cgemm_small_kernel_b0_nr = cgemm_small_kernel_b0_nrTS, + .cgemm_small_kernel_b0_nc = cgemm_small_kernel_b0_ncTS, + .cgemm_small_kernel_b0_tn = cgemm_small_kernel_b0_tnTS, + .cgemm_small_kernel_b0_tt = cgemm_small_kernel_b0_ttTS, + .cgemm_small_kernel_b0_tr = cgemm_small_kernel_b0_trTS, + .cgemm_small_kernel_b0_tc = cgemm_small_kernel_b0_tcTS, + .cgemm_small_kernel_b0_rn = cgemm_small_kernel_b0_rnTS, + .cgemm_small_kernel_b0_rt = cgemm_small_kernel_b0_rtTS, + .cgemm_small_kernel_b0_rr = cgemm_small_kernel_b0_rrTS, + .cgemm_small_kernel_b0_rc = cgemm_small_kernel_b0_rcTS, + .cgemm_small_kernel_b0_cn = cgemm_small_kernel_b0_cnTS, + .cgemm_small_kernel_b0_ct = cgemm_small_kernel_b0_ctTS, + .cgemm_small_kernel_b0_cr = cgemm_small_kernel_b0_crTS, + .cgemm_small_kernel_b0_cc = cgemm_small_kernel_b0_ccTS, #endif - ssymm_outcopyTS, ssymm_oltcopyTS, -#ifndef NO_LAPACK - sneg_tcopyTS, slaswp_ncopyTS, +}; +#endif + +#if (BUILD_COMPLEX==1) +const openblas_csymm_dispatch_t openblas_csymm_dispatchTS = { + .csymm_kernel_n = csymm_kernel_nTS, + .csymm_kernel_l = csymm_kernel_lTS, + .csymm_kernel_r = csymm_kernel_rTS, + .csymm_kernel_b = csymm_kernel_bTS, + .csymm_incopy = csymm_incopyTS, + .csymm_itcopy = csymm_itcopyTS, +#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N + .csymm_iutcopy = csymm_iutcopyTS, + .csymm_iltcopy = csymm_iltcopyTS, #else - NULL,NULL, + .csymm_iutcopy = csymm_outcopyTS, + .csymm_iltcopy = csymm_oltcopyTS, #endif + .csymm_outcopy = csymm_outcopyTS, + .csymm_oltcopy = csymm_oltcopyTS, +}; #endif -#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) - 0, 0, 0, - DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N, -#ifdef DGEMM_DEFAULT_UNROLL_MN - DGEMM_DEFAULT_UNROLL_MN, +#if (BUILD_COMPLEX==1) +const openblas_ctrmm_dispatch_t openblas_ctrmm_dispatchTS = { + .ctrmm_gemm_kernel_n = ctrmm_gemm_kernel_nTS, + .ctrmm_gemm_kernel_l = ctrmm_gemm_kernel_lTS, + .ctrmm_gemm_kernel_r = ctrmm_gemm_kernel_rTS, + .ctrmm_gemm_kernel_b = ctrmm_gemm_kernel_bTS, + .ctrmm_kernel_RN = ctrmm_kernel_RNTS, + .ctrmm_kernel_RT = ctrmm_kernel_RTTS, + .ctrmm_kernel_RR = ctrmm_kernel_RRTS, + .ctrmm_kernel_RC = ctrmm_kernel_RCTS, + .ctrmm_kernel_LN = ctrmm_kernel_LNTS, + .ctrmm_kernel_LT = ctrmm_kernel_LTTS, + .ctrmm_kernel_LR = ctrmm_kernel_LRTS, + .ctrmm_kernel_LC = ctrmm_kernel_LCTS, +#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N + .ctrmm_iunucopy = ctrmm_iunucopyTS, + .ctrmm_iunncopy = ctrmm_iunncopyTS, + .ctrmm_iutucopy = ctrmm_iutucopyTS, + .ctrmm_iutncopy = ctrmm_iutncopyTS, + .ctrmm_ilnucopy = ctrmm_ilnucopyTS, + .ctrmm_ilnncopy = ctrmm_ilnncopyTS, + .ctrmm_iltucopy = ctrmm_iltucopyTS, + .ctrmm_iltncopy = ctrmm_iltncopyTS, #else - MAX(DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N), -#endif + .ctrmm_iunucopy = ctrmm_ounucopyTS, + .ctrmm_iunncopy = ctrmm_ounncopyTS, + .ctrmm_iutucopy = ctrmm_outucopyTS, + .ctrmm_iutncopy = ctrmm_outncopyTS, + .ctrmm_ilnucopy = ctrmm_olnucopyTS, + .ctrmm_ilnncopy = ctrmm_olnncopyTS, + .ctrmm_iltucopy = ctrmm_oltucopyTS, + .ctrmm_iltncopy = ctrmm_oltncopyTS, +#endif + .ctrmm_incopy = ctrmm_incopyTS, + .ctrmm_itcopy = ctrmm_itcopyTS, + .ctrmm_ounucopy = ctrmm_ounucopyTS, + .ctrmm_ounncopy = ctrmm_ounncopyTS, + .ctrmm_outucopy = ctrmm_outucopyTS, + .ctrmm_outncopy = ctrmm_outncopyTS, + .ctrmm_olnucopy = ctrmm_olnucopyTS, + .ctrmm_olnncopy = ctrmm_olnncopyTS, + .ctrmm_oltucopy = ctrmm_oltucopyTS, + .ctrmm_oltncopy = ctrmm_oltncopyTS, +}; #endif +#if (BUILD_COMPLEX==1) +const openblas_ctrsm_dispatch_t openblas_ctrsm_dispatchTS = { + .ctrsm_kernel_LN = ctrsm_kernel_LNTS, + .ctrsm_kernel_LT = ctrsm_kernel_LTTS, + .ctrsm_kernel_LR = ctrsm_kernel_LRTS, + .ctrsm_kernel_LC = ctrsm_kernel_LCTS, + .ctrsm_kernel_RN = ctrsm_kernel_RNTS, + .ctrsm_kernel_RT = ctrsm_kernel_RTTS, + .ctrsm_kernel_RR = ctrsm_kernel_RRTS, + .ctrsm_kernel_RC = ctrsm_kernel_RCTS, +#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N + .ctrsm_iunucopy = ctrsm_iunucopyTS, + .ctrsm_iunncopy = ctrsm_iunncopyTS, + .ctrsm_iutucopy = ctrsm_iutucopyTS, + .ctrsm_iutncopy = ctrsm_iutncopyTS, + .ctrsm_ilnucopy = ctrsm_ilnucopyTS, + .ctrsm_ilnncopy = ctrsm_ilnncopyTS, + .ctrsm_iltucopy = ctrsm_iltucopyTS, + .ctrsm_iltncopy = ctrsm_iltncopyTS, +#else + .ctrsm_iunucopy = ctrsm_ounucopyTS, + .ctrsm_iunncopy = ctrsm_ounncopyTS, + .ctrsm_iutucopy = ctrsm_outucopyTS, + .ctrsm_iutncopy = ctrsm_outncopyTS, + .ctrsm_ilnucopy = ctrsm_olnucopyTS, + .ctrsm_ilnncopy = ctrsm_olnncopyTS, + .ctrsm_iltucopy = ctrsm_oltucopyTS, + .ctrsm_iltncopy = ctrsm_oltncopyTS, +#endif + .ctrsm_ounucopy = ctrsm_ounucopyTS, + .ctrsm_ounncopy = ctrsm_ounncopyTS, + .ctrsm_outucopy = ctrsm_outucopyTS, + .ctrsm_outncopy = ctrsm_outncopyTS, + .ctrsm_olnucopy = ctrsm_olnucopyTS, + .ctrsm_olnncopy = ctrsm_olnncopyTS, + .ctrsm_oltucopy = ctrsm_oltucopyTS, + .ctrsm_oltncopy = ctrsm_oltncopyTS, +}; +#endif -#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) - damax_kTS, damin_kTS, dmax_kTS, dmin_kTS, - idamax_kTS, idamin_kTS, idmax_kTS, idmin_kTS, - dnrm2_kTS, dasum_kTS, +#if (BUILD_COMPLEX==1) +const openblas_chemm_dispatch_t openblas_chemm_dispatchTS = { +#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N + .chemm_iutcopy = chemm_iutcopyTS, + .chemm_iltcopy = chemm_iltcopyTS, +#else + .chemm_iutcopy = chemm_outcopyTS, + .chemm_iltcopy = chemm_oltcopyTS, #endif -#if (BUILD_DOUBLE==1) - dsum_kTS, + .chemm_outcopy = chemm_outcopyTS, + .chemm_oltcopy = chemm_oltcopyTS, +}; #endif -#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) - dcopy_kTS, ddot_kTS, + +#if (BUILD_COMPLEX==1) +const openblas_cgemm3m_dispatch_t openblas_cgemm3m_dispatchTS = { +#if (USE_GEMM3M) + .cgemm3m_kernel = cgemm3m_kernelTS, + .cgemm3m_incopyb = cgemm3m_incopybTS, + .cgemm3m_incopyr = cgemm3m_incopyrTS, + .cgemm3m_incopyi = cgemm3m_incopyiTS, + .cgemm3m_itcopyb = cgemm3m_itcopybTS, + .cgemm3m_itcopyr = cgemm3m_itcopyrTS, + .cgemm3m_itcopyi = cgemm3m_itcopyiTS, + .cgemm3m_oncopyb = cgemm3m_oncopybTS, + .cgemm3m_oncopyr = cgemm3m_oncopyrTS, + .cgemm3m_oncopyi = cgemm3m_oncopyiTS, + .cgemm3m_otcopyb = cgemm3m_otcopybTS, + .cgemm3m_otcopyr = cgemm3m_otcopyrTS, + .cgemm3m_otcopyi = cgemm3m_otcopyiTS, +#else + .cgemm3m_kernel = NULL, + .cgemm3m_incopyb = NULL, + .cgemm3m_incopyr = NULL, + .cgemm3m_incopyi = NULL, + .cgemm3m_itcopyb = NULL, + .cgemm3m_itcopyr = NULL, + .cgemm3m_itcopyi = NULL, + .cgemm3m_oncopyb = NULL, + .cgemm3m_oncopyr = NULL, + .cgemm3m_oncopyi = NULL, + .cgemm3m_otcopyb = NULL, + .cgemm3m_otcopyr = NULL, + .cgemm3m_otcopyi = NULL, #endif -#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) - dsdot_kTS, +}; #endif -#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) - drot_kTS, - drotm_kTS, - daxpy_kTS, - dscal_kTS, - dswap_kTS, - dgemv_nTS, dgemv_tTS, + +#if (BUILD_COMPLEX==1) +const openblas_csymm3m_dispatch_t openblas_csymm3m_dispatchTS = { +#if (USE_GEMM3M) + .csymm3m_iucopyb = csymm3m_iucopybTS, + .csymm3m_ilcopyb = csymm3m_ilcopybTS, + .csymm3m_iucopyr = csymm3m_iucopyrTS, + .csymm3m_ilcopyr = csymm3m_ilcopyrTS, + .csymm3m_iucopyi = csymm3m_iucopyiTS, + .csymm3m_ilcopyi = csymm3m_ilcopyiTS, + .csymm3m_oucopyb = csymm3m_oucopybTS, + .csymm3m_olcopyb = csymm3m_olcopybTS, + .csymm3m_oucopyr = csymm3m_oucopyrTS, + .csymm3m_olcopyr = csymm3m_olcopyrTS, + .csymm3m_oucopyi = csymm3m_oucopyiTS, + .csymm3m_olcopyi = csymm3m_olcopyiTS, +#else + .csymm3m_iucopyb = NULL, + .csymm3m_ilcopyb = NULL, + .csymm3m_iucopyr = NULL, + .csymm3m_ilcopyr = NULL, + .csymm3m_iucopyi = NULL, + .csymm3m_ilcopyi = NULL, + .csymm3m_oucopyb = NULL, + .csymm3m_olcopyb = NULL, + .csymm3m_oucopyr = NULL, + .csymm3m_olcopyr = NULL, + .csymm3m_oucopyi = NULL, + .csymm3m_olcopyi = NULL, #endif -#if (BUILD_DOUBLE==1) - dger_kTS, - dsymv_LTS, dsymv_UTS, +}; #endif -#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) -#ifdef ARCH_ARM64 -#ifdef HAVE_SME - sme_dgemm_kernelTS, +#if (BUILD_COMPLEX==1) +const openblas_chemm3m_dispatch_t openblas_chemm3m_dispatchTS = { +#if (USE_GEMM3M) + .chemm3m_iucopyb = chemm3m_iucopybTS, + .chemm3m_ilcopyb = chemm3m_ilcopybTS, + .chemm3m_iucopyr = chemm3m_iucopyrTS, + .chemm3m_ilcopyr = chemm3m_ilcopyrTS, + .chemm3m_iucopyi = chemm3m_iucopyiTS, + .chemm3m_ilcopyi = chemm3m_ilcopyiTS, + .chemm3m_oucopyb = chemm3m_oucopybTS, + .chemm3m_olcopyb = chemm3m_olcopybTS, + .chemm3m_oucopyr = chemm3m_oucopyrTS, + .chemm3m_olcopyr = chemm3m_olcopyrTS, + .chemm3m_oucopyi = chemm3m_oucopyiTS, + .chemm3m_olcopyi = chemm3m_olcopyiTS, #else - NULL, + .chemm3m_iucopyb = NULL, + .chemm3m_ilcopyb = NULL, + .chemm3m_iucopyr = NULL, + .chemm3m_ilcopyr = NULL, + .chemm3m_iucopyi = NULL, + .chemm3m_ilcopyi = NULL, + .chemm3m_oucopyb = NULL, + .chemm3m_olcopyb = NULL, + .chemm3m_oucopyr = NULL, + .chemm3m_olcopyr = NULL, + .chemm3m_oucopyi = NULL, + .chemm3m_olcopyi = NULL, #endif +}; #endif - dgemm_kernelTS, dgemm_betaTS, dsymm_kernelTS, dtrmm_gemm_kernelTS, -#if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N - dgemm_incopyTS, dgemm_itcopyTS, + +#if (BUILD_COMPLEX==1) +const openblas_cneg_dispatch_t openblas_cneg_dispatchTS = { +#ifndef NO_LAPACK + .cneg_tcopy = cneg_tcopyTS, #else - dgemm_oncopyTS, dgemm_otcopyTS, + .cneg_tcopy = NULL, #endif - dgemm_oncopyTS, dgemm_otcopyTS, +}; #endif -#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) -#ifdef SMALL_MATRIX_OPT - dgemm_small_matrix_permitTS, - dgemm_small_kernel_nnTS, dgemm_small_kernel_ntTS, dgemm_small_kernel_tnTS, dgemm_small_kernel_ttTS, - dgemm_small_kernel_b0_nnTS, dgemm_small_kernel_b0_ntTS, dgemm_small_kernel_b0_tnTS, dgemm_small_kernel_b0_ttTS, +#if (BUILD_COMPLEX==1) +const openblas_claswp_dispatch_t openblas_claswp_dispatchTS = { +#ifndef NO_LAPACK + .claswp_ncopy = claswp_ncopyTS, +#else + .claswp_ncopy = NULL, #endif +}; #endif -#if (BUILD_DOUBLE==1) - dtrsm_kernel_LNTS, dtrsm_kernel_LTTS, dtrsm_kernel_RNTS, dtrsm_kernel_RTTS, -#if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N - dtrsm_iunucopyTS, dtrsm_iunncopyTS, dtrsm_iutucopyTS, dtrsm_iutncopyTS, - dtrsm_ilnucopyTS, dtrsm_ilnncopyTS, dtrsm_iltucopyTS, dtrsm_iltncopyTS, -#else - dtrsm_ounucopyTS, dtrsm_ounncopyTS, dtrsm_outucopyTS, dtrsm_outncopyTS, - dtrsm_olnucopyTS, dtrsm_olnncopyTS, dtrsm_oltucopyTS, dtrsm_oltncopyTS, + +#if (BUILD_COMPLEX16 == 1) +const openblas_zamax_dispatch_t openblas_zamax_dispatchTS = { + .zamax_k = zamax_kTS, +}; #endif - dtrsm_ounucopyTS, dtrsm_ounncopyTS, dtrsm_outucopyTS, dtrsm_outncopyTS, - dtrsm_olnucopyTS, dtrsm_olnncopyTS, dtrsm_oltucopyTS, dtrsm_oltncopyTS, - dtrmm_kernel_RNTS, dtrmm_kernel_RTTS, dtrmm_kernel_LNTS, dtrmm_kernel_LTTS, -#if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N - dtrmm_iunucopyTS, dtrmm_iunncopyTS, dtrmm_iutucopyTS, dtrmm_iutncopyTS, - dtrmm_ilnucopyTS, dtrmm_ilnncopyTS, dtrmm_iltucopyTS, dtrmm_iltncopyTS, -#else - dtrmm_ounucopyTS, dtrmm_ounncopyTS, dtrmm_outucopyTS, dtrmm_outncopyTS, - dtrmm_olnucopyTS, dtrmm_olnncopyTS, dtrmm_oltucopyTS, dtrmm_oltncopyTS, + +#if (BUILD_COMPLEX16 == 1) +const openblas_zamin_dispatch_t openblas_zamin_dispatchTS = { + .zamin_k = zamin_kTS, +}; #endif - dsymm_incopyTS, dsymm_itcopyTS, - dtrmm_incopyTS, dtrmm_itcopyTS, - dtrmm_ounucopyTS, dtrmm_ounncopyTS, dtrmm_outucopyTS, dtrmm_outncopyTS, - dtrmm_olnucopyTS, dtrmm_olnncopyTS, dtrmm_oltucopyTS, dtrmm_oltncopyTS, -#if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N - dsymm_iutcopyTS, dsymm_iltcopyTS, -#else - dsymm_outcopyTS, dsymm_oltcopyTS, + +#if (BUILD_COMPLEX16 == 1) +const openblas_izamax_dispatch_t openblas_izamax_dispatchTS = { + .izamax_k = izamax_kTS, +}; #endif - dsymm_outcopyTS, dsymm_oltcopyTS, -#ifndef NO_LAPACK - dneg_tcopyTS, dlaswp_ncopyTS, -#else - NULL, NULL, +#if (BUILD_COMPLEX16 == 1) +const openblas_izamin_dispatch_t openblas_izamin_dispatchTS = { + .izamin_k = izamin_kTS, +}; #endif +#if (BUILD_COMPLEX16 == 1) +const openblas_znrm2_dispatch_t openblas_znrm2_dispatchTS = { + .znrm2_k = znrm2_kTS, +}; #endif -#ifdef EXPRECISION +#if (BUILD_COMPLEX16 == 1) +const openblas_zasum_dispatch_t openblas_zasum_dispatchTS = { + .zasum_k = zasum_kTS, +}; +#endif - 0, 0, 0, - QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N, MAX(QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N), +#if (BUILD_COMPLEX16 == 1) +const openblas_zsum_dispatch_t openblas_zsum_dispatchTS = { + .zsum_k = zsum_kTS, +}; +#endif - qamax_kTS, qamin_kTS, qmax_kTS, qmin_kTS, - iqamax_kTS, iqamin_kTS, iqmax_kTS, iqmin_kTS, - qnrm2_kTS, qasum_kTS, qsum_kTS, qcopy_kTS, qdot_kTS, - qrot_kTS, qrotm_kTS, qaxpy_kTS, qscal_kTS, qswap_kTS, - qgemv_nTS, qgemv_tTS, qger_kTS, - qsymv_LTS, qsymv_UTS, - qgemm_kernelTS, qgemm_betaTS, -#if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N - qgemm_incopyTS, qgemm_itcopyTS, -#else - qgemm_oncopyTS, qgemm_otcopyTS, +#if (BUILD_COMPLEX16 == 1) +const openblas_zcopy_dispatch_t openblas_zcopy_dispatchTS = { + .zcopy_k = zcopy_kTS, +}; #endif - qgemm_oncopyTS, qgemm_otcopyTS, - qtrsm_kernel_LNTS, qtrsm_kernel_LTTS, qtrsm_kernel_RNTS, qtrsm_kernel_RTTS, -#if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N - qtrsm_iunucopyTS, qtrsm_iunncopyTS, qtrsm_iutucopyTS, qtrsm_iutncopyTS, - qtrsm_ilnucopyTS, qtrsm_ilnncopyTS, qtrsm_iltucopyTS, qtrsm_iltncopyTS, -#else - qtrsm_ounucopyTS, qtrsm_ounncopyTS, qtrsm_outucopyTS, qtrsm_outncopyTS, - qtrsm_olnucopyTS, qtrsm_olnncopyTS, qtrsm_oltucopyTS, qtrsm_oltncopyTS, + +#if (BUILD_COMPLEX16 == 1) +const openblas_zdotu_dispatch_t openblas_zdotu_dispatchTS = { + .zdotu_k = zdotu_kTS, +}; #endif - qtrsm_ounucopyTS, qtrsm_ounncopyTS, qtrsm_outucopyTS, qtrsm_outncopyTS, - qtrsm_olnucopyTS, qtrsm_olnncopyTS, qtrsm_oltucopyTS, qtrsm_oltncopyTS, - qtrmm_kernel_RNTS, qtrmm_kernel_RTTS, qtrmm_kernel_LNTS, qtrmm_kernel_LTTS, -#if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N - qtrmm_iunucopyTS, qtrmm_iunncopyTS, qtrmm_iutucopyTS, qtrmm_iutncopyTS, - qtrmm_ilnucopyTS, qtrmm_ilnncopyTS, qtrmm_iltucopyTS, qtrmm_iltncopyTS, -#else - qtrmm_ounucopyTS, qtrmm_ounncopyTS, qtrmm_outucopyTS, qtrmm_outncopyTS, - qtrmm_olnucopyTS, qtrmm_olnncopyTS, qtrmm_oltucopyTS, qtrmm_oltncopyTS, + +#if (BUILD_COMPLEX16 == 1) +const openblas_zdotc_dispatch_t openblas_zdotc_dispatchTS = { + .zdotc_k = zdotc_kTS, +}; #endif - qtrmm_ounucopyTS, qtrmm_ounncopyTS, qtrmm_outucopyTS, qtrmm_outncopyTS, - qtrmm_olnucopyTS, qtrmm_olnncopyTS, qtrmm_oltucopyTS, qtrmm_oltncopyTS, -#if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N - qsymm_iutcopyTS, qsymm_iltcopyTS, -#else - qsymm_outcopyTS, qsymm_oltcopyTS, + +#if (BUILD_COMPLEX16 == 1) +const openblas_zdrot_dispatch_t openblas_zdrot_dispatchTS = { + .zdrot_k = zdrot_kTS, +}; #endif - qsymm_outcopyTS, qsymm_oltcopyTS, -#ifndef NO_LAPACK - qneg_tcopyTS, qlaswp_ncopyTS, -#else - NULL, NULL, +#if (BUILD_COMPLEX16 == 1) +const openblas_zaxpy_dispatch_t openblas_zaxpy_dispatchTS = { + .zaxpy_k = zaxpy_kTS, +}; #endif +#if (BUILD_COMPLEX16 == 1) +const openblas_zaxpyc_dispatch_t openblas_zaxpyc_dispatchTS = { + .zaxpyc_k = zaxpyc_kTS, +}; #endif -#if (BUILD_COMPLEX) - 0, 0, 0, - CGEMM_DEFAULT_UNROLL_M, CGEMM_DEFAULT_UNROLL_N, -#ifdef CGEMM_DEFAULT_UNROLL_MN - CGEMM_DEFAULT_UNROLL_MN, -#else - MAX(CGEMM_DEFAULT_UNROLL_M, CGEMM_DEFAULT_UNROLL_N), +#if (BUILD_COMPLEX16 == 1) +const openblas_zscal_dispatch_t openblas_zscal_dispatchTS = { + .zscal_k = zscal_kTS, +}; #endif -#if (BUILD_COMPLEX) - camax_kTS, camin_kTS, + +#if (BUILD_COMPLEX16 == 1) +const openblas_zswap_dispatch_t openblas_zswap_dispatchTS = { + .zswap_k = zswap_kTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_zgemv_dispatch_t openblas_zgemv_dispatchTS = { + .zgemv_n = zgemv_nTS, + .zgemv_t = zgemv_tTS, + .zgemv_r = zgemv_rTS, + .zgemv_c = zgemv_cTS, + .zgemv_o = zgemv_oTS, + .zgemv_u = zgemv_uTS, + .zgemv_s = zgemv_sTS, + .zgemv_d = zgemv_dTS, +}; #endif -#if (BUILD_COMPLEX) - icamax_kTS, + +#if (BUILD_COMPLEX16 == 1) +const openblas_zgeru_dispatch_t openblas_zgeru_dispatchTS = { + .zgeru_k = zgeru_kTS, +}; #endif -#if (BUILD_COMPLEX) - icamin_kTS, - cnrm2_kTS, casum_kTS, csum_kTS, + +#if (BUILD_COMPLEX16 == 1) +const openblas_zgerc_dispatch_t openblas_zgerc_dispatchTS = { + .zgerc_k = zgerc_kTS, +}; #endif -#if (BUILD_COMPLEX) - ccopy_kTS, cdotu_kTS, cdotc_kTS, + +#if (BUILD_COMPLEX16 == 1) +const openblas_zgerv_dispatch_t openblas_zgerv_dispatchTS = { + .zgerv_k = zgerv_kTS, +}; #endif -#if (BUILD_COMPLEX) - csrot_kTS, + +#if (BUILD_COMPLEX16 == 1) +const openblas_zgerd_dispatch_t openblas_zgerd_dispatchTS = { + .zgerd_k = zgerd_kTS, +}; #endif -#if (BUILD_COMPLEX) - caxpy_kTS, - caxpyc_kTS, - cscal_kTS, - cswap_kTS, - cgemv_nTS, cgemv_tTS, cgemv_rTS, cgemv_cTS, - cgemv_oTS, cgemv_uTS, cgemv_sTS, cgemv_dTS, +#if (BUILD_COMPLEX16 == 1) +const openblas_zsymv_dispatch_t openblas_zsymv_dispatchTS = { + .zsymv_L = zsymv_LTS, + .zsymv_U = zsymv_UTS, +}; #endif -#if (BUILD_COMPLEX) - cgeru_kTS, cgerc_kTS, cgerv_kTS, cgerd_kTS, - csymv_LTS, csymv_UTS, - chemv_LTS, chemv_UTS, chemv_MTS, chemv_VTS, + +#if (BUILD_COMPLEX16 == 1) +const openblas_zhemv_dispatch_t openblas_zhemv_dispatchTS = { + .zhemv_L = zhemv_LTS, + .zhemv_U = zhemv_UTS, + .zhemv_M = zhemv_MTS, + .zhemv_V = zhemv_VTS, +}; #endif -#if (BUILD_COMPLEX) + +#if (BUILD_COMPLEX16 == 1) +const openblas_zgemm_dispatch_t openblas_zgemm_dispatchTS = { #ifdef ARCH_ARM64 #ifdef HAVE_SME - sme_cgemm_kernelTS, + .sme_zgemm_kernel = sme_zgemm_kernelTS, #else - NULL, + .sme_zgemm_kernel = NULL, #endif #endif - cgemm_kernel_nTS, cgemm_kernel_lTS, cgemm_kernel_rTS, cgemm_kernel_bTS, - cgemm_betaTS, - csymm_kernel_nTS, csymm_kernel_lTS, csymm_kernel_rTS, csymm_kernel_bTS, - ctrmm_gemm_kernel_nTS, ctrmm_gemm_kernel_lTS, ctrmm_gemm_kernel_rTS, ctrmm_gemm_kernel_bTS, -#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N - cgemm_incopyTS, cgemm_itcopyTS, + .zgemm_kernel_n = zgemm_kernel_nTS, + .zgemm_kernel_l = zgemm_kernel_lTS, + .zgemm_kernel_r = zgemm_kernel_rTS, + .zgemm_kernel_b = zgemm_kernel_bTS, + .zgemm_beta = zgemm_betaTS, +#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N + .zgemm_incopy = zgemm_incopyTS, + .zgemm_itcopy = zgemm_itcopyTS, #else - cgemm_oncopyTS, cgemm_otcopyTS, + .zgemm_incopy = zgemm_oncopyTS, + .zgemm_itcopy = zgemm_otcopyTS, #endif - cgemm_oncopyTS, cgemm_otcopyTS, - + .zgemm_oncopy = zgemm_oncopyTS, + .zgemm_otcopy = zgemm_otcopyTS, #ifdef SMALL_MATRIX_OPT - cgemm_small_matrix_permitTS, - cgemm_small_kernel_nnTS, cgemm_small_kernel_ntTS, cgemm_small_kernel_nrTS, cgemm_small_kernel_ncTS, - cgemm_small_kernel_tnTS, cgemm_small_kernel_ttTS, cgemm_small_kernel_trTS, cgemm_small_kernel_tcTS, - cgemm_small_kernel_rnTS, cgemm_small_kernel_rtTS, cgemm_small_kernel_rrTS, cgemm_small_kernel_rcTS, - cgemm_small_kernel_cnTS, cgemm_small_kernel_ctTS, cgemm_small_kernel_crTS, cgemm_small_kernel_ccTS, - cgemm_small_kernel_b0_nnTS, cgemm_small_kernel_b0_ntTS, cgemm_small_kernel_b0_nrTS, cgemm_small_kernel_b0_ncTS, - cgemm_small_kernel_b0_tnTS, cgemm_small_kernel_b0_ttTS, cgemm_small_kernel_b0_trTS, cgemm_small_kernel_b0_tcTS, - cgemm_small_kernel_b0_rnTS, cgemm_small_kernel_b0_rtTS, cgemm_small_kernel_b0_rrTS, cgemm_small_kernel_b0_rcTS, - cgemm_small_kernel_b0_cnTS, cgemm_small_kernel_b0_ctTS, cgemm_small_kernel_b0_crTS, cgemm_small_kernel_b0_ccTS, + .zgemm_small_matrix_permit = zgemm_small_matrix_permitTS, + .zgemm_small_kernel_nn = zgemm_small_kernel_nnTS, + .zgemm_small_kernel_nt = zgemm_small_kernel_ntTS, + .zgemm_small_kernel_nr = zgemm_small_kernel_nrTS, + .zgemm_small_kernel_nc = zgemm_small_kernel_ncTS, + .zgemm_small_kernel_tn = zgemm_small_kernel_tnTS, + .zgemm_small_kernel_tt = zgemm_small_kernel_ttTS, + .zgemm_small_kernel_tr = zgemm_small_kernel_trTS, + .zgemm_small_kernel_tc = zgemm_small_kernel_tcTS, + .zgemm_small_kernel_rn = zgemm_small_kernel_rnTS, + .zgemm_small_kernel_rt = zgemm_small_kernel_rtTS, + .zgemm_small_kernel_rr = zgemm_small_kernel_rrTS, + .zgemm_small_kernel_rc = zgemm_small_kernel_rcTS, + .zgemm_small_kernel_cn = zgemm_small_kernel_cnTS, + .zgemm_small_kernel_ct = zgemm_small_kernel_ctTS, + .zgemm_small_kernel_cr = zgemm_small_kernel_crTS, + .zgemm_small_kernel_cc = zgemm_small_kernel_ccTS, + .zgemm_small_kernel_b0_nn = zgemm_small_kernel_b0_nnTS, + .zgemm_small_kernel_b0_nt = zgemm_small_kernel_b0_ntTS, + .zgemm_small_kernel_b0_nr = zgemm_small_kernel_b0_nrTS, + .zgemm_small_kernel_b0_nc = zgemm_small_kernel_b0_ncTS, + .zgemm_small_kernel_b0_tn = zgemm_small_kernel_b0_tnTS, + .zgemm_small_kernel_b0_tt = zgemm_small_kernel_b0_ttTS, + .zgemm_small_kernel_b0_tr = zgemm_small_kernel_b0_trTS, + .zgemm_small_kernel_b0_tc = zgemm_small_kernel_b0_tcTS, + .zgemm_small_kernel_b0_rn = zgemm_small_kernel_b0_rnTS, + .zgemm_small_kernel_b0_rt = zgemm_small_kernel_b0_rtTS, + .zgemm_small_kernel_b0_rr = zgemm_small_kernel_b0_rrTS, + .zgemm_small_kernel_b0_rc = zgemm_small_kernel_b0_rcTS, + .zgemm_small_kernel_b0_cn = zgemm_small_kernel_b0_cnTS, + .zgemm_small_kernel_b0_ct = zgemm_small_kernel_b0_ctTS, + .zgemm_small_kernel_b0_cr = zgemm_small_kernel_b0_crTS, + .zgemm_small_kernel_b0_cc = zgemm_small_kernel_b0_ccTS, +#endif +}; #endif - ctrsm_kernel_LNTS, ctrsm_kernel_LTTS, ctrsm_kernel_LRTS, ctrsm_kernel_LCTS, - ctrsm_kernel_RNTS, ctrsm_kernel_RTTS, ctrsm_kernel_RRTS, ctrsm_kernel_RCTS, - -#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N - ctrsm_iunucopyTS, ctrsm_iunncopyTS, ctrsm_iutucopyTS, ctrsm_iutncopyTS, - ctrsm_ilnucopyTS, ctrsm_ilnncopyTS, ctrsm_iltucopyTS, ctrsm_iltncopyTS, +#if (BUILD_COMPLEX16 == 1) +const openblas_zsymm_dispatch_t openblas_zsymm_dispatchTS = { + .zsymm_kernel_n = zsymm_kernel_nTS, + .zsymm_kernel_l = zsymm_kernel_lTS, + .zsymm_kernel_r = zsymm_kernel_rTS, + .zsymm_kernel_b = zsymm_kernel_bTS, + .zsymm_incopy = zsymm_incopyTS, + .zsymm_itcopy = zsymm_itcopyTS, +#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N + .zsymm_iutcopy = zsymm_iutcopyTS, + .zsymm_iltcopy = zsymm_iltcopyTS, #else - ctrsm_ounucopyTS, ctrsm_ounncopyTS, ctrsm_outucopyTS, ctrsm_outncopyTS, - ctrsm_olnucopyTS, ctrsm_olnncopyTS, ctrsm_oltucopyTS, ctrsm_oltncopyTS, -#endif - ctrsm_ounucopyTS, ctrsm_ounncopyTS, ctrsm_outucopyTS, ctrsm_outncopyTS, - ctrsm_olnucopyTS, ctrsm_olnncopyTS, ctrsm_oltucopyTS, ctrsm_oltncopyTS, + .zsymm_iutcopy = zsymm_outcopyTS, + .zsymm_iltcopy = zsymm_oltcopyTS, #endif + .zsymm_outcopy = zsymm_outcopyTS, + .zsymm_oltcopy = zsymm_oltcopyTS, +}; #endif -#if (BUILD_COMPLEX) - - ctrmm_kernel_RNTS, ctrmm_kernel_RTTS, ctrmm_kernel_RRTS, ctrmm_kernel_RCTS, - ctrmm_kernel_LNTS, ctrmm_kernel_LTTS, ctrmm_kernel_LRTS, ctrmm_kernel_LCTS, -#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N - ctrmm_iunucopyTS, ctrmm_iunncopyTS, ctrmm_iutucopyTS, ctrmm_iutncopyTS, - ctrmm_ilnucopyTS, ctrmm_ilnncopyTS, ctrmm_iltucopyTS, ctrmm_iltncopyTS, +#if (BUILD_COMPLEX16 == 1) +const openblas_ztrmm_dispatch_t openblas_ztrmm_dispatchTS = { + .ztrmm_gemm_kernel_n = ztrmm_gemm_kernel_nTS, + .ztrmm_gemm_kernel_l = ztrmm_gemm_kernel_lTS, + .ztrmm_gemm_kernel_r = ztrmm_gemm_kernel_rTS, + .ztrmm_gemm_kernel_b = ztrmm_gemm_kernel_bTS, + .ztrmm_kernel_RN = ztrmm_kernel_RNTS, + .ztrmm_kernel_RT = ztrmm_kernel_RTTS, + .ztrmm_kernel_RR = ztrmm_kernel_RRTS, + .ztrmm_kernel_RC = ztrmm_kernel_RCTS, + .ztrmm_kernel_LN = ztrmm_kernel_LNTS, + .ztrmm_kernel_LT = ztrmm_kernel_LTTS, + .ztrmm_kernel_LR = ztrmm_kernel_LRTS, + .ztrmm_kernel_LC = ztrmm_kernel_LCTS, +#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N + .ztrmm_iunucopy = ztrmm_iunucopyTS, + .ztrmm_iunncopy = ztrmm_iunncopyTS, + .ztrmm_iutucopy = ztrmm_iutucopyTS, + .ztrmm_iutncopy = ztrmm_iutncopyTS, + .ztrmm_ilnucopy = ztrmm_ilnucopyTS, + .ztrmm_ilnncopy = ztrmm_ilnncopyTS, + .ztrmm_iltucopy = ztrmm_iltucopyTS, + .ztrmm_iltncopy = ztrmm_iltncopyTS, #else - ctrmm_ounucopyTS, ctrmm_ounncopyTS, ctrmm_outucopyTS, ctrmm_outncopyTS, - ctrmm_olnucopyTS, ctrmm_olnncopyTS, ctrmm_oltucopyTS, ctrmm_oltncopyTS, + .ztrmm_iunucopy = ztrmm_ounucopyTS, + .ztrmm_iunncopy = ztrmm_ounncopyTS, + .ztrmm_iutucopy = ztrmm_outucopyTS, + .ztrmm_iutncopy = ztrmm_outncopyTS, + .ztrmm_ilnucopy = ztrmm_olnucopyTS, + .ztrmm_ilnncopy = ztrmm_olnncopyTS, + .ztrmm_iltucopy = ztrmm_oltucopyTS, + .ztrmm_iltncopy = ztrmm_oltncopyTS, +#endif + .ztrmm_incopy = ztrmm_incopyTS, + .ztrmm_itcopy = ztrmm_itcopyTS, + .ztrmm_ounucopy = ztrmm_ounucopyTS, + .ztrmm_ounncopy = ztrmm_ounncopyTS, + .ztrmm_outucopy = ztrmm_outucopyTS, + .ztrmm_outncopy = ztrmm_outncopyTS, + .ztrmm_olnucopy = ztrmm_olnucopyTS, + .ztrmm_olnncopy = ztrmm_olnncopyTS, + .ztrmm_oltucopy = ztrmm_oltucopyTS, + .ztrmm_oltncopy = ztrmm_oltncopyTS, +}; #endif - csymm_incopyTS, csymm_itcopyTS, - ctrmm_incopyTS, ctrmm_itcopyTS, - ctrmm_ounucopyTS, ctrmm_ounncopyTS, ctrmm_outucopyTS, ctrmm_outncopyTS, - ctrmm_olnucopyTS, ctrmm_olnncopyTS, ctrmm_oltucopyTS, ctrmm_oltncopyTS, -#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N - csymm_iutcopyTS, csymm_iltcopyTS, +#if (BUILD_COMPLEX16 == 1) +const openblas_ztrsm_dispatch_t openblas_ztrsm_dispatchTS = { + .ztrsm_kernel_LN = ztrsm_kernel_LNTS, + .ztrsm_kernel_LT = ztrsm_kernel_LTTS, + .ztrsm_kernel_LR = ztrsm_kernel_LRTS, + .ztrsm_kernel_LC = ztrsm_kernel_LCTS, + .ztrsm_kernel_RN = ztrsm_kernel_RNTS, + .ztrsm_kernel_RT = ztrsm_kernel_RTTS, + .ztrsm_kernel_RR = ztrsm_kernel_RRTS, + .ztrsm_kernel_RC = ztrsm_kernel_RCTS, +#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N + .ztrsm_iunucopy = ztrsm_iunucopyTS, + .ztrsm_iunncopy = ztrsm_iunncopyTS, + .ztrsm_iutucopy = ztrsm_iutucopyTS, + .ztrsm_iutncopy = ztrsm_iutncopyTS, + .ztrsm_ilnucopy = ztrsm_ilnucopyTS, + .ztrsm_ilnncopy = ztrsm_ilnncopyTS, + .ztrsm_iltucopy = ztrsm_iltucopyTS, + .ztrsm_iltncopy = ztrsm_iltncopyTS, #else - csymm_outcopyTS, csymm_oltcopyTS, + .ztrsm_iunucopy = ztrsm_ounucopyTS, + .ztrsm_iunncopy = ztrsm_ounncopyTS, + .ztrsm_iutucopy = ztrsm_outucopyTS, + .ztrsm_iutncopy = ztrsm_outncopyTS, + .ztrsm_ilnucopy = ztrsm_olnucopyTS, + .ztrsm_ilnncopy = ztrsm_olnncopyTS, + .ztrsm_iltucopy = ztrsm_oltucopyTS, + .ztrsm_iltncopy = ztrsm_oltncopyTS, +#endif + .ztrsm_ounucopy = ztrsm_ounucopyTS, + .ztrsm_ounncopy = ztrsm_ounncopyTS, + .ztrsm_outucopy = ztrsm_outucopyTS, + .ztrsm_outncopy = ztrsm_outncopyTS, + .ztrsm_olnucopy = ztrsm_olnucopyTS, + .ztrsm_olnncopy = ztrsm_olnncopyTS, + .ztrsm_oltucopy = ztrsm_oltucopyTS, + .ztrsm_oltncopy = ztrsm_oltncopyTS, +}; #endif - csymm_outcopyTS, csymm_oltcopyTS, -#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N - chemm_iutcopyTS, chemm_iltcopyTS, + +#if (BUILD_COMPLEX16 == 1) +const openblas_zhemm_dispatch_t openblas_zhemm_dispatchTS = { +#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N + .zhemm_iutcopy = zhemm_iutcopyTS, + .zhemm_iltcopy = zhemm_iltcopyTS, #else - chemm_outcopyTS, chemm_oltcopyTS, + .zhemm_iutcopy = zhemm_outcopyTS, + .zhemm_iltcopy = zhemm_oltcopyTS, +#endif + .zhemm_outcopy = zhemm_outcopyTS, + .zhemm_oltcopy = zhemm_oltcopyTS, +}; #endif - chemm_outcopyTS, chemm_oltcopyTS, - - 0, 0, 0, +#if (BUILD_COMPLEX16 == 1) +const openblas_zgemm3m_dispatch_t openblas_zgemm3m_dispatchTS = { #if (USE_GEMM3M) -#ifdef CGEMM3M_DEFAULT_UNROLL_M - CGEMM3M_DEFAULT_UNROLL_M, CGEMM3M_DEFAULT_UNROLL_N, MAX(CGEMM3M_DEFAULT_UNROLL_M, CGEMM3M_DEFAULT_UNROLL_N), + .zgemm3m_kernel = zgemm3m_kernelTS, + .zgemm3m_incopyb = zgemm3m_incopybTS, + .zgemm3m_incopyr = zgemm3m_incopyrTS, + .zgemm3m_incopyi = zgemm3m_incopyiTS, + .zgemm3m_itcopyb = zgemm3m_itcopybTS, + .zgemm3m_itcopyr = zgemm3m_itcopyrTS, + .zgemm3m_itcopyi = zgemm3m_itcopyiTS, + .zgemm3m_oncopyb = zgemm3m_oncopybTS, + .zgemm3m_oncopyr = zgemm3m_oncopyrTS, + .zgemm3m_oncopyi = zgemm3m_oncopyiTS, + .zgemm3m_otcopyb = zgemm3m_otcopybTS, + .zgemm3m_otcopyr = zgemm3m_otcopyrTS, + .zgemm3m_otcopyi = zgemm3m_otcopyiTS, #else - SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N, MAX(SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N), + .zgemm3m_kernel = NULL, + .zgemm3m_incopyb = NULL, + .zgemm3m_incopyr = NULL, + .zgemm3m_incopyi = NULL, + .zgemm3m_itcopyb = NULL, + .zgemm3m_itcopyr = NULL, + .zgemm3m_itcopyi = NULL, + .zgemm3m_oncopyb = NULL, + .zgemm3m_oncopyr = NULL, + .zgemm3m_oncopyi = NULL, + .zgemm3m_otcopyb = NULL, + .zgemm3m_otcopyr = NULL, + .zgemm3m_otcopyi = NULL, +#endif +}; #endif - - cgemm3m_kernelTS, - - cgemm3m_incopybTS, cgemm3m_incopyrTS, - cgemm3m_incopyiTS, cgemm3m_itcopybTS, - cgemm3m_itcopyrTS, cgemm3m_itcopyiTS, - cgemm3m_oncopybTS, cgemm3m_oncopyrTS, - cgemm3m_oncopyiTS, cgemm3m_otcopybTS, - cgemm3m_otcopyrTS, cgemm3m_otcopyiTS, - - csymm3m_iucopybTS, csymm3m_ilcopybTS, - csymm3m_iucopyrTS, csymm3m_ilcopyrTS, - csymm3m_iucopyiTS, csymm3m_ilcopyiTS, - csymm3m_oucopybTS, csymm3m_olcopybTS, - csymm3m_oucopyrTS, csymm3m_olcopyrTS, - csymm3m_oucopyiTS, csymm3m_olcopyiTS, - - chemm3m_iucopybTS, chemm3m_ilcopybTS, - chemm3m_iucopyrTS, chemm3m_ilcopyrTS, - chemm3m_iucopyiTS, chemm3m_ilcopyiTS, - - chemm3m_oucopybTS, chemm3m_olcopybTS, - chemm3m_oucopyrTS, chemm3m_olcopyrTS, - chemm3m_oucopyiTS, chemm3m_olcopyiTS, +#if (BUILD_COMPLEX16 == 1) +const openblas_zsymm3m_dispatch_t openblas_zsymm3m_dispatchTS = { +#if (USE_GEMM3M) + .zsymm3m_iucopyb = zsymm3m_iucopybTS, + .zsymm3m_ilcopyb = zsymm3m_ilcopybTS, + .zsymm3m_iucopyr = zsymm3m_iucopyrTS, + .zsymm3m_ilcopyr = zsymm3m_ilcopyrTS, + .zsymm3m_iucopyi = zsymm3m_iucopyiTS, + .zsymm3m_ilcopyi = zsymm3m_ilcopyiTS, + .zsymm3m_oucopyb = zsymm3m_oucopybTS, + .zsymm3m_olcopyb = zsymm3m_olcopybTS, + .zsymm3m_oucopyr = zsymm3m_oucopyrTS, + .zsymm3m_olcopyr = zsymm3m_olcopyrTS, + .zsymm3m_oucopyi = zsymm3m_oucopyiTS, + .zsymm3m_olcopyi = zsymm3m_olcopyiTS, #else - 0, 0, 0, - - NULL, - - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - - NULL, NULL, - NULL, NULL, - NULL, NULL, - - NULL, NULL, - NULL, NULL, - NULL, NULL, + .zsymm3m_iucopyb = NULL, + .zsymm3m_ilcopyb = NULL, + .zsymm3m_iucopyr = NULL, + .zsymm3m_ilcopyr = NULL, + .zsymm3m_iucopyi = NULL, + .zsymm3m_ilcopyi = NULL, + .zsymm3m_oucopyb = NULL, + .zsymm3m_olcopyb = NULL, + .zsymm3m_oucopyr = NULL, + .zsymm3m_olcopyr = NULL, + .zsymm3m_oucopyi = NULL, + .zsymm3m_olcopyi = NULL, #endif +}; #endif -#if (BUILD_COMPLEX) -#ifndef NO_LAPACK - cneg_tcopyTS, - - claswp_ncopyTS, +#if (BUILD_COMPLEX16 == 1) +const openblas_zhemm3m_dispatch_t openblas_zhemm3m_dispatchTS = { +#if (USE_GEMM3M) + .zhemm3m_iucopyb = zhemm3m_iucopybTS, + .zhemm3m_ilcopyb = zhemm3m_ilcopybTS, + .zhemm3m_iucopyr = zhemm3m_iucopyrTS, + .zhemm3m_ilcopyr = zhemm3m_ilcopyrTS, + .zhemm3m_iucopyi = zhemm3m_iucopyiTS, + .zhemm3m_ilcopyi = zhemm3m_ilcopyiTS, + .zhemm3m_oucopyb = zhemm3m_oucopybTS, + .zhemm3m_olcopyb = zhemm3m_olcopybTS, + .zhemm3m_oucopyr = zhemm3m_oucopyrTS, + .zhemm3m_olcopyr = zhemm3m_olcopyrTS, + .zhemm3m_oucopyi = zhemm3m_oucopyiTS, + .zhemm3m_olcopyi = zhemm3m_olcopyiTS, #else - NULL, NULL, + .zhemm3m_iucopyb = NULL, + .zhemm3m_ilcopyb = NULL, + .zhemm3m_iucopyr = NULL, + .zhemm3m_ilcopyr = NULL, + .zhemm3m_iucopyi = NULL, + .zhemm3m_ilcopyi = NULL, + .zhemm3m_oucopyb = NULL, + .zhemm3m_olcopyb = NULL, + .zhemm3m_oucopyr = NULL, + .zhemm3m_olcopyr = NULL, + .zhemm3m_oucopyi = NULL, + .zhemm3m_olcopyi = NULL, #endif - +}; #endif -#if BUILD_COMPLEX16 == 1 - 0, 0, 0, - ZGEMM_DEFAULT_UNROLL_M, ZGEMM_DEFAULT_UNROLL_N, -#ifdef ZGEMM_DEFAULT_UNROLL_MN - ZGEMM_DEFAULT_UNROLL_MN, +#if (BUILD_COMPLEX16 == 1) +const openblas_zneg_dispatch_t openblas_zneg_dispatchTS = { +#ifndef NO_LAPACK + .zneg_tcopy = zneg_tcopyTS, #else - MAX(ZGEMM_DEFAULT_UNROLL_M, ZGEMM_DEFAULT_UNROLL_N), + .zneg_tcopy = NULL, +#endif +}; #endif - zamax_kTS, zamin_kTS, izamax_kTS, izamin_kTS, - znrm2_kTS, zasum_kTS, zsum_kTS, zcopy_kTS, - zdotu_kTS, zdotc_kTS, zdrot_kTS, - zaxpy_kTS, zaxpyc_kTS, zscal_kTS, zswap_kTS, - - zgemv_nTS, zgemv_tTS, zgemv_rTS, zgemv_cTS, - zgemv_oTS, zgemv_uTS, zgemv_sTS, zgemv_dTS, - zgeru_kTS, zgerc_kTS, zgerv_kTS, zgerd_kTS, - zsymv_LTS, zsymv_UTS, - zhemv_LTS, zhemv_UTS, zhemv_MTS, zhemv_VTS, -#ifdef ARCH_ARM64 -#ifdef HAVE_SME - sme_zgemm_kernelTS, +#if (BUILD_COMPLEX16 == 1) +const openblas_zlaswp_dispatch_t openblas_zlaswp_dispatchTS = { +#ifndef NO_LAPACK + .zlaswp_ncopy = zlaswp_ncopyTS, #else - NULL, + .zlaswp_ncopy = NULL, #endif +}; #endif - zgemm_kernel_nTS, zgemm_kernel_lTS, zgemm_kernel_rTS, zgemm_kernel_bTS, - zgemm_betaTS, - zsymm_kernel_nTS, zsymm_kernel_lTS, zsymm_kernel_rTS, zsymm_kernel_bTS, - ztrmm_gemm_kernel_nTS, ztrmm_gemm_kernel_lTS, ztrmm_gemm_kernel_rTS, ztrmm_gemm_kernel_bTS, -#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N - zgemm_incopyTS, zgemm_itcopyTS, -#else - zgemm_oncopyTS, zgemm_otcopyTS, +#ifdef EXPRECISION +const openblas_xamax_dispatch_t openblas_xamax_dispatchTS = { + .xamax_k = xamax_kTS, +}; #endif - zgemm_oncopyTS, zgemm_otcopyTS, -#ifdef SMALL_MATRIX_OPT - zgemm_small_matrix_permitTS, - zgemm_small_kernel_nnTS, zgemm_small_kernel_ntTS, zgemm_small_kernel_nrTS, zgemm_small_kernel_ncTS, - zgemm_small_kernel_tnTS, zgemm_small_kernel_ttTS, zgemm_small_kernel_trTS, zgemm_small_kernel_tcTS, - zgemm_small_kernel_rnTS, zgemm_small_kernel_rtTS, zgemm_small_kernel_rrTS, zgemm_small_kernel_rcTS, - zgemm_small_kernel_cnTS, zgemm_small_kernel_ctTS, zgemm_small_kernel_crTS, zgemm_small_kernel_ccTS, - zgemm_small_kernel_b0_nnTS, zgemm_small_kernel_b0_ntTS, zgemm_small_kernel_b0_nrTS, zgemm_small_kernel_b0_ncTS, - zgemm_small_kernel_b0_tnTS, zgemm_small_kernel_b0_ttTS, zgemm_small_kernel_b0_trTS, zgemm_small_kernel_b0_tcTS, - zgemm_small_kernel_b0_rnTS, zgemm_small_kernel_b0_rtTS, zgemm_small_kernel_b0_rrTS, zgemm_small_kernel_b0_rcTS, - zgemm_small_kernel_b0_cnTS, zgemm_small_kernel_b0_ctTS, zgemm_small_kernel_b0_crTS, zgemm_small_kernel_b0_ccTS, +#ifdef EXPRECISION +const openblas_xamin_dispatch_t openblas_xamin_dispatchTS = { + .xamin_k = xamin_kTS, +}; #endif - ztrsm_kernel_LNTS, ztrsm_kernel_LTTS, ztrsm_kernel_LRTS, ztrsm_kernel_LCTS, - ztrsm_kernel_RNTS, ztrsm_kernel_RTTS, ztrsm_kernel_RRTS, ztrsm_kernel_RCTS, - -#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N - ztrsm_iunucopyTS, ztrsm_iunncopyTS, ztrsm_iutucopyTS, ztrsm_iutncopyTS, - ztrsm_ilnucopyTS, ztrsm_ilnncopyTS, ztrsm_iltucopyTS, ztrsm_iltncopyTS, -#else - ztrsm_ounucopyTS, ztrsm_ounncopyTS, ztrsm_outucopyTS, ztrsm_outncopyTS, - ztrsm_olnucopyTS, ztrsm_olnncopyTS, ztrsm_oltucopyTS, ztrsm_oltncopyTS, +#ifdef EXPRECISION +const openblas_ixamax_dispatch_t openblas_ixamax_dispatchTS = { + .ixamax_k = ixamax_kTS, +}; #endif - ztrsm_ounucopyTS, ztrsm_ounncopyTS, ztrsm_outucopyTS, ztrsm_outncopyTS, - ztrsm_olnucopyTS, ztrsm_olnncopyTS, ztrsm_oltucopyTS, ztrsm_oltncopyTS, - - ztrmm_kernel_RNTS, ztrmm_kernel_RTTS, ztrmm_kernel_RRTS, ztrmm_kernel_RCTS, - ztrmm_kernel_LNTS, ztrmm_kernel_LTTS, ztrmm_kernel_LRTS, ztrmm_kernel_LCTS, -#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N - ztrmm_iunucopyTS, ztrmm_iunncopyTS, ztrmm_iutucopyTS, ztrmm_iutncopyTS, - ztrmm_ilnucopyTS, ztrmm_ilnncopyTS, ztrmm_iltucopyTS, ztrmm_iltncopyTS, -#else - ztrmm_ounucopyTS, ztrmm_ounncopyTS, ztrmm_outucopyTS, ztrmm_outncopyTS, - ztrmm_olnucopyTS, ztrmm_olnncopyTS, ztrmm_oltucopyTS, ztrmm_oltncopyTS, +#ifdef EXPRECISION +const openblas_ixamin_dispatch_t openblas_ixamin_dispatchTS = { + .ixamin_k = ixamin_kTS, +}; #endif - zsymm_incopyTS, zsymm_itcopyTS, - ztrmm_incopyTS, ztrmm_itcopyTS, - ztrmm_ounucopyTS, ztrmm_ounncopyTS, ztrmm_outucopyTS, ztrmm_outncopyTS, - ztrmm_olnucopyTS, ztrmm_olnncopyTS, ztrmm_oltucopyTS, ztrmm_oltncopyTS, -#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N - zsymm_iutcopyTS, zsymm_iltcopyTS, -#else - zsymm_outcopyTS, zsymm_oltcopyTS, -#endif - zsymm_outcopyTS, zsymm_oltcopyTS, -#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N - zhemm_iutcopyTS, zhemm_iltcopyTS, -#else - zhemm_outcopyTS, zhemm_oltcopyTS, +#ifdef EXPRECISION +const openblas_xnrm2_dispatch_t openblas_xnrm2_dispatchTS = { + .xnrm2_k = xnrm2_kTS, +}; #endif - zhemm_outcopyTS, zhemm_oltcopyTS, - 0, 0, 0, -#if (USE_GEMM3M) -#ifdef ZGEMM3M_DEFAULT_UNROLL_M - ZGEMM3M_DEFAULT_UNROLL_M, ZGEMM3M_DEFAULT_UNROLL_N, MAX(ZGEMM3M_DEFAULT_UNROLL_M, ZGEMM3M_DEFAULT_UNROLL_N), -#else - DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N, MAX(DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N), +#ifdef EXPRECISION +const openblas_xasum_dispatch_t openblas_xasum_dispatchTS = { + .xasum_k = xasum_kTS, +}; #endif +#ifdef EXPRECISION +const openblas_xsum_dispatch_t openblas_xsum_dispatchTS = { + .xsum_k = xsum_kTS, +}; +#endif - zgemm3m_kernelTS, - - zgemm3m_incopybTS, zgemm3m_incopyrTS, - zgemm3m_incopyiTS, zgemm3m_itcopybTS, - zgemm3m_itcopyrTS, zgemm3m_itcopyiTS, - zgemm3m_oncopybTS, zgemm3m_oncopyrTS, - zgemm3m_oncopyiTS, zgemm3m_otcopybTS, - zgemm3m_otcopyrTS, zgemm3m_otcopyiTS, - - zsymm3m_iucopybTS, zsymm3m_ilcopybTS, - zsymm3m_iucopyrTS, zsymm3m_ilcopyrTS, - zsymm3m_iucopyiTS, zsymm3m_ilcopyiTS, - zsymm3m_oucopybTS, zsymm3m_olcopybTS, - zsymm3m_oucopyrTS, zsymm3m_olcopyrTS, - zsymm3m_oucopyiTS, zsymm3m_olcopyiTS, +#ifdef EXPRECISION +const openblas_xcopy_dispatch_t openblas_xcopy_dispatchTS = { + .xcopy_k = xcopy_kTS, +}; +#endif - zhemm3m_iucopybTS, zhemm3m_ilcopybTS, - zhemm3m_iucopyrTS, zhemm3m_ilcopyrTS, - zhemm3m_iucopyiTS, zhemm3m_ilcopyiTS, +#ifdef EXPRECISION +const openblas_xdotu_dispatch_t openblas_xdotu_dispatchTS = { + .xdotu_k = xdotu_kTS, +}; +#endif - zhemm3m_oucopybTS, zhemm3m_olcopybTS, - zhemm3m_oucopyrTS, zhemm3m_olcopyrTS, - zhemm3m_oucopyiTS, zhemm3m_olcopyiTS, -#else - 0, 0, 0, +#ifdef EXPRECISION +const openblas_xdotc_dispatch_t openblas_xdotc_dispatchTS = { + .xdotc_k = xdotc_kTS, +}; +#endif - NULL, +#ifdef EXPRECISION +const openblas_xqrot_dispatch_t openblas_xqrot_dispatchTS = { + .xqrot_k = xqrot_kTS, +}; +#endif - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, +#ifdef EXPRECISION +const openblas_xaxpy_dispatch_t openblas_xaxpy_dispatchTS = { + .xaxpy_k = xaxpy_kTS, +}; +#endif - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, +#ifdef EXPRECISION +const openblas_xaxpyc_dispatch_t openblas_xaxpyc_dispatchTS = { + .xaxpyc_k = xaxpyc_kTS, +}; +#endif - NULL, NULL, - NULL, NULL, - NULL, NULL, +#ifdef EXPRECISION +const openblas_xscal_dispatch_t openblas_xscal_dispatchTS = { + .xscal_k = xscal_kTS, +}; +#endif - NULL, NULL, - NULL, NULL, - NULL, NULL, +#ifdef EXPRECISION +const openblas_xswap_dispatch_t openblas_xswap_dispatchTS = { + .xswap_k = xswap_kTS, +}; #endif -#ifndef NO_LAPACK - zneg_tcopyTS, zlaswp_ncopyTS, -#else - NULL, NULL, +#ifdef EXPRECISION +const openblas_xgemv_dispatch_t openblas_xgemv_dispatchTS = { + .xgemv_n = xgemv_nTS, + .xgemv_t = xgemv_tTS, + .xgemv_r = xgemv_rTS, + .xgemv_c = xgemv_cTS, + .xgemv_o = xgemv_oTS, + .xgemv_u = xgemv_uTS, + .xgemv_s = xgemv_sTS, + .xgemv_d = xgemv_dTS, +}; #endif +#ifdef EXPRECISION +const openblas_xgeru_dispatch_t openblas_xgeru_dispatchTS = { + .xgeru_k = xgeru_kTS, +}; #endif #ifdef EXPRECISION +const openblas_xgerc_dispatch_t openblas_xgerc_dispatchTS = { + .xgerc_k = xgerc_kTS, +}; +#endif - 0, 0, 0, - XGEMM_DEFAULT_UNROLL_M, XGEMM_DEFAULT_UNROLL_N, MAX(XGEMM_DEFAULT_UNROLL_M, XGEMM_DEFAULT_UNROLL_N), +#ifdef EXPRECISION +const openblas_xgerv_dispatch_t openblas_xgerv_dispatchTS = { + .xgerv_k = xgerv_kTS, +}; +#endif - xamax_kTS, xamin_kTS, ixamax_kTS, ixamin_kTS, - xnrm2_kTS, xasum_kTS, xsum_kTS, xcopy_kTS, - xdotu_kTS, xdotc_kTS, xqrot_kTS, - xaxpy_kTS, xaxpyc_kTS, xscal_kTS, xswap_kTS, +#ifdef EXPRECISION +const openblas_xgerd_dispatch_t openblas_xgerd_dispatchTS = { + .xgerd_k = xgerd_kTS, +}; +#endif - xgemv_nTS, xgemv_tTS, xgemv_rTS, xgemv_cTS, - xgemv_oTS, xgemv_uTS, xgemv_sTS, xgemv_dTS, - xgeru_kTS, xgerc_kTS, xgerv_kTS, xgerd_kTS, - xsymv_LTS, xsymv_UTS, - xhemv_LTS, xhemv_UTS, xhemv_MTS, xhemv_VTS, +#ifdef EXPRECISION +const openblas_xsymv_dispatch_t openblas_xsymv_dispatchTS = { + .xsymv_L = xsymv_LTS, + .xsymv_U = xsymv_UTS, +}; +#endif - xgemm_kernel_nTS, xgemm_kernel_lTS, xgemm_kernel_rTS, xgemm_kernel_bTS, - xgemm_betaTS, +#ifdef EXPRECISION +const openblas_xhemv_dispatch_t openblas_xhemv_dispatchTS = { + .xhemv_L = xhemv_LTS, + .xhemv_U = xhemv_UTS, + .xhemv_M = xhemv_MTS, + .xhemv_V = xhemv_VTS, +}; +#endif +#ifdef EXPRECISION +const openblas_xgemm_dispatch_t openblas_xgemm_dispatchTS = { + .xgemm_kernel_n = xgemm_kernel_nTS, + .xgemm_kernel_l = xgemm_kernel_lTS, + .xgemm_kernel_r = xgemm_kernel_rTS, + .xgemm_kernel_b = xgemm_kernel_bTS, + .xgemm_beta = xgemm_betaTS, #if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N - xgemm_incopyTS, xgemm_itcopyTS, + .xgemm_incopy = xgemm_incopyTS, + .xgemm_itcopy = xgemm_itcopyTS, #else - xgemm_oncopyTS, xgemm_otcopyTS, + .xgemm_incopy = xgemm_oncopyTS, + .xgemm_itcopy = xgemm_otcopyTS, +#endif + .xgemm_oncopy = xgemm_oncopyTS, + .xgemm_otcopy = xgemm_otcopyTS, +}; #endif - xgemm_oncopyTS, xgemm_otcopyTS, - - xtrsm_kernel_LNTS, xtrsm_kernel_LTTS, xtrsm_kernel_LRTS, xtrsm_kernel_LCTS, - xtrsm_kernel_RNTS, xtrsm_kernel_RTTS, xtrsm_kernel_RRTS, xtrsm_kernel_RCTS, +#ifdef EXPRECISION +const openblas_xtrsm_dispatch_t openblas_xtrsm_dispatchTS = { + .xtrsm_kernel_LN = xtrsm_kernel_LNTS, + .xtrsm_kernel_LT = xtrsm_kernel_LTTS, + .xtrsm_kernel_LR = xtrsm_kernel_LRTS, + .xtrsm_kernel_LC = xtrsm_kernel_LCTS, + .xtrsm_kernel_RN = xtrsm_kernel_RNTS, + .xtrsm_kernel_RT = xtrsm_kernel_RTTS, + .xtrsm_kernel_RR = xtrsm_kernel_RRTS, + .xtrsm_kernel_RC = xtrsm_kernel_RCTS, #if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N - xtrsm_iunucopyTS, xtrsm_iunncopyTS, xtrsm_iutucopyTS, xtrsm_iutncopyTS, - xtrsm_ilnucopyTS, xtrsm_ilnncopyTS, xtrsm_iltucopyTS, xtrsm_iltncopyTS, + .xtrsm_iunucopy = xtrsm_iunucopyTS, + .xtrsm_iunncopy = xtrsm_iunncopyTS, + .xtrsm_iutucopy = xtrsm_iutucopyTS, + .xtrsm_iutncopy = xtrsm_iutncopyTS, + .xtrsm_ilnucopy = xtrsm_ilnucopyTS, + .xtrsm_ilnncopy = xtrsm_ilnncopyTS, + .xtrsm_iltucopy = xtrsm_iltucopyTS, + .xtrsm_iltncopy = xtrsm_iltncopyTS, #else - xtrsm_ounucopyTS, xtrsm_ounncopyTS, xtrsm_outucopyTS, xtrsm_outncopyTS, - xtrsm_olnucopyTS, xtrsm_olnncopyTS, xtrsm_oltucopyTS, xtrsm_oltncopyTS, + .xtrsm_iunucopy = xtrsm_ounucopyTS, + .xtrsm_iunncopy = xtrsm_ounncopyTS, + .xtrsm_iutucopy = xtrsm_outucopyTS, + .xtrsm_iutncopy = xtrsm_outncopyTS, + .xtrsm_ilnucopy = xtrsm_olnucopyTS, + .xtrsm_ilnncopy = xtrsm_olnncopyTS, + .xtrsm_iltucopy = xtrsm_oltucopyTS, + .xtrsm_iltncopy = xtrsm_oltncopyTS, +#endif + .xtrsm_ounucopy = xtrsm_ounucopyTS, + .xtrsm_ounncopy = xtrsm_ounncopyTS, + .xtrsm_outucopy = xtrsm_outucopyTS, + .xtrsm_outncopy = xtrsm_outncopyTS, + .xtrsm_olnucopy = xtrsm_olnucopyTS, + .xtrsm_olnncopy = xtrsm_olnncopyTS, + .xtrsm_oltucopy = xtrsm_oltucopyTS, + .xtrsm_oltncopy = xtrsm_oltncopyTS, +}; #endif - xtrsm_ounucopyTS, xtrsm_ounncopyTS, xtrsm_outucopyTS, xtrsm_outncopyTS, - xtrsm_olnucopyTS, xtrsm_olnncopyTS, xtrsm_oltucopyTS, xtrsm_oltncopyTS, - - xtrmm_kernel_RNTS, xtrmm_kernel_RTTS, xtrmm_kernel_RRTS, xtrmm_kernel_RCTS, - xtrmm_kernel_LNTS, xtrmm_kernel_LTTS, xtrmm_kernel_LRTS, xtrmm_kernel_LCTS, +#ifdef EXPRECISION +const openblas_xtrmm_dispatch_t openblas_xtrmm_dispatchTS = { + .xtrmm_kernel_RN = xtrmm_kernel_RNTS, + .xtrmm_kernel_RT = xtrmm_kernel_RTTS, + .xtrmm_kernel_RR = xtrmm_kernel_RRTS, + .xtrmm_kernel_RC = xtrmm_kernel_RCTS, + .xtrmm_kernel_LN = xtrmm_kernel_LNTS, + .xtrmm_kernel_LT = xtrmm_kernel_LTTS, + .xtrmm_kernel_LR = xtrmm_kernel_LRTS, + .xtrmm_kernel_LC = xtrmm_kernel_LCTS, #if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N - xtrmm_iunucopyTS, xtrmm_iunncopyTS, xtrmm_iutucopyTS, xtrmm_iutncopyTS, - xtrmm_ilnucopyTS, xtrmm_ilnncopyTS, xtrmm_iltucopyTS, xtrmm_iltncopyTS, + .xtrmm_iunucopy = xtrmm_iunucopyTS, + .xtrmm_iunncopy = xtrmm_iunncopyTS, + .xtrmm_iutucopy = xtrmm_iutucopyTS, + .xtrmm_iutncopy = xtrmm_iutncopyTS, + .xtrmm_ilnucopy = xtrmm_ilnucopyTS, + .xtrmm_ilnncopy = xtrmm_ilnncopyTS, + .xtrmm_iltucopy = xtrmm_iltucopyTS, + .xtrmm_iltncopy = xtrmm_iltncopyTS, #else - xtrmm_ounucopyTS, xtrmm_ounncopyTS, xtrmm_outucopyTS, xtrmm_outncopyTS, - xtrmm_olnucopyTS, xtrmm_olnncopyTS, xtrmm_oltucopyTS, xtrmm_oltncopyTS, + .xtrmm_iunucopy = xtrmm_ounucopyTS, + .xtrmm_iunncopy = xtrmm_ounncopyTS, + .xtrmm_iutucopy = xtrmm_outucopyTS, + .xtrmm_iutncopy = xtrmm_outncopyTS, + .xtrmm_ilnucopy = xtrmm_olnucopyTS, + .xtrmm_ilnncopy = xtrmm_olnncopyTS, + .xtrmm_iltucopy = xtrmm_oltucopyTS, + .xtrmm_iltncopy = xtrmm_oltncopyTS, +#endif + .xtrmm_ounucopy = xtrmm_ounucopyTS, + .xtrmm_ounncopy = xtrmm_ounncopyTS, + .xtrmm_outucopy = xtrmm_outucopyTS, + .xtrmm_outncopy = xtrmm_outncopyTS, + .xtrmm_olnucopy = xtrmm_olnucopyTS, + .xtrmm_olnncopy = xtrmm_olnncopyTS, + .xtrmm_oltucopy = xtrmm_oltucopyTS, + .xtrmm_oltncopy = xtrmm_oltncopyTS, +}; #endif - xtrmm_ounucopyTS, xtrmm_ounncopyTS, xtrmm_outucopyTS, xtrmm_outncopyTS, - xtrmm_olnucopyTS, xtrmm_olnncopyTS, xtrmm_oltucopyTS, xtrmm_oltncopyTS, +#ifdef EXPRECISION +const openblas_xsymm_dispatch_t openblas_xsymm_dispatchTS = { #if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N - xsymm_iutcopyTS, xsymm_iltcopyTS, + .xsymm_iutcopy = xsymm_iutcopyTS, + .xsymm_iltcopy = xsymm_iltcopyTS, #else - xsymm_outcopyTS, xsymm_oltcopyTS, + .xsymm_iutcopy = xsymm_outcopyTS, + .xsymm_iltcopy = xsymm_oltcopyTS, +#endif + .xsymm_outcopy = xsymm_outcopyTS, + .xsymm_oltcopy = xsymm_oltcopyTS, +}; #endif - xsymm_outcopyTS, xsymm_oltcopyTS, + +#ifdef EXPRECISION +const openblas_xhemm_dispatch_t openblas_xhemm_dispatchTS = { #if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N - xhemm_iutcopyTS, xhemm_iltcopyTS, + .xhemm_iutcopy = xhemm_iutcopyTS, + .xhemm_iltcopy = xhemm_iltcopyTS, #else - xhemm_outcopyTS, xhemm_oltcopyTS, + .xhemm_iutcopy = xhemm_outcopyTS, + .xhemm_iltcopy = xhemm_oltcopyTS, +#endif + .xhemm_outcopy = xhemm_outcopyTS, + .xhemm_oltcopy = xhemm_oltcopyTS, +}; #endif - xhemm_outcopyTS, xhemm_oltcopyTS, - 0, 0, 0, +#ifdef EXPRECISION +const openblas_xgemm3m_dispatch_t openblas_xgemm3m_dispatchTS = { #if (USE_GEMM3M) - QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N, MAX(QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N), - - xgemm3m_kernelTS, - - xgemm3m_incopybTS, xgemm3m_incopyrTS, - xgemm3m_incopyiTS, xgemm3m_itcopybTS, - xgemm3m_itcopyrTS, xgemm3m_itcopyiTS, - xgemm3m_oncopybTS, xgemm3m_oncopyrTS, - xgemm3m_oncopyiTS, xgemm3m_otcopybTS, - xgemm3m_otcopyrTS, xgemm3m_otcopyiTS, - - xsymm3m_iucopybTS, xsymm3m_ilcopybTS, - xsymm3m_iucopyrTS, xsymm3m_ilcopyrTS, - xsymm3m_iucopyiTS, xsymm3m_ilcopyiTS, - xsymm3m_oucopybTS, xsymm3m_olcopybTS, - xsymm3m_oucopyrTS, xsymm3m_olcopyrTS, - xsymm3m_oucopyiTS, xsymm3m_olcopyiTS, - - xhemm3m_iucopybTS, xhemm3m_ilcopybTS, - xhemm3m_iucopyrTS, xhemm3m_ilcopyrTS, - xhemm3m_iucopyiTS, xhemm3m_ilcopyiTS, - - xhemm3m_oucopybTS, xhemm3m_olcopybTS, - xhemm3m_oucopyrTS, xhemm3m_olcopyrTS, - xhemm3m_oucopyiTS, xhemm3m_olcopyiTS, + .xgemm3m_kernel = xgemm3m_kernelTS, + .xgemm3m_incopyb = xgemm3m_incopybTS, + .xgemm3m_incopyr = xgemm3m_incopyrTS, + .xgemm3m_incopyi = xgemm3m_incopyiTS, + .xgemm3m_itcopyb = xgemm3m_itcopybTS, + .xgemm3m_itcopyr = xgemm3m_itcopyrTS, + .xgemm3m_itcopyi = xgemm3m_itcopyiTS, + .xgemm3m_oncopyb = xgemm3m_oncopybTS, + .xgemm3m_oncopyr = xgemm3m_oncopyrTS, + .xgemm3m_oncopyi = xgemm3m_oncopyiTS, + .xgemm3m_otcopyb = xgemm3m_otcopybTS, + .xgemm3m_otcopyr = xgemm3m_otcopyrTS, + .xgemm3m_otcopyi = xgemm3m_otcopyiTS, #else - 0, 0, 0, - - NULL, - - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, + .xgemm3m_kernel = NULL, + .xgemm3m_incopyb = NULL, + .xgemm3m_incopyr = NULL, + .xgemm3m_incopyi = NULL, + .xgemm3m_itcopyb = NULL, + .xgemm3m_itcopyr = NULL, + .xgemm3m_itcopyi = NULL, + .xgemm3m_oncopyb = NULL, + .xgemm3m_oncopyr = NULL, + .xgemm3m_oncopyi = NULL, + .xgemm3m_otcopyb = NULL, + .xgemm3m_otcopyr = NULL, + .xgemm3m_otcopyi = NULL, +#endif +}; +#endif - NULL, NULL, - NULL, NULL, - NULL, NULL, +#ifdef EXPRECISION +const openblas_xsymm3m_dispatch_t openblas_xsymm3m_dispatchTS = { +#if (USE_GEMM3M) + .xsymm3m_iucopyb = xsymm3m_iucopybTS, + .xsymm3m_ilcopyb = xsymm3m_ilcopybTS, + .xsymm3m_iucopyr = xsymm3m_iucopyrTS, + .xsymm3m_ilcopyr = xsymm3m_ilcopyrTS, + .xsymm3m_iucopyi = xsymm3m_iucopyiTS, + .xsymm3m_ilcopyi = xsymm3m_ilcopyiTS, + .xsymm3m_oucopyb = xsymm3m_oucopybTS, + .xsymm3m_olcopyb = xsymm3m_olcopybTS, + .xsymm3m_oucopyr = xsymm3m_oucopyrTS, + .xsymm3m_olcopyr = xsymm3m_olcopyrTS, + .xsymm3m_oucopyi = xsymm3m_oucopyiTS, + .xsymm3m_olcopyi = xsymm3m_olcopyiTS, +#else + .xsymm3m_iucopyb = NULL, + .xsymm3m_ilcopyb = NULL, + .xsymm3m_iucopyr = NULL, + .xsymm3m_ilcopyr = NULL, + .xsymm3m_iucopyi = NULL, + .xsymm3m_ilcopyi = NULL, + .xsymm3m_oucopyb = NULL, + .xsymm3m_olcopyb = NULL, + .xsymm3m_oucopyr = NULL, + .xsymm3m_olcopyr = NULL, + .xsymm3m_oucopyi = NULL, + .xsymm3m_olcopyi = NULL, +#endif +}; +#endif - NULL, NULL, - NULL, NULL, - NULL, NULL, +#ifdef EXPRECISION +const openblas_xhemm3m_dispatch_t openblas_xhemm3m_dispatchTS = { +#if (USE_GEMM3M) + .xhemm3m_iucopyb = xhemm3m_iucopybTS, + .xhemm3m_ilcopyb = xhemm3m_ilcopybTS, + .xhemm3m_iucopyr = xhemm3m_iucopyrTS, + .xhemm3m_ilcopyr = xhemm3m_ilcopyrTS, + .xhemm3m_iucopyi = xhemm3m_iucopyiTS, + .xhemm3m_ilcopyi = xhemm3m_ilcopyiTS, + .xhemm3m_oucopyb = xhemm3m_oucopybTS, + .xhemm3m_olcopyb = xhemm3m_olcopybTS, + .xhemm3m_oucopyr = xhemm3m_oucopyrTS, + .xhemm3m_olcopyr = xhemm3m_olcopyrTS, + .xhemm3m_oucopyi = xhemm3m_oucopyiTS, + .xhemm3m_olcopyi = xhemm3m_olcopyiTS, +#else + .xhemm3m_iucopyb = NULL, + .xhemm3m_ilcopyb = NULL, + .xhemm3m_iucopyr = NULL, + .xhemm3m_ilcopyr = NULL, + .xhemm3m_iucopyi = NULL, + .xhemm3m_ilcopyi = NULL, + .xhemm3m_oucopyb = NULL, + .xhemm3m_olcopyb = NULL, + .xhemm3m_oucopyr = NULL, + .xhemm3m_olcopyr = NULL, + .xhemm3m_oucopyi = NULL, + .xhemm3m_olcopyi = NULL, +#endif +}; #endif +#ifdef EXPRECISION +const openblas_xneg_dispatch_t openblas_xneg_dispatchTS = { #ifndef NO_LAPACK - xneg_tcopyTS, xlaswp_ncopyTS, + .xneg_tcopy = xneg_tcopyTS, #else - NULL, NULL, + .xneg_tcopy = NULL, #endif - +}; #endif - init_parameter, - - SNUMOPT, DNUMOPT, QNUMOPT, -#if BUILD_SINGLE == 1 - saxpby_kTS, +#ifdef EXPRECISION +const openblas_xlaswp_dispatch_t openblas_xlaswp_dispatchTS = { +#ifndef NO_LAPACK + .xlaswp_ncopy = xlaswp_ncopyTS, +#else + .xlaswp_ncopy = NULL, #endif -#if BUILD_DOUBLE == 1 - daxpby_kTS, +}; #endif -#if BUILD_COMPLEX == 1 - caxpby_kTS, + +#if (BUILD_SINGLE==1) +const openblas_saxpby_dispatch_t openblas_saxpby_dispatchTS = { + .saxpby_k = saxpby_kTS, +}; #endif -#if BUILD_COMPLEX16== 1 - zaxpby_kTS, + +#if (BUILD_DOUBLE==1) +const openblas_daxpby_dispatch_t openblas_daxpby_dispatchTS = { + .daxpby_k = daxpby_kTS, +}; #endif -#if BUILD_SINGLE == 1 - somatcopy_k_cnTS, somatcopy_k_ctTS, somatcopy_k_rnTS, somatcopy_k_rtTS, +#if (BUILD_COMPLEX==1) +const openblas_caxpby_dispatch_t openblas_caxpby_dispatchTS = { + .caxpby_k = caxpby_kTS, +}; #endif -#if BUILD_DOUBLE== 1 - domatcopy_k_cnTS, domatcopy_k_ctTS, domatcopy_k_rnTS, domatcopy_k_rtTS, + +#if (BUILD_COMPLEX16==1) +const openblas_zaxpby_dispatch_t openblas_zaxpby_dispatchTS = { + .zaxpby_k = zaxpby_kTS, +}; #endif -#if BUILD_COMPLEX == 1 - comatcopy_k_cnTS, comatcopy_k_ctTS, comatcopy_k_rnTS, comatcopy_k_rtTS, - comatcopy_k_cncTS, comatcopy_k_ctcTS, comatcopy_k_rncTS, comatcopy_k_rtcTS, + +#if (BUILD_SINGLE==1) +const openblas_somatcopy_dispatch_t openblas_somatcopy_dispatchTS = { + .somatcopy_k_cn = somatcopy_k_cnTS, + .somatcopy_k_ct = somatcopy_k_ctTS, + .somatcopy_k_rn = somatcopy_k_rnTS, + .somatcopy_k_rt = somatcopy_k_rtTS, +}; #endif -#if BUILD_COMPLEX16 == 1 - zomatcopy_k_cnTS, zomatcopy_k_ctTS, zomatcopy_k_rnTS, zomatcopy_k_rtTS, - zomatcopy_k_cncTS, zomatcopy_k_ctcTS, zomatcopy_k_rncTS, zomatcopy_k_rtcTS, + +#if (BUILD_DOUBLE==1) +const openblas_domatcopy_dispatch_t openblas_domatcopy_dispatchTS = { + .domatcopy_k_cn = domatcopy_k_cnTS, + .domatcopy_k_ct = domatcopy_k_ctTS, + .domatcopy_k_rn = domatcopy_k_rnTS, + .domatcopy_k_rt = domatcopy_k_rtTS, +}; #endif -#if BUILD_SINGLE == 1 - simatcopy_k_cnTS, simatcopy_k_ctTS, simatcopy_k_rnTS, simatcopy_k_rtTS, +#if (BUILD_COMPLEX==1) +const openblas_comatcopy_dispatch_t openblas_comatcopy_dispatchTS = { + .comatcopy_k_cn = comatcopy_k_cnTS, + .comatcopy_k_ct = comatcopy_k_ctTS, + .comatcopy_k_rn = comatcopy_k_rnTS, + .comatcopy_k_rt = comatcopy_k_rtTS, + .comatcopy_k_cnc = comatcopy_k_cncTS, + .comatcopy_k_ctc = comatcopy_k_ctcTS, + .comatcopy_k_rnc = comatcopy_k_rncTS, + .comatcopy_k_rtc = comatcopy_k_rtcTS, +}; #endif -#if BUILD_DOUBLE== 1 - dimatcopy_k_cnTS, dimatcopy_k_ctTS, dimatcopy_k_rnTS, dimatcopy_k_rtTS, + +#if (BUILD_COMPLEX16==1) +const openblas_zomatcopy_dispatch_t openblas_zomatcopy_dispatchTS = { + .zomatcopy_k_cn = zomatcopy_k_cnTS, + .zomatcopy_k_ct = zomatcopy_k_ctTS, + .zomatcopy_k_rn = zomatcopy_k_rnTS, + .zomatcopy_k_rt = zomatcopy_k_rtTS, + .zomatcopy_k_cnc = zomatcopy_k_cncTS, + .zomatcopy_k_ctc = zomatcopy_k_ctcTS, + .zomatcopy_k_rnc = zomatcopy_k_rncTS, + .zomatcopy_k_rtc = zomatcopy_k_rtcTS, +}; #endif -#if BUILD_COMPLEX== 1 - cimatcopy_k_cnTS, cimatcopy_k_ctTS, cimatcopy_k_rnTS, cimatcopy_k_rtTS, - cimatcopy_k_cncTS, cimatcopy_k_ctcTS, cimatcopy_k_rncTS, cimatcopy_k_rtcTS, + +#if (BUILD_SINGLE==1) +const openblas_simatcopy_dispatch_t openblas_simatcopy_dispatchTS = { + .simatcopy_k_cn = simatcopy_k_cnTS, + .simatcopy_k_ct = simatcopy_k_ctTS, + .simatcopy_k_rn = simatcopy_k_rnTS, + .simatcopy_k_rt = simatcopy_k_rtTS, +}; #endif -#if BUILD_COMPLEX16==1 - zimatcopy_k_cnTS, zimatcopy_k_ctTS, zimatcopy_k_rnTS, zimatcopy_k_rtTS, - zimatcopy_k_cncTS, zimatcopy_k_ctcTS, zimatcopy_k_rncTS, zimatcopy_k_rtcTS, + +#if (BUILD_DOUBLE==1) +const openblas_dimatcopy_dispatch_t openblas_dimatcopy_dispatchTS = { + .dimatcopy_k_cn = dimatcopy_k_cnTS, + .dimatcopy_k_ct = dimatcopy_k_ctTS, + .dimatcopy_k_rn = dimatcopy_k_rnTS, + .dimatcopy_k_rt = dimatcopy_k_rtTS, +}; #endif -#if BUILD_SINGLE == 1 - sgeadd_kTS, +#if (BUILD_COMPLEX==1) +const openblas_cimatcopy_dispatch_t openblas_cimatcopy_dispatchTS = { + .cimatcopy_k_cn = cimatcopy_k_cnTS, + .cimatcopy_k_ct = cimatcopy_k_ctTS, + .cimatcopy_k_rn = cimatcopy_k_rnTS, + .cimatcopy_k_rt = cimatcopy_k_rtTS, + .cimatcopy_k_cnc = cimatcopy_k_cncTS, + .cimatcopy_k_ctc = cimatcopy_k_ctcTS, + .cimatcopy_k_rnc = cimatcopy_k_rncTS, + .cimatcopy_k_rtc = cimatcopy_k_rtcTS, +}; #endif -#if BUILD_DOUBLE==1 - dgeadd_kTS, + +#if (BUILD_COMPLEX16==1) +const openblas_zimatcopy_dispatch_t openblas_zimatcopy_dispatchTS = { + .zimatcopy_k_cn = zimatcopy_k_cnTS, + .zimatcopy_k_ct = zimatcopy_k_ctTS, + .zimatcopy_k_rn = zimatcopy_k_rnTS, + .zimatcopy_k_rt = zimatcopy_k_rtTS, + .zimatcopy_k_cnc = zimatcopy_k_cncTS, + .zimatcopy_k_ctc = zimatcopy_k_ctcTS, + .zimatcopy_k_rnc = zimatcopy_k_rncTS, + .zimatcopy_k_rtc = zimatcopy_k_rtcTS, +}; #endif -#if BUILD_COMPLEX==1 - cgeadd_kTS, + +#if (BUILD_SINGLE==1) +const openblas_sgeadd_dispatch_t openblas_sgeadd_dispatchTS = { + .sgeadd_k = sgeadd_kTS, +}; #endif -#if BUILD_COMPLEX16==1 - zgeadd_kTS, + +#if (BUILD_DOUBLE==1) +const openblas_dgeadd_dispatch_t openblas_dgeadd_dispatchTS = { + .dgeadd_k = dgeadd_kTS, +}; +#endif + +#if (BUILD_COMPLEX==1) +const openblas_cgeadd_dispatch_t openblas_cgeadd_dispatchTS = { + .cgeadd_k = cgeadd_kTS, +}; #endif + +#if (BUILD_COMPLEX16==1) +const openblas_zgeadd_dispatch_t openblas_zgeadd_dispatchTS = { + .zgeadd_k = zgeadd_kTS, }; +#endif #if (ARCH_ARM64) static void init_parameter(void) {